Revision d2db2790

b/lib/hypervisor/hv_kvm.py
37 37
import socket
38 38
import stat
39 39
import StringIO
40
import fdsend
41
import copy
42
from bitarray import bitarray
40 43
try:
41 44
  import affinity   # pylint: disable=F0401
42 45
except ImportError:
......
79 82
  constants.HV_KVM_SPICE_USE_TLS,
80 83
  ])
81 84

  
85
FREE = bitarray("0")
86

  
87
def UUIDToKVMId(uuid):
88

  
89
  if uuid:
90
    return "x" + uuid.split("-")[0]
91
  else:
92
    return None
82 93

  
83 94
def _ProbeTapVnetHdr(fd):
84 95
  """Check whether to enable the IFF_VNET_HDR flag.
......
554 565
  # different than -drive is starting)
555 566
  _BOOT_RE = re.compile(r"^-drive\s([^-]|(?<!^)-)*,boot=on\|off", re.M | re.S)
556 567

  
568
  _INFO_PCI_RE = re.compile(r'Bus.*device[ ]*(\d+).*')
569
  _INFO_PCI_CMD = "info pci"
570
  _INFO_VERSION_RE = re.compile(r'^QEMU (\d+)\.(\d+)(\.(\d+))?.*monitor.*', re.M)
571
  _INFO_VERSION_CMD = "info version"
572

  
573
  _DEFAULT_PCI_RESERVATIONS = "11110000000000000000000000000000"
574

  
557 575
  ANCILLARY_FILES = [
558 576
    _KVM_NETWORK_SCRIPT,
559 577
    ]
......
1002 1020
        data.append(info)
1003 1021
    return data
1004 1022

  
1023
  def _GenerateKVMBlockDevicesOptions(self, instance, kvm_cmd, block_devices,
1024
                                      pci_reservations, kvmhelp):
1025

  
1026
    hvp = instance.hvparams
1027
    boot_disk = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
1028

  
1029
    # whether this is an older KVM version that uses the boot=on flag
1030
    # on devices
1031
    needs_boot_flag = self._BOOT_RE.search(kvmhelp)
1032

  
1033
    disk_type = hvp[constants.HV_DISK_TYPE]
1034
    if disk_type == constants.HT_DISK_PARAVIRTUAL:
1035
      if_val = ",if=virtio"
1036
      #TODO: parse kvm -device ? output
1037
      disk_model = "virtio-blk-pci"
1038
    else:
1039
      if_val = ",if=%s" % disk_type
1040
      disk_model = disk_type
1041
    # Cache mode
1042
    disk_cache = hvp[constants.HV_DISK_CACHE]
1043
    if instance.disk_template in constants.DTS_EXT_MIRROR:
1044
      if disk_cache != "none":
1045
        # TODO: make this a hard error, instead of a silent overwrite
1046
        logging.warning("KVM: overriding disk_cache setting '%s' with 'none'"
1047
                        " to prevent shared storage corruption on migration",
1048
                        disk_cache)
1049
      cache_val = ",cache=none"
1050
    elif disk_cache != constants.HT_CACHE_DEFAULT:
1051
      cache_val = ",cache=%s" % disk_cache
1052
    else:
1053
      cache_val = ""
1054
    for cfdev, dev_path in block_devices:
1055
      uuid = UUIDToKVMId(cfdev.uuid)
1056
      if cfdev.mode != constants.DISK_RDWR:
1057
        raise errors.HypervisorError("Instance has read-only disks which"
1058
                                     " are not supported by KVM")
1059
      # TODO: handle FD_LOOP and FD_BLKTAP (?)
1060
      boot_val = ""
1061
      if boot_disk:
1062
        kvm_cmd.extend(["-boot", "c"])
1063
        boot_disk = False
1064
        if needs_boot_flag and disk_type != constants.HT_DISK_IDE:
1065
          boot_val = ",boot=on"
1066
      drive_val = "file=%s,format=raw%s%s" % \
1067
                  (dev_path, boot_val, cache_val)
1068
      if cfdev.uuid is not None:
1069
        #TODO: name id after model
1070
        drive_val += (",if=none,id=%s" % uuid)
1071
        if cfdev.pci is None:
1072
          cfdev.pci = self._GetFreePCISlot(instance, pci_reservations,
1073
                                           live=False)
1074
        drive_val += (",bus=0,unit=%d" % cfdev.pci)
1075
      else:
1076
        drive_val += if_val
1077

  
1078
      kvm_cmd.extend(["-drive", drive_val])
1079

  
1080
      if cfdev.uuid is not None:
1081
        dev_val = ("%s,drive=%s,id=%s" %
1082
                    (disk_model, uuid, uuid))
1083
        if cfdev.pci is None:
1084
          cfdev.pci = self._GetFreePCISlot(instance, pci_reservations,
1085
                                           live=False)
1086
        dev_val += ",bus=pci.0,addr=%s" % hex(cfdev.pci)
1087
        kvm_cmd.extend(["-device", dev_val])
1088

  
1089
    return kvm_cmd
1090

  
1005 1091
  def _GenerateKVMRuntime(self, instance, block_devices, startup_paused,
1006 1092
                          kvmhelp):
1007 1093
    """Generate KVM information to start an instance.
......
1052 1138

  
1053 1139
    kernel_path = hvp[constants.HV_KERNEL_PATH]
1054 1140
    if kernel_path:
1055
      boot_disk = boot_cdrom = boot_floppy = boot_network = False
1141
      boot_cdrom = boot_floppy = boot_network = False
1056 1142
    else:
1057
      boot_disk = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
1058 1143
      boot_cdrom = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_CDROM
1059 1144
      boot_floppy = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_FLOPPY
1060 1145
      boot_network = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_NETWORK
......
1079 1164
    needs_boot_flag = self._BOOT_RE.search(kvmhelp)
1080 1165

  
1081 1166
    disk_type = hvp[constants.HV_DISK_TYPE]
1082
    if disk_type == constants.HT_DISK_PARAVIRTUAL:
1083
      if_val = ",if=virtio"
1084
    else:
1085
      if_val = ",if=%s" % disk_type
1086
    # Cache mode
1087
    disk_cache = hvp[constants.HV_DISK_CACHE]
1088
    if instance.disk_template in constants.DTS_EXT_MIRROR:
1089
      if disk_cache != "none":
1090
        # TODO: make this a hard error, instead of a silent overwrite
1091
        logging.warning("KVM: overriding disk_cache setting '%s' with 'none'"
1092
                        " to prevent shared storage corruption on migration",
1093
                        disk_cache)
1094
      cache_val = ",cache=none"
1095
    elif disk_cache != constants.HT_CACHE_DEFAULT:
1096
      cache_val = ",cache=%s" % disk_cache
1097
    else:
1098
      cache_val = ""
1099
    for cfdev, dev_path in block_devices:
1100
      if cfdev.mode != constants.DISK_RDWR:
1101
        raise errors.HypervisorError("Instance has read-only disks which"
1102
                                     " are not supported by KVM")
1103
      # TODO: handle FD_LOOP and FD_BLKTAP (?)
1104
      boot_val = ""
1105
      if boot_disk:
1106
        kvm_cmd.extend(["-boot", "c"])
1107
        boot_disk = False
1108
        if needs_boot_flag and disk_type != constants.HT_DISK_IDE:
1109
          boot_val = ",boot=on"
1110

  
1111
      drive_val = "file=%s,format=raw%s%s%s" % (dev_path, if_val, boot_val,
1112
                                                cache_val)
1113
      kvm_cmd.extend(["-drive", drive_val])
1114 1167

  
1115 1168
    #Now we can specify a different device type for CDROM devices.
1116 1169
    cdrom_disk_type = hvp[constants.HV_KVM_CDROM_DISK_TYPE]
......
1360 1413
    kvm_nics = instance.nics
1361 1414
    hvparams = hvp
1362 1415

  
1363
    return (kvm_cmd, kvm_nics, hvparams)
1416
    return (kvm_cmd, kvm_nics, hvparams, block_devices)
1364 1417

  
1365 1418
  def _WriteKVMRuntime(self, instance_name, data):
1366 1419
    """Write an instance's KVM runtime
......
1386 1439
    """Save an instance's KVM runtime
1387 1440

  
1388 1441
    """
1389
    kvm_cmd, kvm_nics, hvparams = kvm_runtime
1442
    kvm_cmd, kvm_nics, hvparams, block_devices = kvm_runtime
1443

  
1390 1444
    serialized_nics = [nic.ToDict() for nic in kvm_nics]
1391
    serialized_form = serializer.Dump((kvm_cmd, serialized_nics, hvparams))
1445
    serialized_blockdevs = [(blk.ToDict(), link) for blk, link in block_devices]
1446
    serialized_form = serializer.Dump((kvm_cmd, serialized_nics, hvparams,
1447
                                      serialized_blockdevs))
1448

  
1392 1449
    self._WriteKVMRuntime(instance.name, serialized_form)
1393 1450

  
1394 1451
  def _LoadKVMRuntime(self, instance, serialized_runtime=None):
......
1397 1454
    """
1398 1455
    if not serialized_runtime:
1399 1456
      serialized_runtime = self._ReadKVMRuntime(instance.name)
1457

  
1400 1458
    loaded_runtime = serializer.Load(serialized_runtime)
1401
    kvm_cmd, serialized_nics, hvparams = loaded_runtime
1459
    kvm_cmd, serialized_nics, hvparams, serialized_blockdevs = loaded_runtime
1460

  
1402 1461
    kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics]
1403
    return (kvm_cmd, kvm_nics, hvparams)
1462
    block_devices = [(objects.Disk.FromDict(sdisk), link)
1463
                     for sdisk, link in serialized_blockdevs]
1464

  
1465
    return (kvm_cmd, kvm_nics, hvparams, block_devices)
1404 1466

  
1405 1467
  def _RunKVMCmd(self, name, kvm_cmd, tap_fds=None):
1406 1468
    """Run the KVM cmd and check for errors
......
1425 1487
    if not self._InstancePidAlive(name)[2]:
1426 1488
      raise errors.HypervisorError("Failed to start instance %s" % name)
1427 1489

  
1490
  # pylint: disable=R0914
1428 1491
  def _ExecuteKVMRuntime(self, instance, kvm_runtime, kvmhelp, incoming=None):
1429 1492
    """Execute a KVM cmd, after completing it with some last minute data.
1430 1493

  
......
1448 1511

  
1449 1512
    temp_files = []
1450 1513

  
1451
    kvm_cmd, kvm_nics, up_hvp = kvm_runtime
1514
    kvm_cmd, kvm_nics, up_hvp, block_devices = kvm_runtime
1452 1515
    # the first element of kvm_cmd is always the path to the kvm binary
1453 1516
    kvm_path = kvm_cmd[0]
1517

  
1518
    kvm_cmd_runtime = copy.deepcopy(kvm_cmd)
1519

  
1454 1520
    up_hvp = objects.FillDict(conf_hvp, up_hvp)
1455 1521

  
1456 1522
    # We know it's safe to run as a different user upon migration, so we'll use
......
1469 1535
      utils.WriteFile(keymap_path, data="include en-us\ninclude %s\n" % keymap)
1470 1536
      kvm_cmd.extend(["-k", keymap_path])
1471 1537

  
1538
    pci_reservations = bitarray(self._DEFAULT_PCI_RESERVATIONS)
1539

  
1540
    kvm_cmd = self._GenerateKVMBlockDevicesOptions(instance, kvm_cmd,
1541
                                                   block_devices,
1542
                                                   pci_reservations,
1543
                                                   kvmhelp)
1544

  
1472 1545
    # We have reasons to believe changing something like the nic driver/type
1473 1546
    # upon migration won't exactly fly with the instance kernel, so for nic
1474 1547
    # related parameters we'll use up_hvp
......
1505 1578
      kvm_supports_netdev = self._NETDEV_RE.search(kvmhelp)
1506 1579

  
1507 1580
      for nic_seq, nic in enumerate(kvm_nics):
1581
        uuid = UUIDToKVMId(nic.uuid)
1508 1582
        tapname, tapfd = _OpenTap(vnet_hdr=vnet_hdr)
1509 1583
        tapfds.append(tapfd)
1510 1584
        taps.append(tapname)
1511 1585
        if kvm_supports_netdev:
1512
          nic_val = "%s,mac=%s,netdev=netdev%s" % (nic_model, nic.mac, nic_seq)
1513
          tap_val = "type=tap,id=netdev%s,fd=%d%s" % (nic_seq, tapfd, tap_extra)
1586
          nic_val = "%s,mac=%s" % (nic_model, nic.mac)
1587
          if nic.uuid is not None:
1588
            nic_val += (",netdev=%s,id=%s" %
1589
                        (uuid, uuid))
1590
            if nic.pci is None:
1591
              nic.pci = self._GetFreePCISlot(instance, pci_reservations,
1592
                                             live=False)
1593
            nic_val += (",bus=pci.0,addr=%s" % hex(nic.pci))
1594
          else:
1595
            nic_val += (",netdev=netdev%d,id=virtio-net-pci.%d" %
1596
                        (nic_seq, nic_seq))
1597
          tap_val = ("type=tap,id=%s,fd=%d%s" %
1598
                     (uuid or nic_seq, tapfd, tap_extra))
1514 1599
          kvm_cmd.extend(["-netdev", tap_val, "-device", nic_val])
1515 1600
        else:
1516 1601
          nic_val = "nic,vlan=%s,macaddr=%s,model=%s" % (nic_seq,
......
1633 1718
      # explicitly requested resume the vm status.
1634 1719
      self._CallMonitorCommand(instance.name, self._CONT_CMD)
1635 1720

  
1721
    kvm_runtime_with_pci_info = (kvm_cmd_runtime, kvm_nics,
1722
                                 up_hvp, block_devices)
1723
    return kvm_runtime_with_pci_info
1724

  
1636 1725
  def StartInstance(self, instance, block_devices, startup_paused):
1637 1726
    """Start an instance.
1638 1727

  
......
1643 1732
    kvm_runtime = self._GenerateKVMRuntime(instance, block_devices,
1644 1733
                                           startup_paused, kvmhelp)
1645 1734
    self._SaveKVMRuntime(instance, kvm_runtime)
1646
    self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp)
1735
    kvm_runtime_with_pci_info = self._ExecuteKVMRuntime(instance, kvm_runtime,
1736
                                                        kvmhelp)
1737
    self._SaveKVMRuntime(instance, kvm_runtime_with_pci_info)
1647 1738

  
1648 1739
  def _CallMonitorCommand(self, instance_name, command):
1649 1740
    """Invoke a command on the instance monitor.
......
1663 1754

  
1664 1755
    return result
1665 1756

  
1757
  def _GetFreePCISlot(self, instance, pci_reservations=None, live=False):
1758
    """Get the first available pci slot.
1759

  
1760
    If the instance is running then use info pci monitor command.
1761
    If not then use pci_reservations passed as argument.
1762
    """
1763
    if live:
1764
      slots = bitarray(32)
1765
      slots.setall(False) # pylint: disable=E1101
1766
      output = self._CallMonitorCommand(instance.name, self._INFO_PCI_CMD)
1767
      for line in output.stdout.splitlines():
1768
        match = self._INFO_PCI_RE.search(line)
1769
        if match:
1770
          slot = int(match.group(1))
1771
          slots[slot] = True
1772
    else:
1773
      slots = pci_reservations
1774

  
1775
    [free] = slots.search(FREE, 1) # pylint: disable=E1103
1776
    if not free:
1777
      raise errors.HypervisorError("All PCI slots occupied")
1778

  
1779
    slots[free] = True
1780

  
1781
    return int(free)
1782

  
1783
  def _TryHotplug(self, instance_name):
1784
    """Get QEMU version from the instance's monitor.
1785

  
1786
    Hotplug is supported for running instances and for versions >= 1.0.
1787
    """
1788
    output = self._CallMonitorCommand(instance_name, self._INFO_VERSION_CMD)
1789
    match = self._INFO_VERSION_RE.search(output.stdout)
1790
    if not match:
1791
      return False
1792
    v_major, v_min, _, _ = match.groups()
1793
    return (v_major, v_min) >= (1, 0)
1794

  
1795
  def HotAddDisk(self, instance, disk, dev_path, _):
1796
    """Hotadd new disk to the VM
1797

  
1798
    """
1799
    if self._TryHotplug(instance.name):
1800
      uuid = UUIDToKVMId(disk.uuid)
1801

  
1802
      if disk.pci is None:
1803
        disk.pci = self._GetFreePCISlot(instance, live=True)
1804
      command = ("drive_add dummy file=%s,if=none,id=%s,format=raw" %
1805
                 (dev_path, uuid))
1806

  
1807
      logging.info("Run cmd %s", command)
1808
      output = self._CallMonitorCommand(instance.name, command)
1809

  
1810
      command = ("device_add virtio-blk-pci,bus=pci.0,addr=%s,"
1811
                 "drive=%s,id=%s"
1812
                 % (hex(disk.pci), uuid, uuid))
1813
      logging.info("Run cmd %s", command)
1814
      output = self._CallMonitorCommand(instance.name, command)
1815
      for line in output.stdout.splitlines():
1816
        logging.info("%s", line)
1817

  
1818
      (kvm_cmd, kvm_nics,
1819
       hvparams, block_devices) = self._LoadKVMRuntime(instance)
1820
      block_devices.append((disk, dev_path))
1821
      new_kvm_runtime = (kvm_cmd, kvm_nics, hvparams, block_devices)
1822
      self._SaveKVMRuntime(instance, new_kvm_runtime)
1823

  
1824
  def HotDelDisk(self, instance, disk, _):
1825
    """Hotdel disk to the VM
1826

  
1827
    """
1828
    if self._TryHotplug(instance.name):
1829

  
1830
      uuid = UUIDToKVMId(disk.uuid)
1831

  
1832
      command = "device_del %s" % uuid
1833
      logging.info("Run cmd %s", command)
1834
      output = self._CallMonitorCommand(instance.name, command)
1835
      for line in output.stdout.splitlines():
1836
        logging.info("%s", line)
1837

  
1838
      command = "drive_del %s" % uuid
1839
      logging.info("Run cmd %s", command)
1840
      #output = self._CallMonitorCommand(instance.name, command)
1841
      #for line in output.stdout.splitlines():
1842
      #  logging.info("%s" % line)
1843

  
1844
      (kvm_cmd, kvm_nics,
1845
       hvparams, block_devices) = self._LoadKVMRuntime(instance)
1846
      rem  = [(d, p) for d, p in block_devices
1847
                     if d.uuid == disk.uuid]
1848
      try:
1849
        block_devices.remove(rem[0])
1850
      except (ValueError, IndexError):
1851
        logging.info("Disk with uuid %s disappeared from runtime file", disk.uuid)
1852
      new_kvm_runtime = (kvm_cmd, kvm_nics, hvparams, block_devices)
1853
      self._SaveKVMRuntime(instance, new_kvm_runtime)
1854

  
1855
  def HotAddNic(self, instance, nic, seq):
1856
    """Hotadd new nic to the VM
1857

  
1858
    """
1859
    if self._TryHotplug(instance.name):
1860
      if nic.pci is None:
1861
        nic.pci = self._GetFreePCISlot(instance, live=True)
1862
      mac = nic.mac
1863

  
1864
      uuid = UUIDToKVMId(nic.uuid)
1865

  
1866
      (tap, fd) = _OpenTap()
1867
      logging.info("%s %d", tap, fd)
1868

  
1869
      self._PassTapFd(instance, fd, nic)
1870

  
1871
      command = ("netdev_add tap,id=%s,fd=%s"
1872
                 % (uuid, uuid))
1873
      logging.info("Run cmd %s", command)
1874
      output = self._CallMonitorCommand(instance.name, command)
1875
      for line in output.stdout.splitlines():
1876
        logging.info("%s", line)
1877

  
1878
      command = ("device_add virtio-net-pci,bus=pci.0,addr=%s,mac=%s,"
1879
                 "netdev=%s,id=%s"
1880
                 % (hex(nic.pci), mac, uuid, uuid))
1881
      logging.info("Run cmd %s", command)
1882
      output = self._CallMonitorCommand(instance.name, command)
1883
      for line in output.stdout.splitlines():
1884
        logging.info("%s", line)
1885

  
1886
      self._ConfigureNIC(instance, seq, nic, tap)
1887

  
1888
      (kvm_cmd, kvm_nics,
1889
       hvparams, block_devices) = self._LoadKVMRuntime(instance)
1890
      kvm_nics.append(nic)
1891
      new_kvm_runtime = (kvm_cmd, kvm_nics, hvparams, block_devices)
1892
      self._SaveKVMRuntime(instance, new_kvm_runtime)
1893

  
1894
  def HotDelNic(self, instance, nic, _):
1895
    """Hotadd new nic to the VM
1896

  
1897
    """
1898
    if self._TryHotplug(instance.name):
1899

  
1900
      uuid = UUIDToKVMId(nic.uuid)
1901

  
1902
      command = "device_del %s" % uuid
1903
      logging.info("Run cmd %s", command)
1904
      output = self._CallMonitorCommand(instance.name, command)
1905
      for line in output.stdout.splitlines():
1906
        logging.info("%s", line)
1907

  
1908
      command = "netdev_del %s" % uuid
1909
      logging.info("Run cmd %s", command)
1910
      output = self._CallMonitorCommand(instance.name, command)
1911
      for line in output.stdout.splitlines():
1912
        logging.info("%s", line)
1913

  
1914
      (kvm_cmd, kvm_nics,
1915
       hvparams, block_devices) = self._LoadKVMRuntime(instance)
1916
      rem  = [n for n in kvm_nics if n.uuid == nic.uuid]
1917
      try:
1918
        kvm_nics.remove(rem[0])
1919
      except (ValueError, IndexError):
1920
        logging.info("NIC with uuid %s disappeared from runtime file", nic.uuid)
1921
      new_kvm_runtime = (kvm_cmd, kvm_nics, hvparams, block_devices)
1922
      self._SaveKVMRuntime(instance, new_kvm_runtime)
1923

  
1924
  def _PassTapFd(self, instance, fd, nic):
1925
    monsock = utils.ShellQuote(self._InstanceMonitor(instance.name))
1926
    s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
1927
    s.connect(monsock)
1928
    uuid = UUIDToKVMId(nic.uuid)
1929
    command = "getfd %s\n" % uuid
1930
    fds = [fd]
1931
    logging.info("%s", fds)
1932
    fdsend.sendfds(s, command, fds = fds)
1933
    s.close()
1934

  
1666 1935
  @classmethod
1667 1936
  def _ParseKVMVersion(cls, text):
1668 1937
    """Parse the KVM version from the --help output.
......
1778 2047
    self._SaveKVMRuntime(instance, kvm_runtime)
1779 2048
    kvmpath = instance.hvparams[constants.HV_KVM_PATH]
1780 2049
    kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
1781
    self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp)
2050
    kvm_runtime_with_pci_info = \
2051
      self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp)
2052
    self._SaveKVMRuntime(instance, kvm_runtime_with_pci_info)
1782 2053

  
1783 2054
  def MigrationInfo(self, instance):
1784 2055
    """Get instance information to perform a migration.

Also available in: Unified diff