(2.10) KVM: use running HVPs to calc blockdev options
[ganeti-local] / lib / hypervisor / hv_kvm.py
index 50123ce..de15a89 100644 (file)
@@ -60,7 +60,7 @@ from ganeti.hypervisor import hv_base
 from ganeti.utils import wrapper as utils_wrapper
 
 
-_KVM_NETWORK_SCRIPT = pathutils.CONF_DIR + "/kvm-vif-bridge"
+_KVM_NETWORK_SCRIPT = pathutils.CONF_DIR + "/kvm-ifup-custom"
 _KVM_START_PAUSED_FLAG = "-S"
 
 # TUN/TAP driver constants, taken from <linux/if_tun.h>
@@ -114,6 +114,8 @@ _RUNTIME_ENTRY = {
   constants.HOTPLUG_TARGET_DISK: lambda d, e: (d, e)
   }
 
+_MIGRATION_CAPS_DELIM = ":"
+
 
 def _GenerateDeviceKVMId(dev_type, dev, idx=None):
   """Helper function to generate a unique device name used by KVM
@@ -143,29 +145,36 @@ def _GenerateDeviceKVMId(dev_type, dev, idx=None):
                             " without UUID or PCI info")
 
 
-def _UpdatePCISlots(dev, pci_reservations):
-  """Update pci configuration for a stopped instance
-
-  If dev has a pci slot then reserve it, else find first available
-  in pci_reservations bitarray. It acts on the same objects passed
-  as params so there is no need to return anything.
+def _GetFreeSlot(slots, slot=None, reserve=False):
+  """Helper method to get first available slot in a bitarray
 
-  @type dev: L{objects.Disk} or L{objects.NIC}
-  @param dev: the device object for which we update its pci slot
-  @type pci_reservations: bitarray
-  @param pci_reservations: existing pci reservations for an instance
-  @raise errors.HotplugError: in case an instance has all its slot occupied
+  @type slots: bitarray
+  @param slots: the bitarray to operate on
+  @type slot: integer
+  @param slot: if given we check whether the slot is free
+  @type reserve: boolean
+  @param reserve: whether to reserve the first available slot or not
+  @return: the idx of the (first) available slot
+  @raise errors.HotplugError: If all slots in a bitarray are occupied
+    or the given slot is not free.
 
   """
-  if dev.pci:
-    free = dev.pci
-  else: # pylint: disable=E1103
-    [free] = pci_reservations.search(_AVAILABLE_PCI_SLOT, 1)
-    if not free:
-      raise errors.HypervisorError("All PCI slots occupied")
-    dev.pci = int(free)
+  if slot is not None:
+    assert slot < len(slots)
+    if slots[slot]:
+      raise errors.HypervisorError("Slots %d occupied" % slot)
+
+  else:
+    avail = slots.search(_AVAILABLE_PCI_SLOT, 1)
+    if not avail:
+      raise errors.HypervisorError("All slots occupied")
+
+    slot = int(avail[0])
+
+  if reserve:
+    slots[slot] = True
 
-  pci_reservations[free] = True
+  return slot
 
 
 def _GetExistingDeviceInfo(dev_type, device, runtime):
@@ -456,7 +465,6 @@ class QmpConnection(MonitorSocket):
   _RETURN_KEY = RETURN_KEY = "return"
   _ACTUAL_KEY = ACTUAL_KEY = "actual"
   _ERROR_CLASS_KEY = "class"
-  _ERROR_DATA_KEY = "data"
   _ERROR_DESC_KEY = "desc"
   _EXECUTE_KEY = "execute"
   _ARGUMENTS_KEY = "arguments"
@@ -486,6 +494,10 @@ class QmpConnection(MonitorSocket):
       raise errors.HypervisorError("kvm: QMP communication error (wrong"
                                    " server greeting")
 
+    # This is needed because QMP can return more than one greetings
+    # see https://groups.google.com/d/msg/ganeti-devel/gZYcvHKDooU/SnukC8dgS5AJ
+    self._buf = ""
+
     # Let's put the monitor in command mode using the qmp_capabilities
     # command, or else no command will be executable.
     # (As per the QEMU Protocol Specification 0.1 - section 4)
@@ -600,11 +612,10 @@ class QmpConnection(MonitorSocket):
       err = response[self._ERROR_KEY]
       if err:
         raise errors.HypervisorError("kvm: error executing the %s"
-                                     " command: %s (%s, %s):" %
+                                     " command: %s (%s):" %
                                      (command,
                                       err[self._ERROR_DESC_KEY],
-                                      err[self._ERROR_CLASS_KEY],
-                                      err[self._ERROR_DATA_KEY]))
+                                      err[self._ERROR_CLASS_KEY]))
 
       elif not response[self._EVENT_KEY]:
         return response
@@ -693,6 +704,7 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     constants.HV_MIGRATION_BANDWIDTH: hv_base.REQ_NONNEGATIVE_INT_CHECK,
     constants.HV_MIGRATION_DOWNTIME: hv_base.REQ_NONNEGATIVE_INT_CHECK,
     constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK,
+    constants.HV_KVM_MIGRATION_CAPS: hv_base.NO_CHECK,
     constants.HV_USE_LOCALTIME: hv_base.NO_CHECK,
     constants.HV_DISK_CACHE:
       hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES),
@@ -760,11 +772,18 @@ class KVMHypervisor(hv_base.BaseHypervisor):
 
   _INFO_PCI_RE = re.compile(r'Bus.*device[ ]*(\d+).*')
   _INFO_PCI_CMD = "info pci"
+  _FIND_PCI_DEVICE_RE = \
+    staticmethod(lambda pci, devid:
+      re.compile(r'Bus.*device[ ]*%d,(.*\n){5,6}.*id "%s"' % (pci, devid),
+                 re.M))
+
   _INFO_VERSION_RE = \
     re.compile(r'^QEMU (\d+)\.(\d+)(\.(\d+))?.*monitor.*', re.M)
   _INFO_VERSION_CMD = "info version"
 
-  _DEFAULT_PCI_RESERVATIONS = "11110000000000000000000000000000"
+  # Slot 0 for Host bridge, Slot 1 for ISA bridge, Slot 2 for VGA controller
+  _DEFAULT_PCI_RESERVATIONS = "11100000000000000000000000000000"
+  _SOUNDHW_WITH_PCI_SLOT = ["ac97", "es1370", "hda"]
 
   ANCILLARY_FILES = [
     _KVM_NETWORK_SCRIPT,
@@ -938,11 +957,40 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     return utils.PathJoin(cls._NICS_DIR, instance_name)
 
   @classmethod
-  def _InstanceNICFile(cls, instance_name, seq):
+  def _InstanceNICFile(cls, instance_name, seq_or_uuid):
     """Returns the name of the file containing the tap device for a given NIC
 
     """
-    return utils.PathJoin(cls._InstanceNICDir(instance_name), str(seq))
+    return utils.PathJoin(cls._InstanceNICDir(instance_name), str(seq_or_uuid))
+
+  @classmethod
+  def _GetInstanceNICTap(cls, instance_name, nic):
+    """Returns the tap for the corresponding nic
+
+    Search for tap file named after NIC's uuid.
+    For old instances without uuid indexed tap files returns nothing.
+
+    """
+    try:
+      return utils.ReadFile(cls._InstanceNICFile(instance_name, nic.uuid))
+    except EnvironmentError:
+      pass
+
+  @classmethod
+  def _WriteInstanceNICFiles(cls, instance_name, seq, nic, tap):
+    """Write tap name to both instance NIC files
+
+    """
+    for ident in [seq, nic.uuid]:
+      utils.WriteFile(cls._InstanceNICFile(instance_name, ident), data=tap)
+
+  @classmethod
+  def _RemoveInstanceNICFiles(cls, instance_name, seq, nic):
+    """Write tap name to both instance NIC files
+
+    """
+    for ident in [seq, nic.uuid]:
+      utils.RemoveFile(cls._InstanceNICFile(instance_name, ident))
 
   @classmethod
   def _InstanceKeymapFile(cls, instance_name):
@@ -971,6 +1019,8 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     """Removes an instance's rutime sockets/files/dirs.
 
     """
+    # This takes info from NICDir and RuntimeFile
+    cls._UnconfigureInstanceNICs(instance_name)
     utils.RemoveFile(pidfile)
     utils.RemoveFile(cls._InstanceMonitor(instance_name))
     utils.RemoveFile(cls._InstanceSerial(instance_name))
@@ -1007,30 +1057,34 @@ class KVMHypervisor(hv_base.BaseHypervisor):
         raise
 
   @staticmethod
-  def _ConfigureNIC(instance, seq, nic, tap):
-    """Run the network configuration script for a specified NIC
+  def _CreateNICEnv(instance_name, nic, tap, seq=None, instance_tags=None):
+    """Create environment variables for a specific NIC
 
-    @param instance: instance we're acting on
-    @type instance: instance object
-    @param seq: nic sequence number
-    @type seq: int
-    @param nic: nic we're acting on
-    @type nic: nic object
-    @param tap: the host's tap interface this NIC corresponds to
-    @type tap: str
+    This is needed during NIC ifup/ifdown scripts.
+    Since instance tags may change during NIC creation and removal
+    and because during cleanup instance object is not available we
+    pass them only upon NIC creation (instance startup/NIC hot-plugging).
 
     """
     env = {
       "PATH": "%s:/sbin:/usr/sbin" % os.environ["PATH"],
-      "INSTANCE": instance.name,
+      "INSTANCE": instance_name,
       "MAC": nic.mac,
       "MODE": nic.nicparams[constants.NIC_MODE],
-      "INTERFACE": tap,
-      "INTERFACE_INDEX": str(seq),
       "INTERFACE_UUID": nic.uuid,
-      "TAGS": " ".join(instance.GetTags()),
     }
 
+    if instance_tags:
+      env["TAGS"] = " ".join(instance_tags)
+
+    # This should always be available except for old instances in the
+    # cluster without uuid indexed tap files.
+    if tap:
+      env["INTERFACE"] = tap
+
+    if seq:
+      env["INTERFACE_INDEX"] = str(seq)
+
     if nic.ip:
       env["IP"] = nic.ip
 
@@ -1047,12 +1101,52 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
       env["BRIDGE"] = nic.nicparams[constants.NIC_LINK]
 
+    return env
+
+  @classmethod
+  def _ConfigureNIC(cls, instance, seq, nic, tap):
+    """Run the network configuration script for a specified NIC
+
+    @param instance: instance we're acting on
+    @type instance: instance object
+    @param seq: nic sequence number
+    @type seq: int
+    @param nic: nic we're acting on
+    @type nic: nic object
+    @param tap: the host's tap interface this NIC corresponds to
+    @type tap: str
+
+    """
+    env = cls._CreateNICEnv(instance.name, nic, tap, seq, instance.GetTags())
     result = utils.RunCmd([pathutils.KVM_IFUP, tap], env=env)
     if result.failed:
       raise errors.HypervisorError("Failed to configure interface %s: %s;"
                                    " network configuration script output: %s" %
                                    (tap, result.fail_reason, result.output))
 
+  @classmethod
+  def _UnconfigureNic(cls, instance_name, nic, only_local=True):
+    """Run ifdown script for a specific NIC
+
+    This is executed during instance cleanup and NIC hot-unplug
+
+    @param instance: instance we're acting on
+    @type instance: instance object
+    @param nic: nic we're acting on
+    @type nic: nic object
+    @param localy: whether ifdown script should reset global conf (dns) or not
+    @type localy: boolean
+
+    """
+    tap = cls._GetInstanceNICTap(instance_name, nic)
+    env = cls._CreateNICEnv(instance_name, nic, tap)
+    arg2 = str(only_local).lower()
+    result = utils.RunCmd([pathutils.KVM_IFDOWN, tap, arg2], env=env)
+    if result.failed:
+      raise errors.HypervisorError("Failed to unconfigure interface %s: %s;"
+                                   " network configuration script output: %s" %
+                                   (tap, result.fail_reason, result.output))
+
   @staticmethod
   def _VerifyAffinityPackage():
     if affinity is None:
@@ -1213,12 +1307,14 @@ class KVMHypervisor(hv_base.BaseHypervisor):
         data.append(info)
     return data
 
-  def _GenerateKVMBlockDevicesOptions(self, instance, kvm_disks,
+  def _GenerateKVMBlockDevicesOptions(self, instance, up_hvp, kvm_disks,
                                       kvmhelp, devlist):
     """Generate KVM options regarding instance's block devices.
 
     @type instance: L{objects.Instance}
     @param instance: the instance object
+    @type up_hvp: dict
+    @param up_hvp: the instance's runtime hypervisor parameters
     @type kvm_disks: list of tuples
     @param kvm_disks: list of tuples [(disk, link_name)..]
     @type kvmhelp: string
@@ -1229,12 +1325,11 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     @return: list of command line options eventually used by kvm executable
 
     """
-    hvp = instance.hvparams
-    kernel_path = hvp[constants.HV_KERNEL_PATH]
+    kernel_path = up_hvp[constants.HV_KERNEL_PATH]
     if kernel_path:
       boot_disk = False
     else:
-      boot_disk = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
+      boot_disk = up_hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
 
     # whether this is an older KVM version that uses the boot=on flag
     # on devices
@@ -1242,7 +1337,7 @@ class KVMHypervisor(hv_base.BaseHypervisor):
 
     dev_opts = []
     device_driver = None
-    disk_type = hvp[constants.HV_DISK_TYPE]
+    disk_type = up_hvp[constants.HV_DISK_TYPE]
     if disk_type == constants.HT_DISK_PARAVIRTUAL:
       if_val = ",if=%s" % self._VIRTIO
       try:
@@ -1255,7 +1350,7 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     else:
       if_val = ",if=%s" % disk_type
     # Cache mode
-    disk_cache = hvp[constants.HV_DISK_CACHE]
+    disk_cache = up_hvp[constants.HV_DISK_CACHE]
     if instance.disk_template in constants.DTS_EXT_MIRROR:
       if disk_cache != "none":
         # TODO: make this a hard error, instead of a silent overwrite
@@ -1278,6 +1373,11 @@ class KVMHypervisor(hv_base.BaseHypervisor):
         boot_disk = False
         if needs_boot_flag and disk_type != constants.HT_DISK_IDE:
           boot_val = ",boot=on"
+
+      # For ext we allow overriding disk_cache hypervisor params per disk
+      disk_cache = cfdev.params.get("cache", None)
+      if disk_cache:
+        cache_val = ",cache=%s" % disk_cache
       drive_val = "file=%s,format=raw%s%s%s" % \
                   (link_name, if_val, boot_val, cache_val)
 
@@ -1350,7 +1450,27 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     kvm_cmd.extend(["-smp", ",".join(smp_list)])
 
     kvm_cmd.extend(["-pidfile", pidfile])
-    kvm_cmd.extend(["-balloon", "virtio"])
+
+    pci_reservations = bitarray(self._DEFAULT_PCI_RESERVATIONS)
+
+    # As requested by music lovers
+    if hvp[constants.HV_SOUNDHW]:
+      soundhw = hvp[constants.HV_SOUNDHW]
+      # For some reason only few sound devices require a PCI slot
+      # while the Audio controller *must* be in slot 3.
+      # That's why we bridge this option early in command line
+      if soundhw in self._SOUNDHW_WITH_PCI_SLOT:
+        _ = _GetFreeSlot(pci_reservations, reserve=True)
+      kvm_cmd.extend(["-soundhw", soundhw])
+
+    if hvp[constants.HV_DISK_TYPE] == constants.HT_DISK_SCSI:
+      # The SCSI controller requires another PCI slot.
+      _ = _GetFreeSlot(pci_reservations, reserve=True)
+
+    # Add id to ballon and place to the first available slot (3 or 4)
+    addr = _GetFreeSlot(pci_reservations, reserve=True)
+    pci_info = ",bus=pci.0,addr=%s" % hex(addr)
+    kvm_cmd.extend(["-balloon", "virtio,id=balloon%s" % pci_info])
     kvm_cmd.extend(["-daemonize"])
     if not instance.hvparams[constants.HV_ACPI]:
       kvm_cmd.extend(["-no-acpi"])
@@ -1603,7 +1723,9 @@ class KVMHypervisor(hv_base.BaseHypervisor):
       else:
         # Enable the spice agent communication channel between the host and the
         # agent.
-        kvm_cmd.extend(["-device", "virtio-serial-pci"])
+        addr = _GetFreeSlot(pci_reservations, reserve=True)
+        pci_info = ",bus=pci.0,addr=%s" % hex(addr)
+        kvm_cmd.extend(["-device", "virtio-serial-pci,id=spice%s" % pci_info])
         kvm_cmd.extend([
           "-device",
           "virtserialport,chardev=spicechannel0,name=com.redhat.spice.0",
@@ -1631,10 +1753,6 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     if hvp[constants.HV_CPU_TYPE]:
       kvm_cmd.extend(["-cpu", hvp[constants.HV_CPU_TYPE]])
 
-    # As requested by music lovers
-    if hvp[constants.HV_SOUNDHW]:
-      kvm_cmd.extend(["-soundhw", hvp[constants.HV_SOUNDHW]])
-
     # Pass a -vga option if requested, or if spice is used, for backwards
     # compatibility.
     if hvp[constants.HV_VGA]:
@@ -1650,15 +1768,14 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     if hvp[constants.HV_KVM_EXTRA]:
       kvm_cmd.extend(hvp[constants.HV_KVM_EXTRA].split(" "))
 
-    pci_reservations = bitarray(self._DEFAULT_PCI_RESERVATIONS)
     kvm_disks = []
     for disk, link_name in block_devices:
-      _UpdatePCISlots(disk, pci_reservations)
+      disk.pci = _GetFreeSlot(pci_reservations, disk.pci, True)
       kvm_disks.append((disk, link_name))
 
     kvm_nics = []
     for nic in instance.nics:
-      _UpdatePCISlots(nic, pci_reservations)
+      nic.pci = _GetFreeSlot(pci_reservations, nic.pci, True)
       kvm_nics.append(nic)
 
     hvparams = hvp
@@ -1675,12 +1792,13 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     except EnvironmentError, err:
       raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err)
 
-  def _ReadKVMRuntime(self, instance_name):
+  @classmethod
+  def _ReadKVMRuntime(cls, instance_name):
     """Read an instance's KVM runtime
 
     """
     try:
-      file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name))
+      file_content = utils.ReadFile(cls._InstanceKVMRuntime(instance_name))
     except EnvironmentError, err:
       raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err)
     return file_content
@@ -1699,12 +1817,13 @@ class KVMHypervisor(hv_base.BaseHypervisor):
 
     self._WriteKVMRuntime(instance.name, serialized_form)
 
-  def _LoadKVMRuntime(self, instance, serialized_runtime=None):
+  @classmethod
+  def _LoadKVMRuntime(cls, instance_name, serialized_runtime=None):
     """Load an instance's KVM runtime
 
     """
     if not serialized_runtime:
-      serialized_runtime = self._ReadKVMRuntime(instance.name)
+      serialized_runtime = cls._ReadKVMRuntime(instance_name)
 
     return _AnalyzeSerializedRuntime(serialized_runtime)
 
@@ -1785,6 +1904,7 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     devlist = self._GetKVMOutput(kvm_path, self._KVMOPT_DEVICELIST)
 
     bdev_opts = self._GenerateKVMBlockDevicesOptions(instance,
+                                                     up_hvp,
                                                      kvm_disks,
                                                      kvmhelp,
                                                      devlist)
@@ -1912,8 +2032,8 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     utils.EnsureDirs([(self._InstanceNICDir(instance.name),
                      constants.RUN_DIRS_MODE)])
     for nic_seq, tap in enumerate(taps):
-      utils.WriteFile(self._InstanceNICFile(instance.name, nic_seq),
-                      data=tap)
+      nic = kvm_nics[nic_seq]
+      self._WriteInstanceNICFiles(instance.name, nic_seq, nic, tap)
 
     if vnc_pwd:
       change_cmd = "change vnc password %s" % vnc_pwd
@@ -2014,11 +2134,7 @@ class KVMHypervisor(hv_base.BaseHypervisor):
         slot = int(match.group(1))
         slots[slot] = True
 
-    [free] = slots.search(_AVAILABLE_PCI_SLOT, 1) # pylint: disable=E1101
-    if not free:
-      raise errors.HypervisorError("All PCI slots occupied")
-
-    dev.pci = int(free)
+    dev.pci = _GetFreeSlot(slots)
 
   def VerifyHotplugSupport(self, instance, action, dev_type):
     """Verifies that hotplug is supported.
@@ -2070,11 +2186,34 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     if (int(v_major), int(v_min)) < (1, 0):
       raise errors.HotplugError("Hotplug not supported for qemu versions < 1.0")
 
-  def _CallHotplugCommand(self, name, cmd):
-    output = self._CallMonitorCommand(name, cmd)
-    # TODO: parse output and check if succeeded
-    for line in output.stdout.splitlines():
-      logging.info("%s", line)
+  def _CallHotplugCommands(self, name, cmds):
+    for c in cmds:
+      self._CallMonitorCommand(name, c)
+      time.sleep(1)
+
+  def _VerifyHotplugCommand(self, instance_name, device, dev_type,
+                            should_exist):
+    """Checks if a previous hotplug command has succeeded.
+
+    It issues info pci monitor command and checks depending on should_exist
+    value if an entry with PCI slot and device ID is found or not.
+
+    @raise errors.HypervisorError: if result is not the expected one
+
+    """
+    output = self._CallMonitorCommand(instance_name, self._INFO_PCI_CMD)
+    kvm_devid = _GenerateDeviceKVMId(dev_type, device)
+    match = \
+      self._FIND_PCI_DEVICE_RE(device.pci, kvm_devid).search(output.stdout)
+    if match and not should_exist:
+      msg = "Device %s should have been removed but is still there" % kvm_devid
+      raise errors.HypervisorError(msg)
+
+    if not match and should_exist:
+      msg = "Device %s should have been added but is missing" % kvm_devid
+      raise errors.HypervisorError(msg)
+
+    logging.info("Device %s has been correctly hot-plugged", kvm_devid)
 
   def HotAddDevice(self, instance, dev_type, device, extra, seq):
     """ Helper method to hot-add a new device
@@ -2087,23 +2226,24 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     if device.pci is None:
       self._GetFreePCISlot(instance, device)
     kvm_devid = _GenerateDeviceKVMId(dev_type, device)
-    runtime = self._LoadKVMRuntime(instance)
+    runtime = self._LoadKVMRuntime(instance.name)
     if dev_type == constants.HOTPLUG_TARGET_DISK:
-      command = "drive_add dummy file=%s,if=none,id=%s,format=raw\n" % \
-                 (extra, kvm_devid)
-      command += ("device_add virtio-blk-pci,bus=pci.0,addr=%s,drive=%s,id=%s" %
-                  (hex(device.pci), kvm_devid, kvm_devid))
+      cmds = ["drive_add dummy file=%s,if=none,id=%s,format=raw" %
+                (extra, kvm_devid)]
+      cmds += ["device_add virtio-blk-pci,bus=pci.0,addr=%s,drive=%s,id=%s" %
+                (hex(device.pci), kvm_devid, kvm_devid)]
     elif dev_type == constants.HOTPLUG_TARGET_NIC:
       (tap, fd) = _OpenTap()
       self._ConfigureNIC(instance, seq, device, tap)
       self._PassTapFd(instance, fd, device)
-      command = "netdev_add tap,id=%s,fd=%s\n" % (kvm_devid, kvm_devid)
+      cmds = ["netdev_add tap,id=%s,fd=%s" % (kvm_devid, kvm_devid)]
       args = "virtio-net-pci,bus=pci.0,addr=%s,mac=%s,netdev=%s,id=%s" % \
                (hex(device.pci), device.mac, kvm_devid, kvm_devid)
-      command += "device_add %s" % args
-      utils.WriteFile(self._InstanceNICFile(instance.name, seq), data=tap)
+      cmds += ["device_add %s" % args]
+      self._WriteInstanceNICFiles(instance.name, seq, device, tap)
 
-    self._CallHotplugCommand(instance.name, command)
+    self._CallHotplugCommands(instance.name, cmds)
+    self._VerifyHotplugCommand(instance.name, device, dev_type, True)
     # update relevant entries in runtime file
     index = _DEVICE_RUNTIME_INDEX[dev_type]
     entry = _RUNTIME_ENTRY[dev_type](device, extra)
@@ -2117,18 +2257,20 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     invokes the device specific method.
 
     """
-    runtime = self._LoadKVMRuntime(instance)
+    runtime = self._LoadKVMRuntime(instance.name)
     entry = _GetExistingDeviceInfo(dev_type, device, runtime)
     kvm_device = _RUNTIME_DEVICE[dev_type](entry)
     kvm_devid = _GenerateDeviceKVMId(dev_type, kvm_device)
     if dev_type == constants.HOTPLUG_TARGET_DISK:
-      command = "device_del %s\n" % kvm_devid
-      command += "drive_del %s" % kvm_devid
+      cmds = ["device_del %s" % kvm_devid]
+      cmds += ["drive_del %s" % kvm_devid]
     elif dev_type == constants.HOTPLUG_TARGET_NIC:
-      command = "device_del %s\n" % kvm_devid
-      command += "netdev_del %s" % kvm_devid
-      utils.RemoveFile(self._InstanceNICFile(instance.name, seq))
-    self._CallHotplugCommand(instance.name, command)
+      cmds = ["device_del %s" % kvm_devid]
+      cmds += ["netdev_del %s" % kvm_devid]
+      self._UnconfigureNic(instance.name, kvm_device, False)
+      self._RemoveInstanceNICFiles(instance.name, seq, device)
+    self._CallHotplugCommands(instance.name, cmds)
+    self._VerifyHotplugCommand(instance.name, kvm_device, dev_type, False)
     index = _DEVICE_RUNTIME_INDEX[dev_type]
     runtime[index].remove(entry)
     self._SaveKVMRuntime(instance, runtime)
@@ -2146,7 +2288,6 @@ class KVMHypervisor(hv_base.BaseHypervisor):
       # putting it back in the same pci slot
       device.pci = self.HotDelDevice(instance, dev_type, device, _, seq)
       # TODO: remove sleep when socat gets removed
-      time.sleep(2)
       self.HotAddDevice(instance, dev_type, device, _, seq)
 
   def _PassTapFd(self, instance, fd, nic):
@@ -2254,6 +2395,15 @@ class KVMHypervisor(hv_base.BaseHypervisor):
       else:
         self._CallMonitorCommand(name, "system_powerdown", timeout)
 
+  @classmethod
+  def _UnconfigureInstanceNICs(cls, instance_name, info=None):
+    """Get runtime NICs of an instance and unconfigure them
+
+    """
+    _, kvm_nics, __, ___ = cls._LoadKVMRuntime(instance_name, info)
+    for nic in kvm_nics:
+      cls._UnconfigureNic(instance_name, nic)
+
   def CleanupInstance(self, instance_name):
     """Cleanup after a stopped instance
 
@@ -2276,7 +2426,7 @@ class KVMHypervisor(hv_base.BaseHypervisor):
                                    " not running" % instance.name)
     # StopInstance will delete the saved KVM runtime so:
     # ...first load it...
-    kvm_runtime = self._LoadKVMRuntime(instance)
+    kvm_runtime = self._LoadKVMRuntime(instance.name)
     # ...now we can safely call StopInstance...
     if not self.StopInstance(instance):
       self.StopInstance(instance, force=True)
@@ -2308,7 +2458,7 @@ class KVMHypervisor(hv_base.BaseHypervisor):
     @param target: target host (usually ip), on this node
 
     """
-    kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
+    kvm_runtime = self._LoadKVMRuntime(instance.name, serialized_runtime=info)
     incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT])
     kvmpath = instance.hvparams[constants.HV_KVM_PATH]
     kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
@@ -2325,7 +2475,7 @@ class KVMHypervisor(hv_base.BaseHypervisor):
 
     """
     if success:
-      kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
+      kvm_runtime = self._LoadKVMRuntime(instance.name, serialized_runtime=info)
       kvm_nics = kvm_runtime[1]
 
       for nic_seq, nic in enumerate(kvm_nics):
@@ -2345,6 +2495,7 @@ class KVMHypervisor(hv_base.BaseHypervisor):
 
       self._WriteKVMRuntime(instance.name, info)
     else:
+      self._UnconfigureInstanceNICs(instance.name, info)
       self.StopInstance(instance, force=True)
 
   def MigrateInstance(self, instance, target, live):
@@ -2378,6 +2529,17 @@ class KVMHypervisor(hv_base.BaseHypervisor):
                        instance.hvparams[constants.HV_MIGRATION_DOWNTIME])
     self._CallMonitorCommand(instance_name, migrate_command)
 
+    # These commands are supported in latest qemu versions.
+    # Since _CallMonitorCommand does not catch monitor errors
+    # this does not raise an exception in case command is not supported
+    # TODO: either parse output of command or see if the command supported
+    # via info help (see hotplug)
+    migration_caps = instance.hvparams[constants.HV_KVM_MIGRATION_CAPS]
+    if migration_caps:
+      for c in migration_caps.split(_MIGRATION_CAPS_DELIM):
+        migrate_command = ("migrate_set_capability %s on" % c)
+        self._CallMonitorCommand(instance_name, migrate_command)
+
     migrate_command = "migrate -d tcp:%s:%s" % (target, port)
     self._CallMonitorCommand(instance_name, migrate_command)