4 # Copyright (C) 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
40 from bitarray import bitarray
42 import affinity # pylint: disable=F0401
46 import fdsend # pylint: disable=F0401
50 from ganeti import utils
51 from ganeti import constants
52 from ganeti import errors
53 from ganeti import serializer
54 from ganeti import objects
55 from ganeti import uidpool
56 from ganeti import ssconf
57 from ganeti import netutils
58 from ganeti import pathutils
59 from ganeti.hypervisor import hv_base
60 from ganeti.utils import wrapper as utils_wrapper
63 _KVM_NETWORK_SCRIPT = pathutils.CONF_DIR + "/kvm-vif-bridge"
64 _KVM_START_PAUSED_FLAG = "-S"
66 # TUN/TAP driver constants, taken from <linux/if_tun.h>
67 # They are architecture-independent and already hardcoded in qemu-kvm source,
68 # so we can safely include them here.
69 TUNSETIFF = 0x400454ca
70 TUNGETIFF = 0x800454d2
71 TUNGETFEATURES = 0x800454cf
76 #: SPICE parameters which depend on L{constants.HV_KVM_SPICE_BIND}
77 _SPICE_ADDITIONAL_PARAMS = frozenset([
78 constants.HV_KVM_SPICE_IP_VERSION,
79 constants.HV_KVM_SPICE_PASSWORD_FILE,
80 constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR,
81 constants.HV_KVM_SPICE_JPEG_IMG_COMPR,
82 constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR,
83 constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION,
84 constants.HV_KVM_SPICE_USE_TLS,
87 # Constant bitarray that reflects to a free pci slot
88 # Use it with bitarray.search()
89 _AVAILABLE_PCI_SLOT = bitarray("0")
91 # below constants show the format of runtime file
92 # the nics are in second possition, while the disks in 4th (last)
93 # moreover disk entries are stored in tupples of L{objects.Disk}, dev_path
94 _KVM_NICS_RUNTIME_INDEX = 1
95 _KVM_DISKS_RUNTIME_INDEX = 3
96 _DEVICE_RUNTIME_INDEX = {
97 constants.HOTPLUG_TARGET_DISK: _KVM_DISKS_RUNTIME_INDEX,
98 constants.HOTPLUG_TARGET_NIC: _KVM_NICS_RUNTIME_INDEX
100 _FIND_RUNTIME_ENTRY = {
101 constants.HOTPLUG_TARGET_NIC:
102 lambda nic, kvm_nics: [n for n in kvm_nics if n.uuid == nic.uuid],
103 constants.HOTPLUG_TARGET_DISK:
104 lambda disk, kvm_disks: [(d, l, u) for (d, l, u) in kvm_disks
105 if d.uuid == disk.uuid]
108 constants.HOTPLUG_TARGET_NIC: lambda d: d,
109 constants.HOTPLUG_TARGET_DISK: lambda (d, e, _): d
112 constants.HOTPLUG_TARGET_NIC: lambda d, e: d,
113 constants.HOTPLUG_TARGET_DISK: lambda d, e: (d, e, None)
117 def _GenerateDeviceKVMId(dev_type, dev):
118 """Helper function to generate a unique device name used by KVM
120 QEMU monitor commands use names to identify devices. Here we use their pci
121 slot and a part of their UUID to name them. dev.pci might be None for old
122 devices in the cluster.
124 @type dev_type: sting
125 @param dev_type: device type of param dev
126 @type dev: L{objects.Disk} or L{objects.NIC}
127 @param dev: the device object for which we generate a kvm name
128 @raise errors.HotplugError: in case a device has no pci slot (old devices)
133 raise errors.HotplugError("Hotplug is not supported for %s with UUID %s" %
134 (dev_type, dev.uuid))
136 return "%s-%s-pci-%d" % (dev_type.lower(), dev.uuid.split("-")[0], dev.pci)
139 def _UpdatePCISlots(dev, pci_reservations):
140 """Update pci configuration for a stopped instance
142 If dev has a pci slot then reserve it, else find first available
143 in pci_reservations bitarray. It acts on the same objects passed
144 as params so there is no need to return anything.
146 @type dev: L{objects.Disk} or L{objects.NIC}
147 @param dev: the device object for which we update its pci slot
148 @type pci_reservations: bitarray
149 @param pci_reservations: existing pci reservations for an instance
150 @raise errors.HotplugError: in case an instance has all its slot occupied
155 else: # pylint: disable=E1103
156 [free] = pci_reservations.search(_AVAILABLE_PCI_SLOT, 1)
158 raise errors.HypervisorError("All PCI slots occupied")
161 pci_reservations[free] = True
164 def _GetExistingDeviceInfo(dev_type, device, runtime):
165 """Helper function to get an existing device inside the runtime file
167 Used when an instance is running. Load kvm runtime file and search
168 for a device based on its type and uuid.
170 @type dev_type: sting
171 @param dev_type: device type of param dev
172 @type device: L{objects.Disk} or L{objects.NIC}
173 @param device: the device object for which we generate a kvm name
174 @type runtime: tuple (cmd, nics, hvparams, disks)
175 @param runtime: the runtime data to search for the device
176 @raise errors.HotplugError: in case the requested device does not
177 exist (e.g. device has been added without --hotplug option) or
178 device info has not pci slot (e.g. old devices in the cluster)
181 index = _DEVICE_RUNTIME_INDEX[dev_type]
182 found = _FIND_RUNTIME_ENTRY[dev_type](device, runtime[index])
184 raise errors.HotplugError("Cannot find runtime info for %s with UUID %s" %
185 (dev_type, device.uuid))
190 def _AnalyzeSerializedRuntime(serialized_runtime):
191 """Return runtime entries for a serialized runtime file
193 @type serialized_runtime: string
194 @param serialized_runtime: raw text data read from actual runtime file
195 @return: (cmd, nics, hvparams, bdevs)
199 loaded_runtime = serializer.Load(serialized_runtime)
200 if len(loaded_runtime) == 3:
201 serialized_blockdevs = []
202 kvm_cmd, serialized_nics, hvparams = loaded_runtime
204 kvm_cmd, serialized_nics, hvparams, serialized_blockdevs = loaded_runtime
206 kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics]
207 block_devices = [(objects.Disk.FromDict(sdisk), link, uri)
208 for sdisk, link, uri in serialized_blockdevs]
210 return (kvm_cmd, kvm_nics, hvparams, block_devices)
213 def _GetTunFeatures(fd, _ioctl=fcntl.ioctl):
214 """Retrieves supported TUN features from file descriptor.
216 @see: L{_ProbeTapVnetHdr}
219 req = struct.pack("I", 0)
221 buf = _ioctl(fd, TUNGETFEATURES, req)
222 except EnvironmentError, err:
223 logging.warning("ioctl(TUNGETFEATURES) failed: %s", err)
226 (flags, ) = struct.unpack("I", buf)
230 def _ProbeTapVnetHdr(fd, _features_fn=_GetTunFeatures):
231 """Check whether to enable the IFF_VNET_HDR flag.
233 To do this, _all_ of the following conditions must be met:
234 1. TUNGETFEATURES ioctl() *must* be implemented
235 2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag
236 3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in
237 drivers/net/tun.c there is no way to test this until after the tap device
238 has been created using TUNSETIFF, and there is no way to change the
239 IFF_VNET_HDR flag after creating the interface, catch-22! However both
240 TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27,
241 thus we can expect TUNGETIFF to be present if TUNGETFEATURES is.
244 @param fd: the file descriptor of /dev/net/tun
247 flags = _features_fn(fd)
253 result = bool(flags & IFF_VNET_HDR)
256 logging.warning("Kernel does not support IFF_VNET_HDR, not enabling")
261 def _OpenTap(vnet_hdr=True):
262 """Open a new tap device and return its file descriptor.
264 This is intended to be used by a qemu-type hypervisor together with the -net
265 tap,fd=<fd> command line parameter.
267 @type vnet_hdr: boolean
268 @param vnet_hdr: Enable the VNET Header
269 @return: (ifname, tapfd)
274 tapfd = os.open("/dev/net/tun", os.O_RDWR)
275 except EnvironmentError:
276 raise errors.HypervisorError("Failed to open /dev/net/tun")
278 flags = IFF_TAP | IFF_NO_PI
280 if vnet_hdr and _ProbeTapVnetHdr(tapfd):
281 flags |= IFF_VNET_HDR
283 # The struct ifreq ioctl request (see netdevice(7))
284 ifr = struct.pack("16sh", "", flags)
287 res = fcntl.ioctl(tapfd, TUNSETIFF, ifr)
288 except EnvironmentError, err:
289 raise errors.HypervisorError("Failed to allocate a new TAP device: %s" %
292 # Get the interface name from the ioctl
293 ifname = struct.unpack("16sh", res)[0].strip("\x00")
294 return (ifname, tapfd)
298 """QEMU Messaging Protocol (QMP) message.
301 def __init__(self, data):
302 """Creates a new QMP message based on the passed data.
305 if not isinstance(data, dict):
306 raise TypeError("QmpMessage must be initialized with a dict")
310 def __getitem__(self, field_name):
311 """Get the value of the required field if present, or None.
313 Overrides the [] operator to provide access to the message data,
314 returning None if the required item is not in the message
315 @return: the value of the field_name field, or None if field_name
316 is not contained in the message
319 return self.data.get(field_name, None)
321 def __setitem__(self, field_name, field_value):
322 """Set the value of the required field_name to field_value.
325 self.data[field_name] = field_value
328 """Return the number of fields stored in this QmpMessage.
331 return len(self.data)
333 def __delitem__(self, key):
334 """Delete the specified element from the QmpMessage.
340 def BuildFromJsonString(json_string):
341 """Build a QmpMessage from a JSON encoded string.
343 @type json_string: str
344 @param json_string: JSON string representing the message
345 @rtype: L{QmpMessage}
346 @return: a L{QmpMessage} built from json_string
350 data = serializer.LoadJson(json_string)
351 return QmpMessage(data)
354 # The protocol expects the JSON object to be sent as a single line.
355 return serializer.DumpJson(self.data)
357 def __eq__(self, other):
358 # When comparing two QmpMessages, we are interested in comparing
359 # their internal representation of the message data
360 return self.data == other.data
364 """Connection to the QEMU Monitor using the QEMU Monitor Protocol (QMP).
367 _FIRST_MESSAGE_KEY = "QMP"
370 _RETURN_KEY = RETURN_KEY = "return"
371 _ACTUAL_KEY = ACTUAL_KEY = "actual"
372 _ERROR_CLASS_KEY = "class"
373 _ERROR_DATA_KEY = "data"
374 _ERROR_DESC_KEY = "desc"
375 _EXECUTE_KEY = "execute"
376 _ARGUMENTS_KEY = "arguments"
377 _CAPABILITIES_COMMAND = "qmp_capabilities"
378 _MESSAGE_END_TOKEN = "\r\n"
381 def __init__(self, monitor_filename):
382 """Instantiates the QmpConnection object.
384 @type monitor_filename: string
385 @param monitor_filename: the filename of the UNIX raw socket on which the
386 QMP monitor is listening
389 self.monitor_filename = monitor_filename
390 self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
391 # We want to fail if the server doesn't send a complete message
392 # in a reasonable amount of time
393 self.sock.settimeout(self._SOCKET_TIMEOUT)
394 self._connected = False
397 def _check_socket(self):
400 sock_stat = os.stat(self.monitor_filename)
401 except EnvironmentError, err:
402 if err.errno == errno.ENOENT:
403 raise errors.HypervisorError("No qmp socket found")
405 raise errors.HypervisorError("Error checking qmp socket: %s",
406 utils.ErrnoOrStr(err))
407 if not stat.S_ISSOCK(sock_stat.st_mode):
408 raise errors.HypervisorError("Qmp socket is not a socket")
410 def _check_connection(self):
411 """Make sure that the connection is established.
414 if not self._connected:
415 raise errors.ProgrammerError("To use a QmpConnection you need to first"
416 " invoke connect() on it")
419 """Connects to the QMP monitor.
421 Connects to the UNIX socket and makes sure that we can actually send and
422 receive data to the kvm instance via QMP.
424 @raise errors.HypervisorError: when there are communication errors
425 @raise errors.ProgrammerError: when there are data serialization errors
429 raise errors.ProgrammerError("Cannot connect twice")
433 # Check file existance/stuff
435 self.sock.connect(self.monitor_filename)
436 except EnvironmentError:
437 raise errors.HypervisorError("Can't connect to qmp socket")
438 self._connected = True
440 # Check if we receive a correct greeting message from the server
441 # (As per the QEMU Protocol Specification 0.1 - section 2.2)
442 greeting = self._Recv()
443 if not greeting[self._FIRST_MESSAGE_KEY]:
444 self._connected = False
445 raise errors.HypervisorError("kvm: QMP communication error (wrong"
448 # Let's put the monitor in command mode using the qmp_capabilities
449 # command, or else no command will be executable.
450 # (As per the QEMU Protocol Specification 0.1 - section 4)
451 self.Execute(self._CAPABILITIES_COMMAND)
453 def _ParseMessage(self, buf):
454 """Extract and parse a QMP message from the given buffer.
456 Seeks for a QMP message in the given buf. If found, it parses it and
457 returns it together with the rest of the characters in the buf.
458 If no message is found, returns None and the whole buffer.
460 @raise errors.ProgrammerError: when there are data serialization errors
464 # Check if we got the message end token (CRLF, as per the QEMU Protocol
465 # Specification 0.1 - Section 2.1.1)
466 pos = buf.find(self._MESSAGE_END_TOKEN)
469 message = QmpMessage.BuildFromJsonString(buf[:pos + 1])
470 except Exception, err:
471 raise errors.ProgrammerError("QMP data serialization error: %s" % err)
474 return (message, buf)
477 """Receives a message from QMP and decodes the received JSON object.
480 @return: the received message
481 @raise errors.HypervisorError: when there are communication errors
482 @raise errors.ProgrammerError: when there are data serialization errors
485 self._check_connection()
487 # Check if there is already a message in the buffer
488 (message, self._buf) = self._ParseMessage(self._buf)
492 recv_buffer = StringIO.StringIO(self._buf)
493 recv_buffer.seek(len(self._buf))
496 data = self.sock.recv(4096)
499 recv_buffer.write(data)
501 (message, self._buf) = self._ParseMessage(recv_buffer.getvalue())
505 except socket.timeout, err:
506 raise errors.HypervisorError("Timeout while receiving a QMP message: "
508 except socket.error, err:
509 raise errors.HypervisorError("Unable to receive data from KVM using the"
510 " QMP protocol: %s" % err)
512 def _Send(self, message):
513 """Encodes and sends a message to KVM using QMP.
515 @type message: QmpMessage
516 @param message: message to send to KVM
517 @raise errors.HypervisorError: when there are communication errors
518 @raise errors.ProgrammerError: when there are data serialization errors
521 self._check_connection()
523 message_str = str(message)
524 except Exception, err:
525 raise errors.ProgrammerError("QMP data deserialization error: %s" % err)
528 self.sock.sendall(message_str)
529 except socket.timeout, err:
530 raise errors.HypervisorError("Timeout while sending a QMP message: "
531 "%s (%s)" % (err.string, err.errno))
532 except socket.error, err:
533 raise errors.HypervisorError("Unable to send data from KVM using the"
534 " QMP protocol: %s" % err)
536 def Execute(self, command, arguments=None):
537 """Executes a QMP command and returns the response of the server.
540 @param command: the command to execute
541 @type arguments: dict
542 @param arguments: dictionary of arguments to be passed to the command
544 @return: dictionary representing the received JSON object
545 @raise errors.HypervisorError: when there are communication errors
546 @raise errors.ProgrammerError: when there are data serialization errors
549 self._check_connection()
550 message = QmpMessage({self._EXECUTE_KEY: command})
552 message[self._ARGUMENTS_KEY] = arguments
555 # Events can occur between the sending of the command and the reception
556 # of the response, so we need to filter out messages with the event key.
558 response = self._Recv()
559 err = response[self._ERROR_KEY]
561 raise errors.HypervisorError("kvm: error executing the %s"
562 " command: %s (%s, %s):" %
564 err[self._ERROR_DESC_KEY],
565 err[self._ERROR_CLASS_KEY],
566 err[self._ERROR_DATA_KEY]))
568 elif not response[self._EVENT_KEY]:
572 class KVMHypervisor(hv_base.BaseHypervisor):
573 """KVM hypervisor interface
578 _ROOT_DIR = pathutils.RUN_DIR + "/kvm-hypervisor"
579 _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids
580 _UIDS_DIR = _ROOT_DIR + "/uid" # contains instances reserved uids
581 _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets
582 _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data
583 _NICS_DIR = _ROOT_DIR + "/nic" # contains instances nic <-> tap associations
584 _KEYMAP_DIR = _ROOT_DIR + "/keymap" # contains instances keymaps
585 # KVM instances with chroot enabled are started in empty chroot directories.
586 _CHROOT_DIR = _ROOT_DIR + "/chroot" # for empty chroot directories
587 # After an instance is stopped, its chroot directory is removed.
588 # If the chroot directory is not empty, it can't be removed.
589 # A non-empty chroot directory indicates a possible security incident.
590 # To support forensics, the non-empty chroot directory is quarantined in
591 # a separate directory, called 'chroot-quarantine'.
592 _CHROOT_QUARANTINE_DIR = _ROOT_DIR + "/chroot-quarantine"
593 _DIRS = [_ROOT_DIR, _PIDS_DIR, _UIDS_DIR, _CTRL_DIR, _CONF_DIR, _NICS_DIR,
594 _CHROOT_DIR, _CHROOT_QUARANTINE_DIR, _KEYMAP_DIR]
597 constants.HV_KVM_PATH: hv_base.REQ_FILE_CHECK,
598 constants.HV_KERNEL_PATH: hv_base.OPT_FILE_CHECK,
599 constants.HV_INITRD_PATH: hv_base.OPT_FILE_CHECK,
600 constants.HV_ROOT_PATH: hv_base.NO_CHECK,
601 constants.HV_KERNEL_ARGS: hv_base.NO_CHECK,
602 constants.HV_ACPI: hv_base.NO_CHECK,
603 constants.HV_SERIAL_CONSOLE: hv_base.NO_CHECK,
604 constants.HV_SERIAL_SPEED: hv_base.NO_CHECK,
605 constants.HV_VNC_BIND_ADDRESS: hv_base.NO_CHECK, # will be checked later
606 constants.HV_VNC_TLS: hv_base.NO_CHECK,
607 constants.HV_VNC_X509: hv_base.OPT_DIR_CHECK,
608 constants.HV_VNC_X509_VERIFY: hv_base.NO_CHECK,
609 constants.HV_VNC_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
610 constants.HV_KVM_SPICE_BIND: hv_base.NO_CHECK, # will be checked later
611 constants.HV_KVM_SPICE_IP_VERSION:
612 (False, lambda x: (x == constants.IFACE_NO_IP_VERSION_SPECIFIED or
613 x in constants.VALID_IP_VERSIONS),
614 "The SPICE IP version should be 4 or 6",
616 constants.HV_KVM_SPICE_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
617 constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR:
619 False, constants.HT_KVM_SPICE_VALID_LOSSLESS_IMG_COMPR_OPTIONS),
620 constants.HV_KVM_SPICE_JPEG_IMG_COMPR:
622 False, constants.HT_KVM_SPICE_VALID_LOSSY_IMG_COMPR_OPTIONS),
623 constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR:
625 False, constants.HT_KVM_SPICE_VALID_LOSSY_IMG_COMPR_OPTIONS),
626 constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION:
628 False, constants.HT_KVM_SPICE_VALID_VIDEO_STREAM_DETECTION_OPTIONS),
629 constants.HV_KVM_SPICE_AUDIO_COMPR: hv_base.NO_CHECK,
630 constants.HV_KVM_SPICE_USE_TLS: hv_base.NO_CHECK,
631 constants.HV_KVM_SPICE_TLS_CIPHERS: hv_base.NO_CHECK,
632 constants.HV_KVM_SPICE_USE_VDAGENT: hv_base.NO_CHECK,
633 constants.HV_KVM_FLOPPY_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
634 constants.HV_CDROM_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
635 constants.HV_KVM_CDROM2_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
636 constants.HV_BOOT_ORDER:
637 hv_base.ParamInSet(True, constants.HT_KVM_VALID_BO_TYPES),
638 constants.HV_NIC_TYPE:
639 hv_base.ParamInSet(True, constants.HT_KVM_VALID_NIC_TYPES),
640 constants.HV_DISK_TYPE:
641 hv_base.ParamInSet(True, constants.HT_KVM_VALID_DISK_TYPES),
642 constants.HV_KVM_CDROM_DISK_TYPE:
643 hv_base.ParamInSet(False, constants.HT_KVM_VALID_DISK_TYPES),
644 constants.HV_USB_MOUSE:
645 hv_base.ParamInSet(False, constants.HT_KVM_VALID_MOUSE_TYPES),
646 constants.HV_KEYMAP: hv_base.NO_CHECK,
647 constants.HV_MIGRATION_PORT: hv_base.REQ_NET_PORT_CHECK,
648 constants.HV_MIGRATION_BANDWIDTH: hv_base.REQ_NONNEGATIVE_INT_CHECK,
649 constants.HV_MIGRATION_DOWNTIME: hv_base.REQ_NONNEGATIVE_INT_CHECK,
650 constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK,
651 constants.HV_USE_LOCALTIME: hv_base.NO_CHECK,
652 constants.HV_DISK_CACHE:
653 hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES),
654 constants.HV_SECURITY_MODEL:
655 hv_base.ParamInSet(True, constants.HT_KVM_VALID_SM_TYPES),
656 constants.HV_SECURITY_DOMAIN: hv_base.NO_CHECK,
657 constants.HV_KVM_FLAG:
658 hv_base.ParamInSet(False, constants.HT_KVM_FLAG_VALUES),
659 constants.HV_VHOST_NET: hv_base.NO_CHECK,
660 constants.HV_KVM_USE_CHROOT: hv_base.NO_CHECK,
661 constants.HV_MEM_PATH: hv_base.OPT_DIR_CHECK,
662 constants.HV_REBOOT_BEHAVIOR:
663 hv_base.ParamInSet(True, constants.REBOOT_BEHAVIORS),
664 constants.HV_CPU_MASK: hv_base.OPT_MULTI_CPU_MASK_CHECK,
665 constants.HV_CPU_TYPE: hv_base.NO_CHECK,
666 constants.HV_CPU_CORES: hv_base.OPT_NONNEGATIVE_INT_CHECK,
667 constants.HV_CPU_THREADS: hv_base.OPT_NONNEGATIVE_INT_CHECK,
668 constants.HV_CPU_SOCKETS: hv_base.OPT_NONNEGATIVE_INT_CHECK,
669 constants.HV_SOUNDHW: hv_base.NO_CHECK,
670 constants.HV_USB_DEVICES: hv_base.NO_CHECK,
671 constants.HV_VGA: hv_base.NO_CHECK,
672 constants.HV_KVM_EXTRA: hv_base.NO_CHECK,
673 constants.HV_KVM_MACHINE_VERSION: hv_base.NO_CHECK,
674 constants.HV_VNET_HDR: hv_base.NO_CHECK,
678 _VIRTIO_NET_PCI = "virtio-net-pci"
679 _VIRTIO_BLK_PCI = "virtio-blk-pci"
681 _MIGRATION_STATUS_RE = re.compile(r"Migration\s+status:\s+(\w+)",
683 _MIGRATION_PROGRESS_RE = \
684 re.compile(r"\s*transferred\s+ram:\s+(?P<transferred>\d+)\s+kbytes\s*\n"
685 r"\s*remaining\s+ram:\s+(?P<remaining>\d+)\s+kbytes\s*\n"
686 r"\s*total\s+ram:\s+(?P<total>\d+)\s+kbytes\s*\n", re.I)
688 _MIGRATION_INFO_MAX_BAD_ANSWERS = 5
689 _MIGRATION_INFO_RETRY_DELAY = 2
691 _VERSION_RE = re.compile(r"\b(\d+)\.(\d+)(\.(\d+))?\b")
693 _CPU_INFO_RE = re.compile(r"cpu\s+\#(\d+).*thread_id\s*=\s*(\d+)", re.I)
694 _CPU_INFO_CMD = "info cpus"
697 _DEFAULT_MACHINE_VERSION_RE = re.compile(r"^(\S+).*\(default\)", re.M)
698 _CHECK_MACHINE_VERSION_RE = \
699 staticmethod(lambda x: re.compile(r"^(%s)[ ]+.*PC" % x, re.M))
701 _QMP_RE = re.compile(r"^-qmp\s", re.M)
702 _SPICE_RE = re.compile(r"^-spice\s", re.M)
703 _VHOST_RE = re.compile(r"^-net\s.*,vhost=on|off", re.M)
704 _ENABLE_KVM_RE = re.compile(r"^-enable-kvm\s", re.M)
705 _DISABLE_KVM_RE = re.compile(r"^-disable-kvm\s", re.M)
706 _NETDEV_RE = re.compile(r"^-netdev\s", re.M)
707 _DISPLAY_RE = re.compile(r"^-display\s", re.M)
708 _MACHINE_RE = re.compile(r"^-machine\s", re.M)
709 _VIRTIO_NET_RE = re.compile(r"^name \"%s\"" % _VIRTIO_NET_PCI, re.M)
710 _VIRTIO_BLK_RE = re.compile(r"^name \"%s\"" % _VIRTIO_BLK_PCI, re.M)
711 # match -drive.*boot=on|off on different lines, but in between accept only
712 # dashes not preceeded by a new line (which would mean another option
713 # different than -drive is starting)
714 _BOOT_RE = re.compile(r"^-drive\s([^-]|(?<!^)-)*,boot=on\|off", re.M | re.S)
715 _UUID_RE = re.compile(r"^-uuid\s", re.M)
717 _INFO_PCI_RE = re.compile(r'Bus.*device[ ]*(\d+).*')
718 _INFO_PCI_CMD = "info pci"
720 re.compile(r'^QEMU (\d+)\.(\d+)(\.(\d+))?.*monitor.*', re.M)
721 _INFO_VERSION_CMD = "info version"
723 _DEFAULT_PCI_RESERVATIONS = "11110000000000000000000000000000"
728 ANCILLARY_FILES_OPT = [
732 # Supported kvm options to get output from
733 _KVMOPT_HELP = "help"
734 _KVMOPT_MLIST = "mlist"
735 _KVMOPT_DEVICELIST = "devicelist"
737 # Command to execute to get the output from kvm, and whether to
738 # accept the output even on failure.
740 _KVMOPT_HELP: (["--help"], False),
741 _KVMOPT_MLIST: (["-M", "?"], False),
742 _KVMOPT_DEVICELIST: (["-device", "?"], True),
746 hv_base.BaseHypervisor.__init__(self)
747 # Let's make sure the directories we need exist, even if the RUN_DIR lives
748 # in a tmpfs filesystem or has been otherwise wiped out.
749 dirs = [(dname, constants.RUN_DIRS_MODE) for dname in self._DIRS]
750 utils.EnsureDirs(dirs)
753 def _InstancePidFile(cls, instance_name):
754 """Returns the instance pidfile.
757 return utils.PathJoin(cls._PIDS_DIR, instance_name)
760 def _InstanceUidFile(cls, instance_name):
761 """Returns the instance uidfile.
764 return utils.PathJoin(cls._UIDS_DIR, instance_name)
767 def _InstancePidInfo(cls, pid):
768 """Check pid file for instance information.
770 Check that a pid file is associated with an instance, and retrieve
771 information from its command line.
773 @type pid: string or int
774 @param pid: process id of the instance to check
776 @return: (instance_name, memory, vcpus)
777 @raise errors.HypervisorError: when an instance cannot be found
780 alive = utils.IsProcessAlive(pid)
782 raise errors.HypervisorError("Cannot get info for pid %s" % pid)
784 cmdline_file = utils.PathJoin("/proc", str(pid), "cmdline")
786 cmdline = utils.ReadFile(cmdline_file)
787 except EnvironmentError, err:
788 raise errors.HypervisorError("Can't open cmdline file for pid %s: %s" %
795 arg_list = cmdline.split("\x00")
797 arg = arg_list.pop(0)
799 instance = arg_list.pop(0)
801 memory = int(arg_list.pop(0))
803 vcpus = int(arg_list.pop(0).split(",")[0])
806 raise errors.HypervisorError("Pid %s doesn't contain a ganeti kvm"
809 return (instance, memory, vcpus)
811 def _InstancePidAlive(self, instance_name):
812 """Returns the instance pidfile, pid, and liveness.
814 @type instance_name: string
815 @param instance_name: instance name
817 @return: (pid file name, pid, liveness)
820 pidfile = self._InstancePidFile(instance_name)
821 pid = utils.ReadPidFile(pidfile)
825 cmd_instance = self._InstancePidInfo(pid)[0]
826 alive = (cmd_instance == instance_name)
827 except errors.HypervisorError:
830 return (pidfile, pid, alive)
832 def _CheckDown(self, instance_name):
833 """Raises an error unless the given instance is down.
836 alive = self._InstancePidAlive(instance_name)[2]
838 raise errors.HypervisorError("Failed to start instance %s: %s" %
839 (instance_name, "already running"))
842 def _InstanceMonitor(cls, instance_name):
843 """Returns the instance monitor socket name
846 return utils.PathJoin(cls._CTRL_DIR, "%s.monitor" % instance_name)
849 def _InstanceSerial(cls, instance_name):
850 """Returns the instance serial socket name
853 return utils.PathJoin(cls._CTRL_DIR, "%s.serial" % instance_name)
856 def _InstanceQmpMonitor(cls, instance_name):
857 """Returns the instance serial QMP socket name
860 return utils.PathJoin(cls._CTRL_DIR, "%s.qmp" % instance_name)
863 def _SocatUnixConsoleParams():
864 """Returns the correct parameters for socat
866 If we have a new-enough socat we can use raw mode with an escape character.
869 if constants.SOCAT_USE_ESCAPE:
870 return "raw,echo=0,escape=%s" % constants.SOCAT_ESCAPE_CODE
872 return "echo=0,icanon=0"
875 def _InstanceKVMRuntime(cls, instance_name):
876 """Returns the instance KVM runtime filename
879 return utils.PathJoin(cls._CONF_DIR, "%s.runtime" % instance_name)
882 def _InstanceChrootDir(cls, instance_name):
883 """Returns the name of the KVM chroot dir of the instance
886 return utils.PathJoin(cls._CHROOT_DIR, instance_name)
889 def _InstanceNICDir(cls, instance_name):
890 """Returns the name of the directory holding the tap device files for a
894 return utils.PathJoin(cls._NICS_DIR, instance_name)
897 def _InstanceNICFile(cls, instance_name, seq):
898 """Returns the name of the file containing the tap device for a given NIC
901 return utils.PathJoin(cls._InstanceNICDir(instance_name), str(seq))
904 def _InstanceKeymapFile(cls, instance_name):
905 """Returns the name of the file containing the keymap for a given instance
908 return utils.PathJoin(cls._KEYMAP_DIR, instance_name)
911 def _TryReadUidFile(cls, uid_file):
912 """Try to read a uid file
915 if os.path.exists(uid_file):
917 uid = int(utils.ReadOneLineFile(uid_file))
919 except EnvironmentError:
920 logging.warning("Can't read uid file", exc_info=True)
921 except (TypeError, ValueError):
922 logging.warning("Can't parse uid file contents", exc_info=True)
926 def _RemoveInstanceRuntimeFiles(cls, pidfile, instance_name):
927 """Removes an instance's rutime sockets/files/dirs.
930 utils.RemoveFile(pidfile)
931 utils.RemoveFile(cls._InstanceMonitor(instance_name))
932 utils.RemoveFile(cls._InstanceSerial(instance_name))
933 utils.RemoveFile(cls._InstanceQmpMonitor(instance_name))
934 utils.RemoveFile(cls._InstanceKVMRuntime(instance_name))
935 utils.RemoveFile(cls._InstanceKeymapFile(instance_name))
936 uid_file = cls._InstanceUidFile(instance_name)
937 uid = cls._TryReadUidFile(uid_file)
938 utils.RemoveFile(uid_file)
940 uidpool.ReleaseUid(uid)
942 shutil.rmtree(cls._InstanceNICDir(instance_name))
944 if err.errno != errno.ENOENT:
947 chroot_dir = cls._InstanceChrootDir(instance_name)
948 utils.RemoveDir(chroot_dir)
950 if err.errno == errno.ENOTEMPTY:
951 # The chroot directory is expected to be empty, but it isn't.
952 new_chroot_dir = tempfile.mkdtemp(dir=cls._CHROOT_QUARANTINE_DIR,
955 utils.TimestampForFilename()))
956 logging.warning("The chroot directory of instance %s can not be"
957 " removed as it is not empty. Moving it to the"
958 " quarantine instead. Please investigate the"
959 " contents (%s) and clean up manually",
960 instance_name, new_chroot_dir)
961 utils.RenameFile(chroot_dir, new_chroot_dir)
966 def _ConfigureNIC(instance, seq, nic, tap):
967 """Run the network configuration script for a specified NIC
969 @param instance: instance we're acting on
970 @type instance: instance object
971 @param seq: nic sequence number
973 @param nic: nic we're acting on
974 @type nic: nic object
975 @param tap: the host's tap interface this NIC corresponds to
980 tags = " ".join(instance.tags)
985 "PATH": "%s:/sbin:/usr/sbin" % os.environ["PATH"],
986 "INSTANCE": instance.name,
988 "MODE": nic.nicparams[constants.NIC_MODE],
990 "INTERFACE_INDEX": str(seq),
997 if nic.nicparams[constants.NIC_LINK]:
998 env["LINK"] = nic.nicparams[constants.NIC_LINK]
1001 n = objects.Network.FromDict(nic.netinfo)
1002 env.update(n.HooksDict())
1004 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
1005 env["BRIDGE"] = nic.nicparams[constants.NIC_LINK]
1007 result = utils.RunCmd([pathutils.KVM_IFUP, tap], env=env)
1009 raise errors.HypervisorError("Failed to configure interface %s: %s;"
1010 " network configuration script output: %s" %
1011 (tap, result.fail_reason, result.output))
1014 def _VerifyAffinityPackage():
1015 if affinity is None:
1016 raise errors.HypervisorError("affinity Python package not"
1017 " found; cannot use CPU pinning under KVM")
1020 def _BuildAffinityCpuMask(cpu_list):
1021 """Create a CPU mask suitable for sched_setaffinity from a list of
1024 See man taskset for more info on sched_setaffinity masks.
1025 For example: [ 0, 2, 5, 6 ] will return 101 (0x65, 0..01100101).
1027 @type cpu_list: list of int
1028 @param cpu_list: list of physical CPU numbers to map to vCPUs in order
1030 @return: a bit mask of CPU affinities
1033 if cpu_list == constants.CPU_PINNING_OFF:
1034 return constants.CPU_PINNING_ALL_KVM
1036 return sum(2 ** cpu for cpu in cpu_list)
1039 def _AssignCpuAffinity(cls, cpu_mask, process_id, thread_dict):
1040 """Change CPU affinity for running VM according to given CPU mask.
1042 @param cpu_mask: CPU mask as given by the user. e.g. "0-2,4:all:1,3"
1043 @type cpu_mask: string
1044 @param process_id: process ID of KVM process. Used to pin entire VM
1046 @type process_id: int
1047 @param thread_dict: map of virtual CPUs to KVM thread IDs
1048 @type thread_dict: dict int:int
1051 # Convert the string CPU mask to a list of list of int's
1052 cpu_list = utils.ParseMultiCpuMask(cpu_mask)
1054 if len(cpu_list) == 1:
1055 all_cpu_mapping = cpu_list[0]
1056 if all_cpu_mapping == constants.CPU_PINNING_OFF:
1057 # If CPU pinning has 1 entry that's "all", then do nothing
1060 # If CPU pinning has one non-all entry, map the entire VM to
1061 # one set of physical CPUs
1062 cls._VerifyAffinityPackage()
1063 affinity.set_process_affinity_mask(
1064 process_id, cls._BuildAffinityCpuMask(all_cpu_mapping))
1066 # The number of vCPUs mapped should match the number of vCPUs
1067 # reported by KVM. This was already verified earlier, so
1068 # here only as a sanity check.
1069 assert len(thread_dict) == len(cpu_list)
1070 cls._VerifyAffinityPackage()
1072 # For each vCPU, map it to the proper list of physical CPUs
1073 for vcpu, i in zip(cpu_list, range(len(cpu_list))):
1074 affinity.set_process_affinity_mask(thread_dict[i],
1075 cls._BuildAffinityCpuMask(vcpu))
1077 def _GetVcpuThreadIds(self, instance_name):
1078 """Get a mapping of vCPU no. to thread IDs for the instance
1080 @type instance_name: string
1081 @param instance_name: instance in question
1082 @rtype: dictionary of int:int
1083 @return: a dictionary mapping vCPU numbers to thread IDs
1087 output = self._CallMonitorCommand(instance_name, self._CPU_INFO_CMD)
1088 for line in output.stdout.splitlines():
1089 match = self._CPU_INFO_RE.search(line)
1092 grp = map(int, match.groups())
1093 result[grp[0]] = grp[1]
1097 def _ExecuteCpuAffinity(self, instance_name, cpu_mask):
1098 """Complete CPU pinning.
1100 @type instance_name: string
1101 @param instance_name: name of instance
1102 @type cpu_mask: string
1103 @param cpu_mask: CPU pinning mask as entered by user
1106 # Get KVM process ID, to be used if need to pin entire VM
1107 _, pid, _ = self._InstancePidAlive(instance_name)
1108 # Get vCPU thread IDs, to be used if need to pin vCPUs separately
1109 thread_dict = self._GetVcpuThreadIds(instance_name)
1110 # Run CPU pinning, based on configured mask
1111 self._AssignCpuAffinity(cpu_mask, pid, thread_dict)
1113 def ListInstances(self, hvparams=None):
1114 """Get the list of running instances.
1116 We can do this by listing our live instances directory and
1117 checking whether the associated kvm process is still alive.
1121 for name in os.listdir(self._PIDS_DIR):
1122 if self._InstancePidAlive(name)[2]:
1126 def GetInstanceInfo(self, instance_name, hvparams=None):
1127 """Get instance properties.
1129 @type instance_name: string
1130 @param instance_name: the instance name
1131 @type hvparams: dict of strings
1132 @param hvparams: hvparams to be used with this instance
1133 @rtype: tuple of strings
1134 @return: (name, id, memory, vcpus, stat, times)
1137 _, pid, alive = self._InstancePidAlive(instance_name)
1141 _, memory, vcpus = self._InstancePidInfo(pid)
1146 qmp = QmpConnection(self._InstanceQmpMonitor(instance_name))
1148 vcpus = len(qmp.Execute("query-cpus")[qmp.RETURN_KEY])
1149 # Will fail if ballooning is not enabled, but we can then just resort to
1151 mem_bytes = qmp.Execute("query-balloon")[qmp.RETURN_KEY][qmp.ACTUAL_KEY]
1152 memory = mem_bytes / 1048576
1153 except errors.HypervisorError:
1156 return (instance_name, pid, memory, vcpus, istat, times)
1158 def GetAllInstancesInfo(self, hvparams=None):
1159 """Get properties of all instances.
1161 @type hvparams: dict of strings
1162 @param hvparams: hypervisor parameter
1163 @return: list of tuples (name, id, memory, vcpus, stat, times)
1167 for name in os.listdir(self._PIDS_DIR):
1169 info = self.GetInstanceInfo(name)
1170 except errors.HypervisorError:
1171 # Ignore exceptions due to instances being shut down
1177 def _GenerateKVMBlockDevicesOptions(self, instance, block_devices, kvmhelp):
1179 hvp = instance.hvparams
1180 boot_disk = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
1181 kvm_path = hvp[constants.HV_KVM_PATH]
1183 # whether this is an older KVM version that uses the boot=on flag
1185 needs_boot_flag = self._BOOT_RE.search(kvmhelp)
1188 device_driver = None
1189 disk_type = hvp[constants.HV_DISK_TYPE]
1190 if disk_type == constants.HT_DISK_PARAVIRTUAL:
1191 if_val = ",if=%s" % self._VIRTIO
1193 devlist = self._GetKVMOutput(kvm_path, self._KVMOPT_DEVICELIST)
1194 if self._VIRTIO_BLK_RE.search(devlist):
1196 # will be passed in -device option as driver
1197 device_driver = self._VIRTIO_BLK_PCI
1198 except errors.HypervisorError, _:
1201 if_val = ",if=%s" % disk_type
1203 disk_cache = hvp[constants.HV_DISK_CACHE]
1204 if instance.disk_template in constants.DTS_EXT_MIRROR:
1205 if disk_cache != "none":
1206 # TODO: make this a hard error, instead of a silent overwrite
1207 logging.warning("KVM: overriding disk_cache setting '%s' with 'none'"
1208 " to prevent shared storage corruption on migration",
1210 cache_val = ",cache=none"
1211 elif disk_cache != constants.HT_CACHE_DEFAULT:
1212 cache_val = ",cache=%s" % disk_cache
1215 for cfdev, link_name, uri in block_devices:
1216 if cfdev.mode != constants.DISK_RDWR:
1217 raise errors.HypervisorError("Instance has read-only disks which"
1218 " are not supported by KVM")
1219 # TODO: handle FD_LOOP and FD_BLKTAP (?)
1222 dev_opts.extend(["-boot", "c"])
1224 if needs_boot_flag and disk_type != constants.HT_DISK_IDE:
1225 boot_val = ",boot=on"
1227 access_mode = cfdev.params.get(constants.LDP_ACCESS,
1228 constants.DISK_KERNELSPACE)
1229 if (uri and access_mode == constants.DISK_USERSPACE):
1232 drive_uri = link_name
1234 drive_val = "file=%s,format=raw%s%s%s" % \
1235 (drive_uri, if_val, boot_val, cache_val)
1238 # block_devices are the 4th entry of runtime file that did not exist in
1239 # the past. That means that cfdev should always have pci slot and
1240 # _GenerateDeviceKVMId() will not raise a exception.
1241 kvm_devid = _GenerateDeviceKVMId(constants.HOTPLUG_TARGET_DISK, cfdev)
1242 drive_val += (",id=%s" % kvm_devid)
1243 drive_val += (",bus=0,unit=%d" % cfdev.pci)
1244 dev_val = ("%s,drive=%s,id=%s" %
1245 (device_driver, kvm_devid, kvm_devid))
1246 dev_val += ",bus=pci.0,addr=%s" % hex(cfdev.pci)
1247 dev_opts.extend(["-device", dev_val])
1249 dev_opts.extend(["-drive", drive_val])
1253 def _GenerateKVMRuntime(self, instance, block_devices, startup_paused,
1255 """Generate KVM information to start an instance.
1257 @type kvmhelp: string
1258 @param kvmhelp: output of kvm --help
1259 @attention: this function must not have any side-effects; for
1260 example, it must not write to the filesystem, or read values
1261 from the current system the are expected to differ between
1262 nodes, since it is only run once at instance startup;
1263 actions/kvm arguments that can vary between systems should be
1264 done in L{_ExecuteKVMRuntime}
1267 # pylint: disable=R0912,R0914,R0915
1268 hvp = instance.hvparams
1269 self.ValidateParameters(hvp)
1271 pidfile = self._InstancePidFile(instance.name)
1272 kvm = hvp[constants.HV_KVM_PATH]
1274 # used just by the vnc server, if enabled
1275 kvm_cmd.extend(["-name", instance.name])
1276 kvm_cmd.extend(["-m", instance.beparams[constants.BE_MAXMEM]])
1278 smp_list = ["%s" % instance.beparams[constants.BE_VCPUS]]
1279 if hvp[constants.HV_CPU_CORES]:
1280 smp_list.append("cores=%s" % hvp[constants.HV_CPU_CORES])
1281 if hvp[constants.HV_CPU_THREADS]:
1282 smp_list.append("threads=%s" % hvp[constants.HV_CPU_THREADS])
1283 if hvp[constants.HV_CPU_SOCKETS]:
1284 smp_list.append("sockets=%s" % hvp[constants.HV_CPU_SOCKETS])
1286 kvm_cmd.extend(["-smp", ",".join(smp_list)])
1288 kvm_cmd.extend(["-pidfile", pidfile])
1289 kvm_cmd.extend(["-balloon", "virtio"])
1290 kvm_cmd.extend(["-daemonize"])
1291 if not instance.hvparams[constants.HV_ACPI]:
1292 kvm_cmd.extend(["-no-acpi"])
1293 if instance.hvparams[constants.HV_REBOOT_BEHAVIOR] == \
1294 constants.INSTANCE_REBOOT_EXIT:
1295 kvm_cmd.extend(["-no-reboot"])
1297 mversion = hvp[constants.HV_KVM_MACHINE_VERSION]
1299 mversion = self._GetDefaultMachineVersion(kvm)
1300 if self._MACHINE_RE.search(kvmhelp):
1301 # TODO (2.8): kernel_irqchip and kvm_shadow_mem machine properties, as
1302 # extra hypervisor parameters. We should also investigate whether and how
1303 # shadow_mem should be considered for the resource model.
1304 if (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED):
1305 specprop = ",accel=kvm"
1308 machinespec = "%s%s" % (mversion, specprop)
1309 kvm_cmd.extend(["-machine", machinespec])
1311 kvm_cmd.extend(["-M", mversion])
1312 if (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED and
1313 self._ENABLE_KVM_RE.search(kvmhelp)):
1314 kvm_cmd.extend(["-enable-kvm"])
1315 elif (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_DISABLED and
1316 self._DISABLE_KVM_RE.search(kvmhelp)):
1317 kvm_cmd.extend(["-disable-kvm"])
1319 kernel_path = hvp[constants.HV_KERNEL_PATH]
1321 boot_cdrom = boot_floppy = boot_network = False
1323 boot_cdrom = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_CDROM
1324 boot_floppy = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_FLOPPY
1325 boot_network = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_NETWORK
1328 kvm_cmd.extend([_KVM_START_PAUSED_FLAG])
1331 kvm_cmd.extend(["-boot", "n"])
1333 # whether this is an older KVM version that uses the boot=on flag
1335 needs_boot_flag = self._BOOT_RE.search(kvmhelp)
1337 disk_type = hvp[constants.HV_DISK_TYPE]
1339 #Now we can specify a different device type for CDROM devices.
1340 cdrom_disk_type = hvp[constants.HV_KVM_CDROM_DISK_TYPE]
1341 if not cdrom_disk_type:
1342 cdrom_disk_type = disk_type
1344 iso_image = hvp[constants.HV_CDROM_IMAGE_PATH]
1346 options = ",format=raw,media=cdrom"
1347 # set cdrom 'if' type
1349 actual_cdrom_type = constants.HT_DISK_IDE
1350 elif cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
1351 actual_cdrom_type = "virtio"
1353 actual_cdrom_type = cdrom_disk_type
1354 if_val = ",if=%s" % actual_cdrom_type
1355 # set boot flag, if needed
1358 kvm_cmd.extend(["-boot", "d"])
1360 boot_val = ",boot=on"
1361 # and finally build the entire '-drive' value
1362 drive_val = "file=%s%s%s%s" % (iso_image, options, if_val, boot_val)
1363 kvm_cmd.extend(["-drive", drive_val])
1365 iso_image2 = hvp[constants.HV_KVM_CDROM2_IMAGE_PATH]
1367 options = ",format=raw,media=cdrom"
1368 if cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
1369 if_val = ",if=virtio"
1371 if_val = ",if=%s" % cdrom_disk_type
1372 drive_val = "file=%s%s%s" % (iso_image2, options, if_val)
1373 kvm_cmd.extend(["-drive", drive_val])
1375 floppy_image = hvp[constants.HV_KVM_FLOPPY_IMAGE_PATH]
1377 options = ",format=raw,media=disk"
1379 kvm_cmd.extend(["-boot", "a"])
1380 options = "%s,boot=on" % options
1381 if_val = ",if=floppy"
1382 options = "%s%s" % (options, if_val)
1383 drive_val = "file=%s%s" % (floppy_image, options)
1384 kvm_cmd.extend(["-drive", drive_val])
1387 kvm_cmd.extend(["-kernel", kernel_path])
1388 initrd_path = hvp[constants.HV_INITRD_PATH]
1390 kvm_cmd.extend(["-initrd", initrd_path])
1391 root_append = ["root=%s" % hvp[constants.HV_ROOT_PATH],
1392 hvp[constants.HV_KERNEL_ARGS]]
1393 if hvp[constants.HV_SERIAL_CONSOLE]:
1394 serial_speed = hvp[constants.HV_SERIAL_SPEED]
1395 root_append.append("console=ttyS0,%s" % serial_speed)
1396 kvm_cmd.extend(["-append", " ".join(root_append)])
1398 mem_path = hvp[constants.HV_MEM_PATH]
1400 kvm_cmd.extend(["-mem-path", mem_path, "-mem-prealloc"])
1402 monitor_dev = ("unix:%s,server,nowait" %
1403 self._InstanceMonitor(instance.name))
1404 kvm_cmd.extend(["-monitor", monitor_dev])
1405 if hvp[constants.HV_SERIAL_CONSOLE]:
1406 serial_dev = ("unix:%s,server,nowait" %
1407 self._InstanceSerial(instance.name))
1408 kvm_cmd.extend(["-serial", serial_dev])
1410 kvm_cmd.extend(["-serial", "none"])
1412 mouse_type = hvp[constants.HV_USB_MOUSE]
1413 vnc_bind_address = hvp[constants.HV_VNC_BIND_ADDRESS]
1414 spice_bind = hvp[constants.HV_KVM_SPICE_BIND]
1415 spice_ip_version = None
1417 kvm_cmd.extend(["-usb"])
1420 kvm_cmd.extend(["-usbdevice", mouse_type])
1421 elif vnc_bind_address:
1422 kvm_cmd.extend(["-usbdevice", constants.HT_MOUSE_TABLET])
1424 if vnc_bind_address:
1425 if netutils.IsValidInterface(vnc_bind_address):
1426 if_addresses = netutils.GetInterfaceIpAddresses(vnc_bind_address)
1427 if_ip4_addresses = if_addresses[constants.IP4_VERSION]
1428 if len(if_ip4_addresses) < 1:
1429 logging.error("Could not determine IPv4 address of interface %s",
1432 vnc_bind_address = if_ip4_addresses[0]
1433 if netutils.IP4Address.IsValid(vnc_bind_address):
1434 if instance.network_port > constants.VNC_BASE_PORT:
1435 display = instance.network_port - constants.VNC_BASE_PORT
1436 if vnc_bind_address == constants.IP4_ADDRESS_ANY:
1437 vnc_arg = ":%d" % (display)
1439 vnc_arg = "%s:%d" % (vnc_bind_address, display)
1441 logging.error("Network port is not a valid VNC display (%d < %d),"
1442 " not starting VNC",
1443 instance.network_port, constants.VNC_BASE_PORT)
1446 # Only allow tls and other option when not binding to a file, for now.
1447 # kvm/qemu gets confused otherwise about the filename to use.
1449 if hvp[constants.HV_VNC_TLS]:
1450 vnc_append = "%s,tls" % vnc_append
1451 if hvp[constants.HV_VNC_X509_VERIFY]:
1452 vnc_append = "%s,x509verify=%s" % (vnc_append,
1453 hvp[constants.HV_VNC_X509])
1454 elif hvp[constants.HV_VNC_X509]:
1455 vnc_append = "%s,x509=%s" % (vnc_append,
1456 hvp[constants.HV_VNC_X509])
1457 if hvp[constants.HV_VNC_PASSWORD_FILE]:
1458 vnc_append = "%s,password" % vnc_append
1460 vnc_arg = "%s%s" % (vnc_arg, vnc_append)
1463 vnc_arg = "unix:%s/%s.vnc" % (vnc_bind_address, instance.name)
1465 kvm_cmd.extend(["-vnc", vnc_arg])
1467 # FIXME: this is wrong here; the iface ip address differs
1468 # between systems, so it should be done in _ExecuteKVMRuntime
1469 if netutils.IsValidInterface(spice_bind):
1470 # The user specified a network interface, we have to figure out the IP
1472 addresses = netutils.GetInterfaceIpAddresses(spice_bind)
1473 spice_ip_version = hvp[constants.HV_KVM_SPICE_IP_VERSION]
1475 # if the user specified an IP version and the interface does not
1476 # have that kind of IP addresses, throw an exception
1477 if spice_ip_version != constants.IFACE_NO_IP_VERSION_SPECIFIED:
1478 if not addresses[spice_ip_version]:
1479 raise errors.HypervisorError("SPICE: Unable to get an IPv%s address"
1480 " for %s" % (spice_ip_version,
1483 # the user did not specify an IP version, we have to figure it out
1484 elif (addresses[constants.IP4_VERSION] and
1485 addresses[constants.IP6_VERSION]):
1486 # we have both ipv4 and ipv6, let's use the cluster default IP
1488 cluster_family = ssconf.SimpleStore().GetPrimaryIPFamily()
1489 spice_ip_version = \
1490 netutils.IPAddress.GetVersionFromAddressFamily(cluster_family)
1491 elif addresses[constants.IP4_VERSION]:
1492 spice_ip_version = constants.IP4_VERSION
1493 elif addresses[constants.IP6_VERSION]:
1494 spice_ip_version = constants.IP6_VERSION
1496 raise errors.HypervisorError("SPICE: Unable to get an IP address"
1497 " for %s" % (spice_bind))
1499 spice_address = addresses[spice_ip_version][0]
1502 # spice_bind is known to be a valid IP address, because
1503 # ValidateParameters checked it.
1504 spice_address = spice_bind
1506 spice_arg = "addr=%s" % spice_address
1507 if hvp[constants.HV_KVM_SPICE_USE_TLS]:
1508 spice_arg = ("%s,tls-port=%s,x509-cacert-file=%s" %
1509 (spice_arg, instance.network_port,
1510 pathutils.SPICE_CACERT_FILE))
1511 spice_arg = ("%s,x509-key-file=%s,x509-cert-file=%s" %
1512 (spice_arg, pathutils.SPICE_CERT_FILE,
1513 pathutils.SPICE_CERT_FILE))
1514 tls_ciphers = hvp[constants.HV_KVM_SPICE_TLS_CIPHERS]
1516 spice_arg = "%s,tls-ciphers=%s" % (spice_arg, tls_ciphers)
1518 spice_arg = "%s,port=%s" % (spice_arg, instance.network_port)
1520 if not hvp[constants.HV_KVM_SPICE_PASSWORD_FILE]:
1521 spice_arg = "%s,disable-ticketing" % spice_arg
1523 if spice_ip_version:
1524 spice_arg = "%s,ipv%s" % (spice_arg, spice_ip_version)
1526 # Image compression options
1527 img_lossless = hvp[constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR]
1528 img_jpeg = hvp[constants.HV_KVM_SPICE_JPEG_IMG_COMPR]
1529 img_zlib_glz = hvp[constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR]
1531 spice_arg = "%s,image-compression=%s" % (spice_arg, img_lossless)
1533 spice_arg = "%s,jpeg-wan-compression=%s" % (spice_arg, img_jpeg)
1535 spice_arg = "%s,zlib-glz-wan-compression=%s" % (spice_arg, img_zlib_glz)
1537 # Video stream detection
1538 video_streaming = hvp[constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION]
1540 spice_arg = "%s,streaming-video=%s" % (spice_arg, video_streaming)
1542 # Audio compression, by default in qemu-kvm it is on
1543 if not hvp[constants.HV_KVM_SPICE_AUDIO_COMPR]:
1544 spice_arg = "%s,playback-compression=off" % spice_arg
1545 if not hvp[constants.HV_KVM_SPICE_USE_VDAGENT]:
1546 spice_arg = "%s,agent-mouse=off" % spice_arg
1548 # Enable the spice agent communication channel between the host and the
1550 kvm_cmd.extend(["-device", "virtio-serial-pci"])
1553 "virtserialport,chardev=spicechannel0,name=com.redhat.spice.0",
1555 kvm_cmd.extend(["-chardev", "spicevmc,id=spicechannel0,name=vdagent"])
1557 logging.info("KVM: SPICE will listen on port %s", instance.network_port)
1558 kvm_cmd.extend(["-spice", spice_arg])
1561 # From qemu 1.4 -nographic is incompatible with -daemonize. The new way
1562 # also works in earlier versions though (tested with 1.1 and 1.3)
1563 if self._DISPLAY_RE.search(kvmhelp):
1564 kvm_cmd.extend(["-display", "none"])
1566 kvm_cmd.extend(["-nographic"])
1568 if hvp[constants.HV_USE_LOCALTIME]:
1569 kvm_cmd.extend(["-localtime"])
1571 if hvp[constants.HV_KVM_USE_CHROOT]:
1572 kvm_cmd.extend(["-chroot", self._InstanceChrootDir(instance.name)])
1574 # Add qemu-KVM -cpu param
1575 if hvp[constants.HV_CPU_TYPE]:
1576 kvm_cmd.extend(["-cpu", hvp[constants.HV_CPU_TYPE]])
1578 # As requested by music lovers
1579 if hvp[constants.HV_SOUNDHW]:
1580 kvm_cmd.extend(["-soundhw", hvp[constants.HV_SOUNDHW]])
1582 # Pass a -vga option if requested, or if spice is used, for backwards
1584 if hvp[constants.HV_VGA]:
1585 kvm_cmd.extend(["-vga", hvp[constants.HV_VGA]])
1587 kvm_cmd.extend(["-vga", "qxl"])
1589 # Various types of usb devices, comma separated
1590 if hvp[constants.HV_USB_DEVICES]:
1591 for dev in hvp[constants.HV_USB_DEVICES].split(","):
1592 kvm_cmd.extend(["-usbdevice", dev])
1594 # Set system UUID to instance UUID
1595 if self._UUID_RE.search(kvmhelp):
1596 kvm_cmd.extend(["-uuid", instance.uuid])
1598 if hvp[constants.HV_KVM_EXTRA]:
1599 kvm_cmd.extend(hvp[constants.HV_KVM_EXTRA].split(" "))
1601 pci_reservations = bitarray(self._DEFAULT_PCI_RESERVATIONS)
1603 for disk, link_name, uri in block_devices:
1604 _UpdatePCISlots(disk, pci_reservations)
1605 kvm_disks.append((disk, link_name, uri))
1608 for nic in instance.nics:
1609 _UpdatePCISlots(nic, pci_reservations)
1610 kvm_nics.append(nic)
1614 return (kvm_cmd, kvm_nics, hvparams, kvm_disks)
1616 def _WriteKVMRuntime(self, instance_name, data):
1617 """Write an instance's KVM runtime
1621 utils.WriteFile(self._InstanceKVMRuntime(instance_name),
1623 except EnvironmentError, err:
1624 raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err)
1626 def _ReadKVMRuntime(self, instance_name):
1627 """Read an instance's KVM runtime
1631 file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name))
1632 except EnvironmentError, err:
1633 raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err)
1636 def _SaveKVMRuntime(self, instance, kvm_runtime):
1637 """Save an instance's KVM runtime
1640 kvm_cmd, kvm_nics, hvparams, block_devices = kvm_runtime
1642 serialized_nics = [nic.ToDict() for nic in kvm_nics]
1643 serialized_blockdevs = [(blk.ToDict(), link, uri)
1644 for blk, link, uri in block_devices]
1645 serialized_form = serializer.Dump((kvm_cmd, serialized_nics, hvparams,
1646 serialized_blockdevs))
1648 self._WriteKVMRuntime(instance.name, serialized_form)
1650 def _LoadKVMRuntime(self, instance, serialized_runtime=None):
1651 """Load an instance's KVM runtime
1654 if not serialized_runtime:
1655 serialized_runtime = self._ReadKVMRuntime(instance.name)
1657 return _AnalyzeSerializedRuntime(serialized_runtime)
1659 def _RunKVMCmd(self, name, kvm_cmd, tap_fds=None):
1660 """Run the KVM cmd and check for errors
1663 @param name: instance name
1664 @type kvm_cmd: list of strings
1665 @param kvm_cmd: runcmd input for kvm
1666 @type tap_fds: list of int
1667 @param tap_fds: fds of tap devices opened by Ganeti
1671 result = utils.RunCmd(kvm_cmd, noclose_fds=tap_fds)
1674 utils_wrapper.CloseFdNoError(fd)
1677 raise errors.HypervisorError("Failed to start instance %s: %s (%s)" %
1678 (name, result.fail_reason, result.output))
1679 if not self._InstancePidAlive(name)[2]:
1680 raise errors.HypervisorError("Failed to start instance %s" % name)
1682 # 52/50 local variables
1683 # pylint: disable=R0914
1684 def _ExecuteKVMRuntime(self, instance, kvm_runtime, kvmhelp, incoming=None):
1685 """Execute a KVM cmd, after completing it with some last minute data.
1687 @type incoming: tuple of strings
1688 @param incoming: (target_host_ip, port)
1689 @type kvmhelp: string
1690 @param kvmhelp: output of kvm --help
1693 # Small _ExecuteKVMRuntime hv parameters programming howto:
1694 # - conf_hvp contains the parameters as configured on ganeti. they might
1695 # have changed since the instance started; only use them if the change
1696 # won't affect the inside of the instance (which hasn't been rebooted).
1697 # - up_hvp contains the parameters as they were when the instance was
1698 # started, plus any new parameter which has been added between ganeti
1699 # versions: it is paramount that those default to a value which won't
1700 # affect the inside of the instance as well.
1701 conf_hvp = instance.hvparams
1702 name = instance.name
1703 self._CheckDown(name)
1707 kvm_cmd, kvm_nics, up_hvp, block_devices = kvm_runtime
1708 # the first element of kvm_cmd is always the path to the kvm binary
1709 kvm_path = kvm_cmd[0]
1710 up_hvp = objects.FillDict(conf_hvp, up_hvp)
1712 # We know it's safe to run as a different user upon migration, so we'll use
1713 # the latest conf, from conf_hvp.
1714 security_model = conf_hvp[constants.HV_SECURITY_MODEL]
1715 if security_model == constants.HT_SM_USER:
1716 kvm_cmd.extend(["-runas", conf_hvp[constants.HV_SECURITY_DOMAIN]])
1718 keymap = conf_hvp[constants.HV_KEYMAP]
1720 keymap_path = self._InstanceKeymapFile(name)
1721 # If a keymap file is specified, KVM won't use its internal defaults. By
1722 # first including the "en-us" layout, an error on loading the actual
1723 # layout (e.g. because it can't be found) won't lead to a non-functional
1724 # keyboard. A keyboard with incorrect keys is still better than none.
1725 utils.WriteFile(keymap_path, data="include en-us\ninclude %s\n" % keymap)
1726 kvm_cmd.extend(["-k", keymap_path])
1728 # We have reasons to believe changing something like the nic driver/type
1729 # upon migration won't exactly fly with the instance kernel, so for nic
1730 # related parameters we'll use up_hvp
1734 kvm_cmd.extend(["-net", "none"])
1738 nic_type = up_hvp[constants.HV_NIC_TYPE]
1739 if nic_type == constants.HT_NIC_PARAVIRTUAL:
1740 nic_model = self._VIRTIO
1742 devlist = self._GetKVMOutput(kvm_path, self._KVMOPT_DEVICELIST)
1743 if self._VIRTIO_NET_RE.search(devlist):
1744 nic_model = self._VIRTIO_NET_PCI
1745 vnet_hdr = up_hvp[constants.HV_VNET_HDR]
1746 except errors.HypervisorError, _:
1747 # Older versions of kvm don't support DEVICE_LIST, but they don't
1748 # have new virtio syntax either.
1751 if up_hvp[constants.HV_VHOST_NET]:
1752 # check for vhost_net support
1753 if self._VHOST_RE.search(kvmhelp):
1754 tap_extra = ",vhost=on"
1756 raise errors.HypervisorError("vhost_net is configured"
1757 " but it is not available")
1759 nic_model = nic_type
1761 kvm_supports_netdev = self._NETDEV_RE.search(kvmhelp)
1763 for nic_seq, nic in enumerate(kvm_nics):
1764 tapname, tapfd = _OpenTap(vnet_hdr=vnet_hdr)
1765 tapfds.append(tapfd)
1766 taps.append(tapname)
1767 if kvm_supports_netdev:
1768 nic_val = "%s,mac=%s" % (nic_model, nic.mac)
1770 # kvm_nics already exist in old runtime files and thus there might
1771 # be some entries without pci slot (therefore try: except:)
1772 kvm_devid = _GenerateDeviceKVMId(constants.HOTPLUG_TARGET_NIC, nic)
1774 nic_val += (",id=%s,bus=pci.0,addr=%s" % (kvm_devid, hex(nic.pci)))
1775 except errors.HotplugError:
1776 netdev = "netdev%d" % nic_seq
1777 nic_val += (",netdev=%s" % netdev)
1778 tap_val = ("type=tap,id=%s,fd=%d%s" %
1779 (netdev, tapfd, tap_extra))
1780 kvm_cmd.extend(["-netdev", tap_val, "-device", nic_val])
1782 nic_val = "nic,vlan=%s,macaddr=%s,model=%s" % (nic_seq,
1784 tap_val = "tap,vlan=%s,fd=%d" % (nic_seq, tapfd)
1785 kvm_cmd.extend(["-net", tap_val, "-net", nic_val])
1788 target, port = incoming
1789 kvm_cmd.extend(["-incoming", "tcp:%s:%s" % (target, port)])
1791 # Changing the vnc password doesn't bother the guest that much. At most it
1792 # will surprise people who connect to it. Whether positively or negatively
1794 vnc_pwd_file = conf_hvp[constants.HV_VNC_PASSWORD_FILE]
1798 vnc_pwd = utils.ReadOneLineFile(vnc_pwd_file, strict=True)
1799 except EnvironmentError, err:
1800 raise errors.HypervisorError("Failed to open VNC password file %s: %s"
1801 % (vnc_pwd_file, err))
1803 if conf_hvp[constants.HV_KVM_USE_CHROOT]:
1804 utils.EnsureDirs([(self._InstanceChrootDir(name),
1805 constants.SECURE_DIR_MODE)])
1807 # Automatically enable QMP if version is >= 0.14
1808 if self._QMP_RE.search(kvmhelp):
1809 logging.debug("Enabling QMP")
1810 kvm_cmd.extend(["-qmp", "unix:%s,server,nowait" %
1811 self._InstanceQmpMonitor(instance.name)])
1813 # Configure the network now for starting instances and bridged interfaces,
1814 # during FinalizeMigration for incoming instances' routed interfaces
1815 for nic_seq, nic in enumerate(kvm_nics):
1817 nic.nicparams[constants.NIC_MODE] != constants.NIC_MODE_BRIDGED):
1819 self._ConfigureNIC(instance, nic_seq, nic, taps[nic_seq])
1821 bdev_opts = self._GenerateKVMBlockDevicesOptions(instance,
1824 kvm_cmd.extend(bdev_opts)
1825 # CPU affinity requires kvm to start paused, so we set this flag if the
1826 # instance is not already paused and if we are not going to accept a
1827 # migrating instance. In the latter case, pausing is not needed.
1828 start_kvm_paused = not (_KVM_START_PAUSED_FLAG in kvm_cmd) and not incoming
1829 if start_kvm_paused:
1830 kvm_cmd.extend([_KVM_START_PAUSED_FLAG])
1832 # Note: CPU pinning is using up_hvp since changes take effect
1833 # during instance startup anyway, and to avoid problems when soft
1834 # rebooting the instance.
1836 if up_hvp.get(constants.HV_CPU_MASK, None):
1839 if security_model == constants.HT_SM_POOL:
1840 ss = ssconf.SimpleStore()
1841 uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\n")
1842 all_uids = set(uidpool.ExpandUidPool(uid_pool))
1843 uid = uidpool.RequestUnusedUid(all_uids)
1845 username = pwd.getpwuid(uid.GetUid()).pw_name
1846 kvm_cmd.extend(["-runas", username])
1847 self._RunKVMCmd(name, kvm_cmd, tapfds)
1849 uidpool.ReleaseUid(uid)
1853 utils.WriteFile(self._InstanceUidFile(name), data=uid.AsStr())
1855 self._RunKVMCmd(name, kvm_cmd, tapfds)
1857 utils.EnsureDirs([(self._InstanceNICDir(instance.name),
1858 constants.RUN_DIRS_MODE)])
1859 for nic_seq, tap in enumerate(taps):
1860 utils.WriteFile(self._InstanceNICFile(instance.name, nic_seq),
1864 change_cmd = "change vnc password %s" % vnc_pwd
1865 self._CallMonitorCommand(instance.name, change_cmd)
1867 # Setting SPICE password. We are not vulnerable to malicious passwordless
1868 # connection attempts because SPICE by default does not allow connections
1869 # if neither a password nor the "disable_ticketing" options are specified.
1870 # As soon as we send the password via QMP, that password is a valid ticket
1872 spice_password_file = conf_hvp[constants.HV_KVM_SPICE_PASSWORD_FILE]
1873 if spice_password_file:
1876 spice_pwd = utils.ReadOneLineFile(spice_password_file, strict=True)
1877 except EnvironmentError, err:
1878 raise errors.HypervisorError("Failed to open SPICE password file %s: %s"
1879 % (spice_password_file, err))
1881 qmp = QmpConnection(self._InstanceQmpMonitor(instance.name))
1884 "protocol": "spice",
1885 "password": spice_pwd,
1887 qmp.Execute("set_password", arguments)
1889 for filename in temp_files:
1890 utils.RemoveFile(filename)
1892 # If requested, set CPU affinity and resume instance execution
1894 self._ExecuteCpuAffinity(instance.name, up_hvp[constants.HV_CPU_MASK])
1896 start_memory = self._InstanceStartupMemory(instance)
1897 if start_memory < instance.beparams[constants.BE_MAXMEM]:
1898 self.BalloonInstanceMemory(instance, start_memory)
1900 if start_kvm_paused:
1901 # To control CPU pinning, ballooning, and vnc/spice passwords
1902 # the VM was started in a frozen state. If freezing was not
1903 # explicitly requested resume the vm status.
1904 self._CallMonitorCommand(instance.name, self._CONT_CMD)
1906 def StartInstance(self, instance, block_devices, startup_paused):
1907 """Start an instance.
1910 self._CheckDown(instance.name)
1911 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
1912 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
1913 kvm_runtime = self._GenerateKVMRuntime(instance, block_devices,
1914 startup_paused, kvmhelp)
1915 self._SaveKVMRuntime(instance, kvm_runtime)
1916 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp)
1918 def _CallMonitorCommand(self, instance_name, command):
1919 """Invoke a command on the instance monitor.
1922 # TODO: Replace monitor calls with QMP once KVM >= 0.14 is the minimum
1923 # version. The monitor protocol is designed for human consumption, whereas
1924 # QMP is made for programmatic usage. In the worst case QMP can also
1925 # execute monitor commands. As it is, all calls to socat take at least
1926 # 500ms and likely more: socat can't detect the end of the reply and waits
1927 # for 500ms of no data received before exiting (500 ms is the default for
1928 # the "-t" parameter).
1929 socat = ("echo %s | %s STDIO UNIX-CONNECT:%s" %
1930 (utils.ShellQuote(command),
1931 constants.SOCAT_PATH,
1932 utils.ShellQuote(self._InstanceMonitor(instance_name))))
1933 result = utils.RunCmd(socat)
1935 msg = ("Failed to send command '%s' to instance '%s', reason '%s',"
1937 (command, instance_name, result.fail_reason, result.output))
1938 raise errors.HypervisorError(msg)
1942 def _GetFreePCISlot(self, instance, dev):
1943 """Get the first available pci slot of a runnung instance.
1946 slots = bitarray(32)
1947 slots.setall(False) # pylint: disable=E1101
1948 output = self._CallMonitorCommand(instance.name, self._INFO_PCI_CMD)
1949 for line in output.stdout.splitlines():
1950 match = self._INFO_PCI_RE.search(line)
1952 slot = int(match.group(1))
1955 [free] = slots.search(_AVAILABLE_PCI_SLOT, 1) # pylint: disable=E1101
1957 raise errors.HypervisorError("All PCI slots occupied")
1961 def HotplugSupported(self, instance, action, dev_type):
1962 """Check if hotplug is supported.
1964 Hotplug is *not* supported in case of:
1965 - qemu versions < 1.0
1966 - security models and chroot (disk hotplug)
1967 - fdsend module is missing (nic hot-add)
1969 @raise errors.HypervisorError: in previous cases
1972 output = self._CallMonitorCommand(instance.name, self._INFO_VERSION_CMD)
1973 # TODO: search for netdev_add, drive_add, device_add.....
1974 match = self._INFO_VERSION_RE.search(output.stdout)
1976 raise errors.HotplugError("Try hotplug only in running instances.")
1977 v_major, v_min, _, _ = match.groups()
1978 if (v_major, v_min) < (1, 0):
1979 raise errors.HotplugError("Hotplug not supported for qemu versions < 1.0")
1981 if dev_type == constants.HOTPLUG_TARGET_DISK:
1982 hvp = instance.hvparams
1983 security_model = hvp[constants.HV_SECURITY_MODEL]
1984 use_chroot = hvp[constants.HV_KVM_USE_CHROOT]
1986 raise errors.HotplugError("Disk hotplug is not supported"
1987 " in case of chroot.")
1988 if security_model != constants.HT_SM_NONE:
1989 raise errors.HotplugError("Disk Hotplug is not supported in case"
1990 " security models are used.")
1992 if (dev_type == constants.HOTPLUG_TARGET_NIC and
1993 action == constants.HOTPLUG_ACTION_ADD and not fdsend):
1994 raise errors.HotplugError("Cannot hot-add NIC."
1995 " fdsend python module is missing.")
1998 def _CallHotplugCommand(self, name, cmd):
1999 output = self._CallMonitorCommand(name, cmd)
2000 # TODO: parse output and check if succeeded
2001 for line in output.stdout.splitlines():
2002 logging.info("%s", line)
2005 def _ParseKVMVersion(cls, text):
2006 """Parse the KVM version from the --help output.
2009 @param text: output of kvm --help
2010 @return: (version, v_maj, v_min, v_rev)
2011 @raise errors.HypervisorError: when the KVM version cannot be retrieved
2014 match = cls._VERSION_RE.search(text.splitlines()[0])
2016 raise errors.HypervisorError("Unable to get KVM version")
2018 v_all = match.group(0)
2019 v_maj = int(match.group(1))
2020 v_min = int(match.group(2))
2022 v_rev = int(match.group(4))
2025 return (v_all, v_maj, v_min, v_rev)
2028 def _GetKVMOutput(cls, kvm_path, option):
2029 """Return the output of a kvm invocation
2031 @type kvm_path: string
2032 @param kvm_path: path to the kvm executable
2033 @type option: a key of _KVMOPTS_CMDS
2034 @param option: kvm option to fetch the output from
2035 @return: output a supported kvm invocation
2036 @raise errors.HypervisorError: when the KVM help output cannot be retrieved
2039 assert option in cls._KVMOPTS_CMDS, "Invalid output option"
2041 optlist, can_fail = cls._KVMOPTS_CMDS[option]
2043 result = utils.RunCmd([kvm_path] + optlist)
2044 if result.failed and not can_fail:
2045 raise errors.HypervisorError("Unable to get KVM %s output" %
2046 " ".join(cls._KVMOPTS_CMDS[option]))
2047 return result.output
2050 def _GetKVMVersion(cls, kvm_path):
2051 """Return the installed KVM version.
2053 @return: (version, v_maj, v_min, v_rev)
2054 @raise errors.HypervisorError: when the KVM version cannot be retrieved
2057 return cls._ParseKVMVersion(cls._GetKVMOutput(kvm_path, cls._KVMOPT_HELP))
2060 def _GetDefaultMachineVersion(cls, kvm_path):
2061 """Return the default hardware revision (e.g. pc-1.1)
2064 output = cls._GetKVMOutput(kvm_path, cls._KVMOPT_MLIST)
2065 match = cls._DEFAULT_MACHINE_VERSION_RE.search(output)
2067 return match.group(1)
2071 def StopInstance(self, instance, force=False, retry=False, name=None):
2072 """Stop an instance.
2075 if name is not None and not force:
2076 raise errors.HypervisorError("Cannot shutdown cleanly by name only")
2078 name = instance.name
2079 acpi = instance.hvparams[constants.HV_ACPI]
2082 _, pid, alive = self._InstancePidAlive(name)
2083 if pid > 0 and alive:
2084 if force or not acpi:
2085 utils.KillProcess(pid)
2087 self._CallMonitorCommand(name, "system_powerdown")
2089 def CleanupInstance(self, instance_name):
2090 """Cleanup after a stopped instance
2093 pidfile, pid, alive = self._InstancePidAlive(instance_name)
2094 if pid > 0 and alive:
2095 raise errors.HypervisorError("Cannot cleanup a live instance")
2096 self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
2098 def RebootInstance(self, instance):
2099 """Reboot an instance.
2102 # For some reason if we do a 'send-key ctrl-alt-delete' to the control
2103 # socket the instance will stop, but now power up again. So we'll resort
2104 # to shutdown and restart.
2105 _, _, alive = self._InstancePidAlive(instance.name)
2107 raise errors.HypervisorError("Failed to reboot instance %s:"
2108 " not running" % instance.name)
2109 # StopInstance will delete the saved KVM runtime so:
2110 # ...first load it...
2111 kvm_runtime = self._LoadKVMRuntime(instance)
2112 # ...now we can safely call StopInstance...
2113 if not self.StopInstance(instance):
2114 self.StopInstance(instance, force=True)
2115 # ...and finally we can save it again, and execute it...
2116 self._SaveKVMRuntime(instance, kvm_runtime)
2117 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
2118 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
2119 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp)
2121 def MigrationInfo(self, instance):
2122 """Get instance information to perform a migration.
2124 @type instance: L{objects.Instance}
2125 @param instance: instance to be migrated
2127 @return: content of the KVM runtime file
2130 return self._ReadKVMRuntime(instance.name)
2132 def AcceptInstance(self, instance, info, target):
2133 """Prepare to accept an instance.
2135 @type instance: L{objects.Instance}
2136 @param instance: instance to be accepted
2138 @param info: content of the KVM runtime file on the source node
2139 @type target: string
2140 @param target: target host (usually ip), on this node
2143 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
2144 incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT])
2145 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
2146 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
2147 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp,
2148 incoming=incoming_address)
2150 def FinalizeMigrationDst(self, instance, info, success):
2151 """Finalize the instance migration on the target node.
2153 Stop the incoming mode KVM.
2155 @type instance: L{objects.Instance}
2156 @param instance: instance whose migration is being finalized
2160 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
2161 kvm_nics = kvm_runtime[1]
2163 for nic_seq, nic in enumerate(kvm_nics):
2164 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2165 # Bridged interfaces have already been configured
2168 tap = utils.ReadFile(self._InstanceNICFile(instance.name, nic_seq))
2169 except EnvironmentError, err:
2170 logging.warning("Failed to find host interface for %s NIC #%d: %s",
2171 instance.name, nic_seq, str(err))
2174 self._ConfigureNIC(instance, nic_seq, nic, tap)
2175 except errors.HypervisorError, err:
2176 logging.warning(str(err))
2178 self._WriteKVMRuntime(instance.name, info)
2180 self.StopInstance(instance, force=True)
2182 def MigrateInstance(self, cluster_name, instance, target, live):
2183 """Migrate an instance to a target node.
2185 The migration will not be attempted if the instance is not
2188 @type cluster_name: string
2189 @param cluster_name: name of the cluster
2190 @type instance: L{objects.Instance}
2191 @param instance: the instance to be migrated
2192 @type target: string
2193 @param target: ip address of the target node
2195 @param live: perform a live migration
2198 instance_name = instance.name
2199 port = instance.hvparams[constants.HV_MIGRATION_PORT]
2200 _, _, alive = self._InstancePidAlive(instance_name)
2202 raise errors.HypervisorError("Instance not running, cannot migrate")
2205 self._CallMonitorCommand(instance_name, "stop")
2207 migrate_command = ("migrate_set_speed %dm" %
2208 instance.hvparams[constants.HV_MIGRATION_BANDWIDTH])
2209 self._CallMonitorCommand(instance_name, migrate_command)
2211 migrate_command = ("migrate_set_downtime %dms" %
2212 instance.hvparams[constants.HV_MIGRATION_DOWNTIME])
2213 self._CallMonitorCommand(instance_name, migrate_command)
2215 migrate_command = "migrate -d tcp:%s:%s" % (target, port)
2216 self._CallMonitorCommand(instance_name, migrate_command)
2218 def FinalizeMigrationSource(self, instance, success, live):
2219 """Finalize the instance migration on the source node.
2221 @type instance: L{objects.Instance}
2222 @param instance: the instance that was migrated
2224 @param success: whether the migration succeeded or not
2226 @param live: whether the user requested a live migration or not
2230 pidfile, pid, _ = self._InstancePidAlive(instance.name)
2231 utils.KillProcess(pid)
2232 self._RemoveInstanceRuntimeFiles(pidfile, instance.name)
2234 self._CallMonitorCommand(instance.name, self._CONT_CMD)
2236 def GetMigrationStatus(self, instance):
2237 """Get the migration status
2239 @type instance: L{objects.Instance}
2240 @param instance: the instance that is being migrated
2241 @rtype: L{objects.MigrationStatus}
2242 @return: the status of the current migration (one of
2243 L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional
2244 progress info that can be retrieved from the hypervisor
2247 info_command = "info migrate"
2248 for _ in range(self._MIGRATION_INFO_MAX_BAD_ANSWERS):
2249 result = self._CallMonitorCommand(instance.name, info_command)
2250 match = self._MIGRATION_STATUS_RE.search(result.stdout)
2252 if not result.stdout:
2253 logging.info("KVM: empty 'info migrate' result")
2255 logging.warning("KVM: unknown 'info migrate' result: %s",
2258 status = match.group(1)
2259 if status in constants.HV_KVM_MIGRATION_VALID_STATUSES:
2260 migration_status = objects.MigrationStatus(status=status)
2261 match = self._MIGRATION_PROGRESS_RE.search(result.stdout)
2263 migration_status.transferred_ram = match.group("transferred")
2264 migration_status.total_ram = match.group("total")
2266 return migration_status
2268 logging.warning("KVM: unknown migration status '%s'", status)
2270 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
2272 return objects.MigrationStatus(status=constants.HV_MIGRATION_FAILED)
2274 def BalloonInstanceMemory(self, instance, mem):
2275 """Balloon an instance memory to a certain value.
2277 @type instance: L{objects.Instance}
2278 @param instance: instance to be accepted
2280 @param mem: actual memory size to use for instance runtime
2283 self._CallMonitorCommand(instance.name, "balloon %d" % mem)
2285 def GetNodeInfo(self, hvparams=None):
2286 """Return information about the node.
2288 @type hvparams: dict of strings
2289 @param hvparams: hypervisor parameters, not used in this class
2291 @return: a dict as returned by L{BaseHypervisor.GetLinuxNodeInfo} plus
2293 - hv_version: the hypervisor version in the form (major, minor,
2297 result = self.GetLinuxNodeInfo()
2298 # FIXME: this is the global kvm version, but the actual version can be
2299 # customized as an hv parameter. we should use the nodegroup's default kvm
2300 # path parameter here.
2301 _, v_major, v_min, v_rev = self._GetKVMVersion(constants.KVM_PATH)
2302 result[constants.HV_NODEINFO_KEY_VERSION] = (v_major, v_min, v_rev)
2306 def GetInstanceConsole(cls, instance, primary_node, hvparams, beparams):
2307 """Return a command for connecting to the console of an instance.
2310 if hvparams[constants.HV_SERIAL_CONSOLE]:
2311 cmd = [pathutils.KVM_CONSOLE_WRAPPER,
2312 constants.SOCAT_PATH, utils.ShellQuote(instance.name),
2313 utils.ShellQuote(cls._InstanceMonitor(instance.name)),
2314 "STDIO,%s" % cls._SocatUnixConsoleParams(),
2315 "UNIX-CONNECT:%s" % cls._InstanceSerial(instance.name)]
2316 return objects.InstanceConsole(instance=instance.name,
2317 kind=constants.CONS_SSH,
2318 host=primary_node.name,
2319 user=constants.SSH_CONSOLE_USER,
2322 vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS]
2323 if vnc_bind_address and instance.network_port > constants.VNC_BASE_PORT:
2324 display = instance.network_port - constants.VNC_BASE_PORT
2325 return objects.InstanceConsole(instance=instance.name,
2326 kind=constants.CONS_VNC,
2327 host=vnc_bind_address,
2328 port=instance.network_port,
2331 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2333 return objects.InstanceConsole(instance=instance.name,
2334 kind=constants.CONS_SPICE,
2336 port=instance.network_port)
2338 return objects.InstanceConsole(instance=instance.name,
2339 kind=constants.CONS_MESSAGE,
2340 message=("No serial shell for instance %s" %
2343 def Verify(self, hvparams=None):
2344 """Verify the hypervisor.
2346 Check that the required binaries exist.
2348 @type hvparams: dict of strings
2349 @param hvparams: hypervisor parameters to be verified against, not used here
2351 @return: Problem description if something is wrong, C{None} otherwise
2355 # FIXME: this is the global kvm binary, but the actual path can be
2356 # customized as an hv parameter; we should use the nodegroup's
2357 # default kvm path parameter here.
2358 if not os.path.exists(constants.KVM_PATH):
2359 msgs.append("The KVM binary ('%s') does not exist" % constants.KVM_PATH)
2360 if not os.path.exists(constants.SOCAT_PATH):
2361 msgs.append("The socat binary ('%s') does not exist" %
2362 constants.SOCAT_PATH)
2364 return self._FormatVerifyResults(msgs)
2367 def CheckParameterSyntax(cls, hvparams):
2368 """Check the given parameters for validity.
2370 @type hvparams: dict
2371 @param hvparams: dictionary with parameter names/value
2372 @raise errors.HypervisorError: when a parameter is not valid
2375 super(KVMHypervisor, cls).CheckParameterSyntax(hvparams)
2377 kernel_path = hvparams[constants.HV_KERNEL_PATH]
2379 if not hvparams[constants.HV_ROOT_PATH]:
2380 raise errors.HypervisorError("Need a root partition for the instance,"
2381 " if a kernel is defined")
2383 if (hvparams[constants.HV_VNC_X509_VERIFY] and
2384 not hvparams[constants.HV_VNC_X509]):
2385 raise errors.HypervisorError("%s must be defined, if %s is" %
2386 (constants.HV_VNC_X509,
2387 constants.HV_VNC_X509_VERIFY))
2389 if hvparams[constants.HV_SERIAL_CONSOLE]:
2390 serial_speed = hvparams[constants.HV_SERIAL_SPEED]
2391 valid_speeds = constants.VALID_SERIAL_SPEEDS
2392 if not serial_speed or serial_speed not in valid_speeds:
2393 raise errors.HypervisorError("Invalid serial console speed, must be"
2395 utils.CommaJoin(valid_speeds))
2397 boot_order = hvparams[constants.HV_BOOT_ORDER]
2398 if (boot_order == constants.HT_BO_CDROM and
2399 not hvparams[constants.HV_CDROM_IMAGE_PATH]):
2400 raise errors.HypervisorError("Cannot boot from cdrom without an"
2403 security_model = hvparams[constants.HV_SECURITY_MODEL]
2404 if security_model == constants.HT_SM_USER:
2405 if not hvparams[constants.HV_SECURITY_DOMAIN]:
2406 raise errors.HypervisorError("A security domain (user to run kvm as)"
2407 " must be specified")
2408 elif (security_model == constants.HT_SM_NONE or
2409 security_model == constants.HT_SM_POOL):
2410 if hvparams[constants.HV_SECURITY_DOMAIN]:
2411 raise errors.HypervisorError("Cannot have a security domain when the"
2412 " security model is 'none' or 'pool'")
2414 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2415 spice_ip_version = hvparams[constants.HV_KVM_SPICE_IP_VERSION]
2417 if spice_ip_version != constants.IFACE_NO_IP_VERSION_SPECIFIED:
2418 # if an IP version is specified, the spice_bind parameter must be an
2420 if (netutils.IP4Address.IsValid(spice_bind) and
2421 spice_ip_version != constants.IP4_VERSION):
2422 raise errors.HypervisorError("SPICE: Got an IPv4 address (%s), but"
2423 " the specified IP version is %s" %
2424 (spice_bind, spice_ip_version))
2426 if (netutils.IP6Address.IsValid(spice_bind) and
2427 spice_ip_version != constants.IP6_VERSION):
2428 raise errors.HypervisorError("SPICE: Got an IPv6 address (%s), but"
2429 " the specified IP version is %s" %
2430 (spice_bind, spice_ip_version))
2432 # All the other SPICE parameters depend on spice_bind being set. Raise an
2433 # error if any of them is set without it.
2434 for param in _SPICE_ADDITIONAL_PARAMS:
2436 raise errors.HypervisorError("SPICE: %s requires %s to be set" %
2437 (param, constants.HV_KVM_SPICE_BIND))
2440 def ValidateParameters(cls, hvparams):
2441 """Check the given parameters for validity.
2443 @type hvparams: dict
2444 @param hvparams: dictionary with parameter names/value
2445 @raise errors.HypervisorError: when a parameter is not valid
2448 super(KVMHypervisor, cls).ValidateParameters(hvparams)
2450 kvm_path = hvparams[constants.HV_KVM_PATH]
2452 security_model = hvparams[constants.HV_SECURITY_MODEL]
2453 if security_model == constants.HT_SM_USER:
2454 username = hvparams[constants.HV_SECURITY_DOMAIN]
2456 pwd.getpwnam(username)
2458 raise errors.HypervisorError("Unknown security domain user %s"
2460 vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS]
2461 if vnc_bind_address:
2462 bound_to_addr = netutils.IP4Address.IsValid(vnc_bind_address)
2463 is_interface = netutils.IsValidInterface(vnc_bind_address)
2464 is_path = utils.IsNormAbsPath(vnc_bind_address)
2465 if not bound_to_addr and not is_interface and not is_path:
2466 raise errors.HypervisorError("VNC: The %s parameter must be either"
2467 " a valid IP address, an interface name,"
2468 " or an absolute path" %
2469 constants.HV_KVM_SPICE_BIND)
2471 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2473 # only one of VNC and SPICE can be used currently.
2474 if hvparams[constants.HV_VNC_BIND_ADDRESS]:
2475 raise errors.HypervisorError("Both SPICE and VNC are configured, but"
2476 " only one of them can be used at a"
2479 # check that KVM supports SPICE
2480 kvmhelp = cls._GetKVMOutput(kvm_path, cls._KVMOPT_HELP)
2481 if not cls._SPICE_RE.search(kvmhelp):
2482 raise errors.HypervisorError("SPICE is configured, but it is not"
2483 " supported according to 'kvm --help'")
2485 # if spice_bind is not an IP address, it must be a valid interface
2486 bound_to_addr = (netutils.IP4Address.IsValid(spice_bind) or
2487 netutils.IP6Address.IsValid(spice_bind))
2488 if not bound_to_addr and not netutils.IsValidInterface(spice_bind):
2489 raise errors.HypervisorError("SPICE: The %s parameter must be either"
2490 " a valid IP address or interface name" %
2491 constants.HV_KVM_SPICE_BIND)
2493 machine_version = hvparams[constants.HV_KVM_MACHINE_VERSION]
2495 output = cls._GetKVMOutput(kvm_path, cls._KVMOPT_MLIST)
2496 if not cls._CHECK_MACHINE_VERSION_RE(machine_version).search(output):
2497 raise errors.HypervisorError("Unsupported machine version: %s" %
2501 def PowercycleNode(cls, hvparams=None):
2502 """KVM powercycle, just a wrapper over Linux powercycle.
2504 @type hvparams: dict of strings
2505 @param hvparams: hypervisor params to be used on this node
2508 cls.LinuxPowercycle()