4 # Copyright (C) 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
40 from bitarray import bitarray
42 import affinity # pylint: disable=F0401
46 from ganeti import utils
47 from ganeti import constants
48 from ganeti import errors
49 from ganeti import serializer
50 from ganeti import objects
51 from ganeti import uidpool
52 from ganeti import ssconf
53 from ganeti import netutils
54 from ganeti import pathutils
55 from ganeti.hypervisor import hv_base
56 from ganeti.utils import wrapper as utils_wrapper
59 _KVM_NETWORK_SCRIPT = pathutils.CONF_DIR + "/kvm-vif-bridge"
60 _KVM_START_PAUSED_FLAG = "-S"
62 # TUN/TAP driver constants, taken from <linux/if_tun.h>
63 # They are architecture-independent and already hardcoded in qemu-kvm source,
64 # so we can safely include them here.
65 TUNSETIFF = 0x400454ca
66 TUNGETIFF = 0x800454d2
67 TUNGETFEATURES = 0x800454cf
72 #: SPICE parameters which depend on L{constants.HV_KVM_SPICE_BIND}
73 _SPICE_ADDITIONAL_PARAMS = frozenset([
74 constants.HV_KVM_SPICE_IP_VERSION,
75 constants.HV_KVM_SPICE_PASSWORD_FILE,
76 constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR,
77 constants.HV_KVM_SPICE_JPEG_IMG_COMPR,
78 constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR,
79 constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION,
80 constants.HV_KVM_SPICE_USE_TLS,
83 # Constant bitarray that reflects to a free pci slot
84 # Use it with bitarray.search()
85 _AVAILABLE_PCI_SLOT = bitarray("0")
87 # below constants show the format of runtime file
88 # the nics are in second possition, while the disks in 4th (last)
89 # moreover disk entries are stored in tupples of L{objects.Disk}, dev_path
90 _KVM_NICS_RUNTIME_INDEX = 1
91 _KVM_DISKS_RUNTIME_INDEX = 3
92 _DEVICE_RUNTIME_INDEX = {
93 constants.HOTPLUG_TARGET_DISK: _KVM_DISKS_RUNTIME_INDEX,
94 constants.HOTPLUG_TARGET_NIC: _KVM_NICS_RUNTIME_INDEX
96 _FIND_RUNTIME_ENTRY = {
97 constants.HOTPLUG_TARGET_NIC:
98 lambda nic, kvm_nics: [n for n in kvm_nics if n.uuid == nic.uuid],
99 constants.HOTPLUG_TARGET_DISK:
100 lambda disk, kvm_disks: [(d, l) for (d, l) in kvm_disks
101 if d.uuid == disk.uuid]
104 constants.HOTPLUG_TARGET_NIC: lambda d: d,
105 constants.HOTPLUG_TARGET_DISK: lambda (d, e): d
108 constants.HOTPLUG_TARGET_NIC: lambda d, e: d,
109 constants.HOTPLUG_TARGET_DISK: lambda d, e: (d, e)
113 def _GenerateDeviceKVMId(dev_type, dev):
114 """Helper function to generate a unique device name used by KVM
116 QEMU monitor commands use names to identify devices. Here we use their pci
117 slot and a part of their UUID to name them. dev.pci might be None for old
118 devices in the cluster.
120 @type dev_type: sting
121 @param dev_type: device type of param dev
122 @type dev: L{objects.Disk} or L{objects.NIC}
123 @param dev: the device object for which we generate a kvm name
124 @raise errors.HotplugError: in case a device has no pci slot (old devices)
129 raise errors.HotplugError("Hotplug is not supported for %s with UUID %s" %
130 (dev_type, dev.uuid))
132 return "%s-%s-pci-%d" % (dev_type.lower(), dev.uuid.split("-")[0], dev.pci)
135 def _UpdatePCISlots(dev, pci_reservations):
136 """Update pci configuration for a stopped instance
138 If dev has a pci slot the reserve it, else find first available
139 in pci_reservations bitarray. It acts on the same objects passed
140 as params so there is no need to return anything.
142 @type dev: L{objects.Disk} or L{objects.NIC}
143 @param dev: the device object for which we update its pci slot
144 @type pci_reservations: bitarray
145 @param pci_reservations: existing pci reservations for an instance
146 @raise errors.HotplugError: in case an instance has all its slot occupied
151 else: # pylint: disable=E1103
152 [free] = pci_reservations.search(_AVAILABLE_PCI_SLOT, 1)
154 raise errors.HypervisorError("All PCI slots occupied")
157 pci_reservations[free] = True
160 def _GetExistingDeviceInfo(dev_type, device, runtime):
161 """Helper function to get an existing device inside the runtime file
163 Used when an instance is running. Load kvm runtime file and search
164 for a device based on its type and uuid.
166 @type dev_type: sting
167 @param dev_type: device type of param dev
168 @type device: L{objects.Disk} or L{objects.NIC}
169 @param device: the device object for which we generate a kvm name
170 @type runtime: tuple (cmd, nics, hvparams, disks)
171 @param runtime: the runtime data to search for the device
172 @raise errors.HotplugError: in case the requested device does not
173 exist (e.g. device has been added without --hotplug option) or
174 device info has not pci slot (e.g. old devices in the cluster)
177 index = _DEVICE_RUNTIME_INDEX[dev_type]
178 found = _FIND_RUNTIME_ENTRY[dev_type](device, runtime[index])
180 raise errors.HotplugError("Cannot find runtime info for %s with UUID %s" %
181 (dev_type, device.uuid))
186 def _GetTunFeatures(fd, _ioctl=fcntl.ioctl):
187 """Retrieves supported TUN features from file descriptor.
189 @see: L{_ProbeTapVnetHdr}
192 req = struct.pack("I", 0)
194 buf = _ioctl(fd, TUNGETFEATURES, req)
195 except EnvironmentError, err:
196 logging.warning("ioctl(TUNGETFEATURES) failed: %s", err)
199 (flags, ) = struct.unpack("I", buf)
203 def _ProbeTapVnetHdr(fd, _features_fn=_GetTunFeatures):
204 """Check whether to enable the IFF_VNET_HDR flag.
206 To do this, _all_ of the following conditions must be met:
207 1. TUNGETFEATURES ioctl() *must* be implemented
208 2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag
209 3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in
210 drivers/net/tun.c there is no way to test this until after the tap device
211 has been created using TUNSETIFF, and there is no way to change the
212 IFF_VNET_HDR flag after creating the interface, catch-22! However both
213 TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27,
214 thus we can expect TUNGETIFF to be present if TUNGETFEATURES is.
217 @param fd: the file descriptor of /dev/net/tun
220 flags = _features_fn(fd)
226 result = bool(flags & IFF_VNET_HDR)
229 logging.warning("Kernel does not support IFF_VNET_HDR, not enabling")
234 def _OpenTap(vnet_hdr=True):
235 """Open a new tap device and return its file descriptor.
237 This is intended to be used by a qemu-type hypervisor together with the -net
238 tap,fd=<fd> command line parameter.
240 @type vnet_hdr: boolean
241 @param vnet_hdr: Enable the VNET Header
242 @return: (ifname, tapfd)
247 tapfd = os.open("/dev/net/tun", os.O_RDWR)
248 except EnvironmentError:
249 raise errors.HypervisorError("Failed to open /dev/net/tun")
251 flags = IFF_TAP | IFF_NO_PI
253 if vnet_hdr and _ProbeTapVnetHdr(tapfd):
254 flags |= IFF_VNET_HDR
256 # The struct ifreq ioctl request (see netdevice(7))
257 ifr = struct.pack("16sh", "", flags)
260 res = fcntl.ioctl(tapfd, TUNSETIFF, ifr)
261 except EnvironmentError, err:
262 raise errors.HypervisorError("Failed to allocate a new TAP device: %s" %
265 # Get the interface name from the ioctl
266 ifname = struct.unpack("16sh", res)[0].strip("\x00")
267 return (ifname, tapfd)
271 """QEMU Messaging Protocol (QMP) message.
274 def __init__(self, data):
275 """Creates a new QMP message based on the passed data.
278 if not isinstance(data, dict):
279 raise TypeError("QmpMessage must be initialized with a dict")
283 def __getitem__(self, field_name):
284 """Get the value of the required field if present, or None.
286 Overrides the [] operator to provide access to the message data,
287 returning None if the required item is not in the message
288 @return: the value of the field_name field, or None if field_name
289 is not contained in the message
292 return self.data.get(field_name, None)
294 def __setitem__(self, field_name, field_value):
295 """Set the value of the required field_name to field_value.
298 self.data[field_name] = field_value
301 def BuildFromJsonString(json_string):
302 """Build a QmpMessage from a JSON encoded string.
304 @type json_string: str
305 @param json_string: JSON string representing the message
306 @rtype: L{QmpMessage}
307 @return: a L{QmpMessage} built from json_string
311 data = serializer.LoadJson(json_string)
312 return QmpMessage(data)
315 # The protocol expects the JSON object to be sent as a single line.
316 return serializer.DumpJson(self.data)
318 def __eq__(self, other):
319 # When comparing two QmpMessages, we are interested in comparing
320 # their internal representation of the message data
321 return self.data == other.data
325 """Connection to the QEMU Monitor using the QEMU Monitor Protocol (QMP).
328 _FIRST_MESSAGE_KEY = "QMP"
331 _RETURN_KEY = RETURN_KEY = "return"
332 _ACTUAL_KEY = ACTUAL_KEY = "actual"
333 _ERROR_CLASS_KEY = "class"
334 _ERROR_DATA_KEY = "data"
335 _ERROR_DESC_KEY = "desc"
336 _EXECUTE_KEY = "execute"
337 _ARGUMENTS_KEY = "arguments"
338 _CAPABILITIES_COMMAND = "qmp_capabilities"
339 _MESSAGE_END_TOKEN = "\r\n"
342 def __init__(self, monitor_filename):
343 """Instantiates the QmpConnection object.
345 @type monitor_filename: string
346 @param monitor_filename: the filename of the UNIX raw socket on which the
347 QMP monitor is listening
350 self.monitor_filename = monitor_filename
351 self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
352 # We want to fail if the server doesn't send a complete message
353 # in a reasonable amount of time
354 self.sock.settimeout(self._SOCKET_TIMEOUT)
355 self._connected = False
358 def _check_socket(self):
361 sock_stat = os.stat(self.monitor_filename)
362 except EnvironmentError, err:
363 if err.errno == errno.ENOENT:
364 raise errors.HypervisorError("No qmp socket found")
366 raise errors.HypervisorError("Error checking qmp socket: %s",
367 utils.ErrnoOrStr(err))
368 if not stat.S_ISSOCK(sock_stat.st_mode):
369 raise errors.HypervisorError("Qmp socket is not a socket")
371 def _check_connection(self):
372 """Make sure that the connection is established.
375 if not self._connected:
376 raise errors.ProgrammerError("To use a QmpConnection you need to first"
377 " invoke connect() on it")
380 """Connects to the QMP monitor.
382 Connects to the UNIX socket and makes sure that we can actually send and
383 receive data to the kvm instance via QMP.
385 @raise errors.HypervisorError: when there are communication errors
386 @raise errors.ProgrammerError: when there are data serialization errors
390 raise errors.ProgrammerError("Cannot connect twice")
394 # Check file existance/stuff
396 self.sock.connect(self.monitor_filename)
397 except EnvironmentError:
398 raise errors.HypervisorError("Can't connect to qmp socket")
399 self._connected = True
401 # Check if we receive a correct greeting message from the server
402 # (As per the QEMU Protocol Specification 0.1 - section 2.2)
403 greeting = self._Recv()
404 if not greeting[self._FIRST_MESSAGE_KEY]:
405 self._connected = False
406 raise errors.HypervisorError("kvm: QMP communication error (wrong"
409 # Let's put the monitor in command mode using the qmp_capabilities
410 # command, or else no command will be executable.
411 # (As per the QEMU Protocol Specification 0.1 - section 4)
412 self.Execute(self._CAPABILITIES_COMMAND)
414 def _ParseMessage(self, buf):
415 """Extract and parse a QMP message from the given buffer.
417 Seeks for a QMP message in the given buf. If found, it parses it and
418 returns it together with the rest of the characters in the buf.
419 If no message is found, returns None and the whole buffer.
421 @raise errors.ProgrammerError: when there are data serialization errors
425 # Check if we got the message end token (CRLF, as per the QEMU Protocol
426 # Specification 0.1 - Section 2.1.1)
427 pos = buf.find(self._MESSAGE_END_TOKEN)
430 message = QmpMessage.BuildFromJsonString(buf[:pos + 1])
431 except Exception, err:
432 raise errors.ProgrammerError("QMP data serialization error: %s" % err)
435 return (message, buf)
438 """Receives a message from QMP and decodes the received JSON object.
441 @return: the received message
442 @raise errors.HypervisorError: when there are communication errors
443 @raise errors.ProgrammerError: when there are data serialization errors
446 self._check_connection()
448 # Check if there is already a message in the buffer
449 (message, self._buf) = self._ParseMessage(self._buf)
453 recv_buffer = StringIO.StringIO(self._buf)
454 recv_buffer.seek(len(self._buf))
457 data = self.sock.recv(4096)
460 recv_buffer.write(data)
462 (message, self._buf) = self._ParseMessage(recv_buffer.getvalue())
466 except socket.timeout, err:
467 raise errors.HypervisorError("Timeout while receiving a QMP message: "
469 except socket.error, err:
470 raise errors.HypervisorError("Unable to receive data from KVM using the"
471 " QMP protocol: %s" % err)
473 def _Send(self, message):
474 """Encodes and sends a message to KVM using QMP.
476 @type message: QmpMessage
477 @param message: message to send to KVM
478 @raise errors.HypervisorError: when there are communication errors
479 @raise errors.ProgrammerError: when there are data serialization errors
482 self._check_connection()
484 message_str = str(message)
485 except Exception, err:
486 raise errors.ProgrammerError("QMP data deserialization error: %s" % err)
489 self.sock.sendall(message_str)
490 except socket.timeout, err:
491 raise errors.HypervisorError("Timeout while sending a QMP message: "
492 "%s (%s)" % (err.string, err.errno))
493 except socket.error, err:
494 raise errors.HypervisorError("Unable to send data from KVM using the"
495 " QMP protocol: %s" % err)
497 def Execute(self, command, arguments=None):
498 """Executes a QMP command and returns the response of the server.
501 @param command: the command to execute
502 @type arguments: dict
503 @param arguments: dictionary of arguments to be passed to the command
505 @return: dictionary representing the received JSON object
506 @raise errors.HypervisorError: when there are communication errors
507 @raise errors.ProgrammerError: when there are data serialization errors
510 self._check_connection()
511 message = QmpMessage({self._EXECUTE_KEY: command})
513 message[self._ARGUMENTS_KEY] = arguments
516 # Events can occur between the sending of the command and the reception
517 # of the response, so we need to filter out messages with the event key.
519 response = self._Recv()
520 err = response[self._ERROR_KEY]
522 raise errors.HypervisorError("kvm: error executing the %s"
523 " command: %s (%s, %s):" %
525 err[self._ERROR_DESC_KEY],
526 err[self._ERROR_CLASS_KEY],
527 err[self._ERROR_DATA_KEY]))
529 elif not response[self._EVENT_KEY]:
533 class KVMHypervisor(hv_base.BaseHypervisor):
534 """KVM hypervisor interface
539 _ROOT_DIR = pathutils.RUN_DIR + "/kvm-hypervisor"
540 _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids
541 _UIDS_DIR = _ROOT_DIR + "/uid" # contains instances reserved uids
542 _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets
543 _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data
544 _NICS_DIR = _ROOT_DIR + "/nic" # contains instances nic <-> tap associations
545 _KEYMAP_DIR = _ROOT_DIR + "/keymap" # contains instances keymaps
546 # KVM instances with chroot enabled are started in empty chroot directories.
547 _CHROOT_DIR = _ROOT_DIR + "/chroot" # for empty chroot directories
548 # After an instance is stopped, its chroot directory is removed.
549 # If the chroot directory is not empty, it can't be removed.
550 # A non-empty chroot directory indicates a possible security incident.
551 # To support forensics, the non-empty chroot directory is quarantined in
552 # a separate directory, called 'chroot-quarantine'.
553 _CHROOT_QUARANTINE_DIR = _ROOT_DIR + "/chroot-quarantine"
554 _DIRS = [_ROOT_DIR, _PIDS_DIR, _UIDS_DIR, _CTRL_DIR, _CONF_DIR, _NICS_DIR,
555 _CHROOT_DIR, _CHROOT_QUARANTINE_DIR, _KEYMAP_DIR]
558 constants.HV_KVM_PATH: hv_base.REQ_FILE_CHECK,
559 constants.HV_KERNEL_PATH: hv_base.OPT_FILE_CHECK,
560 constants.HV_INITRD_PATH: hv_base.OPT_FILE_CHECK,
561 constants.HV_ROOT_PATH: hv_base.NO_CHECK,
562 constants.HV_KERNEL_ARGS: hv_base.NO_CHECK,
563 constants.HV_ACPI: hv_base.NO_CHECK,
564 constants.HV_SERIAL_CONSOLE: hv_base.NO_CHECK,
565 constants.HV_SERIAL_SPEED: hv_base.NO_CHECK,
566 constants.HV_VNC_BIND_ADDRESS:
567 (False, lambda x: (netutils.IP4Address.IsValid(x) or
568 utils.IsNormAbsPath(x)),
569 "The VNC bind address must be either a valid IP address or an absolute"
570 " pathname", None, None),
571 constants.HV_VNC_TLS: hv_base.NO_CHECK,
572 constants.HV_VNC_X509: hv_base.OPT_DIR_CHECK,
573 constants.HV_VNC_X509_VERIFY: hv_base.NO_CHECK,
574 constants.HV_VNC_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
575 constants.HV_KVM_SPICE_BIND: hv_base.NO_CHECK, # will be checked later
576 constants.HV_KVM_SPICE_IP_VERSION:
577 (False, lambda x: (x == constants.IFACE_NO_IP_VERSION_SPECIFIED or
578 x in constants.VALID_IP_VERSIONS),
579 "The SPICE IP version should be 4 or 6",
581 constants.HV_KVM_SPICE_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
582 constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR:
584 False, constants.HT_KVM_SPICE_VALID_LOSSLESS_IMG_COMPR_OPTIONS),
585 constants.HV_KVM_SPICE_JPEG_IMG_COMPR:
587 False, constants.HT_KVM_SPICE_VALID_LOSSY_IMG_COMPR_OPTIONS),
588 constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR:
590 False, constants.HT_KVM_SPICE_VALID_LOSSY_IMG_COMPR_OPTIONS),
591 constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION:
593 False, constants.HT_KVM_SPICE_VALID_VIDEO_STREAM_DETECTION_OPTIONS),
594 constants.HV_KVM_SPICE_AUDIO_COMPR: hv_base.NO_CHECK,
595 constants.HV_KVM_SPICE_USE_TLS: hv_base.NO_CHECK,
596 constants.HV_KVM_SPICE_TLS_CIPHERS: hv_base.NO_CHECK,
597 constants.HV_KVM_SPICE_USE_VDAGENT: hv_base.NO_CHECK,
598 constants.HV_KVM_FLOPPY_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
599 constants.HV_CDROM_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
600 constants.HV_KVM_CDROM2_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
601 constants.HV_BOOT_ORDER:
602 hv_base.ParamInSet(True, constants.HT_KVM_VALID_BO_TYPES),
603 constants.HV_NIC_TYPE:
604 hv_base.ParamInSet(True, constants.HT_KVM_VALID_NIC_TYPES),
605 constants.HV_DISK_TYPE:
606 hv_base.ParamInSet(True, constants.HT_KVM_VALID_DISK_TYPES),
607 constants.HV_KVM_CDROM_DISK_TYPE:
608 hv_base.ParamInSet(False, constants.HT_KVM_VALID_DISK_TYPES),
609 constants.HV_USB_MOUSE:
610 hv_base.ParamInSet(False, constants.HT_KVM_VALID_MOUSE_TYPES),
611 constants.HV_KEYMAP: hv_base.NO_CHECK,
612 constants.HV_MIGRATION_PORT: hv_base.REQ_NET_PORT_CHECK,
613 constants.HV_MIGRATION_BANDWIDTH: hv_base.REQ_NONNEGATIVE_INT_CHECK,
614 constants.HV_MIGRATION_DOWNTIME: hv_base.REQ_NONNEGATIVE_INT_CHECK,
615 constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK,
616 constants.HV_USE_LOCALTIME: hv_base.NO_CHECK,
617 constants.HV_DISK_CACHE:
618 hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES),
619 constants.HV_SECURITY_MODEL:
620 hv_base.ParamInSet(True, constants.HT_KVM_VALID_SM_TYPES),
621 constants.HV_SECURITY_DOMAIN: hv_base.NO_CHECK,
622 constants.HV_KVM_FLAG:
623 hv_base.ParamInSet(False, constants.HT_KVM_FLAG_VALUES),
624 constants.HV_VHOST_NET: hv_base.NO_CHECK,
625 constants.HV_KVM_USE_CHROOT: hv_base.NO_CHECK,
626 constants.HV_MEM_PATH: hv_base.OPT_DIR_CHECK,
627 constants.HV_REBOOT_BEHAVIOR:
628 hv_base.ParamInSet(True, constants.REBOOT_BEHAVIORS),
629 constants.HV_CPU_MASK: hv_base.OPT_MULTI_CPU_MASK_CHECK,
630 constants.HV_CPU_TYPE: hv_base.NO_CHECK,
631 constants.HV_CPU_CORES: hv_base.OPT_NONNEGATIVE_INT_CHECK,
632 constants.HV_CPU_THREADS: hv_base.OPT_NONNEGATIVE_INT_CHECK,
633 constants.HV_CPU_SOCKETS: hv_base.OPT_NONNEGATIVE_INT_CHECK,
634 constants.HV_SOUNDHW: hv_base.NO_CHECK,
635 constants.HV_USB_DEVICES: hv_base.NO_CHECK,
636 constants.HV_VGA: hv_base.NO_CHECK,
637 constants.HV_KVM_EXTRA: hv_base.NO_CHECK,
638 constants.HV_KVM_MACHINE_VERSION: hv_base.NO_CHECK,
639 constants.HV_VNET_HDR: hv_base.NO_CHECK,
643 _VIRTIO_NET_PCI = "virtio-net-pci"
644 _VIRTIO_BLK_PCI = "virtio-blk-pci"
646 _MIGRATION_STATUS_RE = re.compile("Migration\s+status:\s+(\w+)",
648 _MIGRATION_PROGRESS_RE = \
649 re.compile(r"\s*transferred\s+ram:\s+(?P<transferred>\d+)\s+kbytes\s*\n"
650 r"\s*remaining\s+ram:\s+(?P<remaining>\d+)\s+kbytes\s*\n"
651 r"\s*total\s+ram:\s+(?P<total>\d+)\s+kbytes\s*\n", re.I)
653 _MIGRATION_INFO_MAX_BAD_ANSWERS = 5
654 _MIGRATION_INFO_RETRY_DELAY = 2
656 _VERSION_RE = re.compile(r"\b(\d+)\.(\d+)(\.(\d+))?\b")
658 _CPU_INFO_RE = re.compile(r"cpu\s+\#(\d+).*thread_id\s*=\s*(\d+)", re.I)
659 _CPU_INFO_CMD = "info cpus"
662 _DEFAULT_MACHINE_VERSION_RE = re.compile(r"^(\S+).*\(default\)", re.M)
663 _CHECK_MACHINE_VERSION_RE = \
664 staticmethod(lambda x: re.compile(r"^(%s)[ ]+.*PC" % x, re.M))
666 _QMP_RE = re.compile(r"^-qmp\s", re.M)
667 _SPICE_RE = re.compile(r"^-spice\s", re.M)
668 _VHOST_RE = re.compile(r"^-net\s.*,vhost=on|off", re.M)
669 _ENABLE_KVM_RE = re.compile(r"^-enable-kvm\s", re.M)
670 _DISABLE_KVM_RE = re.compile(r"^-disable-kvm\s", re.M)
671 _NETDEV_RE = re.compile(r"^-netdev\s", re.M)
672 _DISPLAY_RE = re.compile(r"^-display\s", re.M)
673 _MACHINE_RE = re.compile(r"^-machine\s", re.M)
674 _VIRTIO_NET_RE = re.compile(r"^name \"%s\"" % _VIRTIO_NET_PCI, re.M)
675 _VIRTIO_BLK_RE = re.compile(r"^name \"%s\"" % _VIRTIO_BLK_PCI, re.M)
676 # match -drive.*boot=on|off on different lines, but in between accept only
677 # dashes not preceeded by a new line (which would mean another option
678 # different than -drive is starting)
679 _BOOT_RE = re.compile(r"^-drive\s([^-]|(?<!^)-)*,boot=on\|off", re.M | re.S)
681 _INFO_PCI_RE = re.compile(r'Bus.*device[ ]*(\d+).*')
682 _INFO_PCI_CMD = "info pci"
684 re.compile(r'^QEMU (\d+)\.(\d+)(\.(\d+))?.*monitor.*', re.M)
685 _INFO_VERSION_CMD = "info version"
687 _DEFAULT_PCI_RESERVATIONS = "11110000000000000000000000000000"
692 ANCILLARY_FILES_OPT = [
696 # Supported kvm options to get output from
697 _KVMOPT_HELP = "help"
698 _KVMOPT_MLIST = "mlist"
699 _KVMOPT_DEVICELIST = "devicelist"
701 # Command to execute to get the output from kvm, and whether to
702 # accept the output even on failure.
704 _KVMOPT_HELP: (["--help"], False),
705 _KVMOPT_MLIST: (["-M", "?"], False),
706 _KVMOPT_DEVICELIST: (["-device", "?"], True),
710 hv_base.BaseHypervisor.__init__(self)
711 # Let's make sure the directories we need exist, even if the RUN_DIR lives
712 # in a tmpfs filesystem or has been otherwise wiped out.
713 dirs = [(dname, constants.RUN_DIRS_MODE) for dname in self._DIRS]
714 utils.EnsureDirs(dirs)
717 def _InstancePidFile(cls, instance_name):
718 """Returns the instance pidfile.
721 return utils.PathJoin(cls._PIDS_DIR, instance_name)
724 def _InstanceUidFile(cls, instance_name):
725 """Returns the instance uidfile.
728 return utils.PathJoin(cls._UIDS_DIR, instance_name)
731 def _InstancePidInfo(cls, pid):
732 """Check pid file for instance information.
734 Check that a pid file is associated with an instance, and retrieve
735 information from its command line.
737 @type pid: string or int
738 @param pid: process id of the instance to check
740 @return: (instance_name, memory, vcpus)
741 @raise errors.HypervisorError: when an instance cannot be found
744 alive = utils.IsProcessAlive(pid)
746 raise errors.HypervisorError("Cannot get info for pid %s" % pid)
748 cmdline_file = utils.PathJoin("/proc", str(pid), "cmdline")
750 cmdline = utils.ReadFile(cmdline_file)
751 except EnvironmentError, err:
752 raise errors.HypervisorError("Can't open cmdline file for pid %s: %s" %
759 arg_list = cmdline.split("\x00")
761 arg = arg_list.pop(0)
763 instance = arg_list.pop(0)
765 memory = int(arg_list.pop(0))
767 vcpus = int(arg_list.pop(0).split(",")[0])
770 raise errors.HypervisorError("Pid %s doesn't contain a ganeti kvm"
773 return (instance, memory, vcpus)
775 def _InstancePidAlive(self, instance_name):
776 """Returns the instance pidfile, pid, and liveness.
778 @type instance_name: string
779 @param instance_name: instance name
781 @return: (pid file name, pid, liveness)
784 pidfile = self._InstancePidFile(instance_name)
785 pid = utils.ReadPidFile(pidfile)
789 cmd_instance = self._InstancePidInfo(pid)[0]
790 alive = (cmd_instance == instance_name)
791 except errors.HypervisorError:
794 return (pidfile, pid, alive)
796 def _CheckDown(self, instance_name):
797 """Raises an error unless the given instance is down.
800 alive = self._InstancePidAlive(instance_name)[2]
802 raise errors.HypervisorError("Failed to start instance %s: %s" %
803 (instance_name, "already running"))
806 def _InstanceMonitor(cls, instance_name):
807 """Returns the instance monitor socket name
810 return utils.PathJoin(cls._CTRL_DIR, "%s.monitor" % instance_name)
813 def _InstanceSerial(cls, instance_name):
814 """Returns the instance serial socket name
817 return utils.PathJoin(cls._CTRL_DIR, "%s.serial" % instance_name)
820 def _InstanceQmpMonitor(cls, instance_name):
821 """Returns the instance serial QMP socket name
824 return utils.PathJoin(cls._CTRL_DIR, "%s.qmp" % instance_name)
827 def _SocatUnixConsoleParams():
828 """Returns the correct parameters for socat
830 If we have a new-enough socat we can use raw mode with an escape character.
833 if constants.SOCAT_USE_ESCAPE:
834 return "raw,echo=0,escape=%s" % constants.SOCAT_ESCAPE_CODE
836 return "echo=0,icanon=0"
839 def _InstanceKVMRuntime(cls, instance_name):
840 """Returns the instance KVM runtime filename
843 return utils.PathJoin(cls._CONF_DIR, "%s.runtime" % instance_name)
846 def _InstanceChrootDir(cls, instance_name):
847 """Returns the name of the KVM chroot dir of the instance
850 return utils.PathJoin(cls._CHROOT_DIR, instance_name)
853 def _InstanceNICDir(cls, instance_name):
854 """Returns the name of the directory holding the tap device files for a
858 return utils.PathJoin(cls._NICS_DIR, instance_name)
861 def _InstanceNICFile(cls, instance_name, seq):
862 """Returns the name of the file containing the tap device for a given NIC
865 return utils.PathJoin(cls._InstanceNICDir(instance_name), str(seq))
868 def _InstanceKeymapFile(cls, instance_name):
869 """Returns the name of the file containing the keymap for a given instance
872 return utils.PathJoin(cls._KEYMAP_DIR, instance_name)
875 def _TryReadUidFile(cls, uid_file):
876 """Try to read a uid file
879 if os.path.exists(uid_file):
881 uid = int(utils.ReadOneLineFile(uid_file))
883 except EnvironmentError:
884 logging.warning("Can't read uid file", exc_info=True)
885 except (TypeError, ValueError):
886 logging.warning("Can't parse uid file contents", exc_info=True)
890 def _RemoveInstanceRuntimeFiles(cls, pidfile, instance_name):
891 """Removes an instance's rutime sockets/files/dirs.
894 utils.RemoveFile(pidfile)
895 utils.RemoveFile(cls._InstanceMonitor(instance_name))
896 utils.RemoveFile(cls._InstanceSerial(instance_name))
897 utils.RemoveFile(cls._InstanceQmpMonitor(instance_name))
898 utils.RemoveFile(cls._InstanceKVMRuntime(instance_name))
899 utils.RemoveFile(cls._InstanceKeymapFile(instance_name))
900 uid_file = cls._InstanceUidFile(instance_name)
901 uid = cls._TryReadUidFile(uid_file)
902 utils.RemoveFile(uid_file)
904 uidpool.ReleaseUid(uid)
906 shutil.rmtree(cls._InstanceNICDir(instance_name))
908 if err.errno != errno.ENOENT:
911 chroot_dir = cls._InstanceChrootDir(instance_name)
912 utils.RemoveDir(chroot_dir)
914 if err.errno == errno.ENOTEMPTY:
915 # The chroot directory is expected to be empty, but it isn't.
916 new_chroot_dir = tempfile.mkdtemp(dir=cls._CHROOT_QUARANTINE_DIR,
919 utils.TimestampForFilename()))
920 logging.warning("The chroot directory of instance %s can not be"
921 " removed as it is not empty. Moving it to the"
922 " quarantine instead. Please investigate the"
923 " contents (%s) and clean up manually",
924 instance_name, new_chroot_dir)
925 utils.RenameFile(chroot_dir, new_chroot_dir)
930 def _ConfigureNIC(instance, seq, nic, tap):
931 """Run the network configuration script for a specified NIC
933 @param instance: instance we're acting on
934 @type instance: instance object
935 @param seq: nic sequence number
937 @param nic: nic we're acting on
938 @type nic: nic object
939 @param tap: the host's tap interface this NIC corresponds to
944 tags = " ".join(instance.tags)
949 "PATH": "%s:/sbin:/usr/sbin" % os.environ["PATH"],
950 "INSTANCE": instance.name,
952 "MODE": nic.nicparams[constants.NIC_MODE],
954 "INTERFACE_INDEX": str(seq),
961 if nic.nicparams[constants.NIC_LINK]:
962 env["LINK"] = nic.nicparams[constants.NIC_LINK]
965 n = objects.Network.FromDict(nic.netinfo)
966 env.update(n.HooksDict())
968 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
969 env["BRIDGE"] = nic.nicparams[constants.NIC_LINK]
971 result = utils.RunCmd([pathutils.KVM_IFUP, tap], env=env)
973 raise errors.HypervisorError("Failed to configure interface %s: %s;"
974 " network configuration script output: %s" %
975 (tap, result.fail_reason, result.output))
978 def _VerifyAffinityPackage():
980 raise errors.HypervisorError("affinity Python package not"
981 " found; cannot use CPU pinning under KVM")
984 def _BuildAffinityCpuMask(cpu_list):
985 """Create a CPU mask suitable for sched_setaffinity from a list of
988 See man taskset for more info on sched_setaffinity masks.
989 For example: [ 0, 2, 5, 6 ] will return 101 (0x65, 0..01100101).
991 @type cpu_list: list of int
992 @param cpu_list: list of physical CPU numbers to map to vCPUs in order
994 @return: a bit mask of CPU affinities
997 if cpu_list == constants.CPU_PINNING_OFF:
998 return constants.CPU_PINNING_ALL_KVM
1000 return sum(2 ** cpu for cpu in cpu_list)
1003 def _AssignCpuAffinity(cls, cpu_mask, process_id, thread_dict):
1004 """Change CPU affinity for running VM according to given CPU mask.
1006 @param cpu_mask: CPU mask as given by the user. e.g. "0-2,4:all:1,3"
1007 @type cpu_mask: string
1008 @param process_id: process ID of KVM process. Used to pin entire VM
1010 @type process_id: int
1011 @param thread_dict: map of virtual CPUs to KVM thread IDs
1012 @type thread_dict: dict int:int
1015 # Convert the string CPU mask to a list of list of int's
1016 cpu_list = utils.ParseMultiCpuMask(cpu_mask)
1018 if len(cpu_list) == 1:
1019 all_cpu_mapping = cpu_list[0]
1020 if all_cpu_mapping == constants.CPU_PINNING_OFF:
1021 # If CPU pinning has 1 entry that's "all", then do nothing
1024 # If CPU pinning has one non-all entry, map the entire VM to
1025 # one set of physical CPUs
1026 cls._VerifyAffinityPackage()
1027 affinity.set_process_affinity_mask(
1028 process_id, cls._BuildAffinityCpuMask(all_cpu_mapping))
1030 # The number of vCPUs mapped should match the number of vCPUs
1031 # reported by KVM. This was already verified earlier, so
1032 # here only as a sanity check.
1033 assert len(thread_dict) == len(cpu_list)
1034 cls._VerifyAffinityPackage()
1036 # For each vCPU, map it to the proper list of physical CPUs
1037 for vcpu, i in zip(cpu_list, range(len(cpu_list))):
1038 affinity.set_process_affinity_mask(thread_dict[i],
1039 cls._BuildAffinityCpuMask(vcpu))
1041 def _GetVcpuThreadIds(self, instance_name):
1042 """Get a mapping of vCPU no. to thread IDs for the instance
1044 @type instance_name: string
1045 @param instance_name: instance in question
1046 @rtype: dictionary of int:int
1047 @return: a dictionary mapping vCPU numbers to thread IDs
1051 output = self._CallMonitorCommand(instance_name, self._CPU_INFO_CMD)
1052 for line in output.stdout.splitlines():
1053 match = self._CPU_INFO_RE.search(line)
1056 grp = map(int, match.groups())
1057 result[grp[0]] = grp[1]
1061 def _ExecuteCpuAffinity(self, instance_name, cpu_mask):
1062 """Complete CPU pinning.
1064 @type instance_name: string
1065 @param instance_name: name of instance
1066 @type cpu_mask: string
1067 @param cpu_mask: CPU pinning mask as entered by user
1070 # Get KVM process ID, to be used if need to pin entire VM
1071 _, pid, _ = self._InstancePidAlive(instance_name)
1072 # Get vCPU thread IDs, to be used if need to pin vCPUs separately
1073 thread_dict = self._GetVcpuThreadIds(instance_name)
1074 # Run CPU pinning, based on configured mask
1075 self._AssignCpuAffinity(cpu_mask, pid, thread_dict)
1077 def ListInstances(self):
1078 """Get the list of running instances.
1080 We can do this by listing our live instances directory and
1081 checking whether the associated kvm process is still alive.
1085 for name in os.listdir(self._PIDS_DIR):
1086 if self._InstancePidAlive(name)[2]:
1090 def GetInstanceInfo(self, instance_name):
1091 """Get instance properties.
1093 @type instance_name: string
1094 @param instance_name: the instance name
1095 @rtype: tuple of strings
1096 @return: (name, id, memory, vcpus, stat, times)
1099 _, pid, alive = self._InstancePidAlive(instance_name)
1103 _, memory, vcpus = self._InstancePidInfo(pid)
1108 qmp = QmpConnection(self._InstanceQmpMonitor(instance_name))
1110 vcpus = len(qmp.Execute("query-cpus")[qmp.RETURN_KEY])
1111 # Will fail if ballooning is not enabled, but we can then just resort to
1113 mem_bytes = qmp.Execute("query-balloon")[qmp.RETURN_KEY][qmp.ACTUAL_KEY]
1114 memory = mem_bytes / 1048576
1115 except errors.HypervisorError:
1118 return (instance_name, pid, memory, vcpus, istat, times)
1120 def GetAllInstancesInfo(self):
1121 """Get properties of all instances.
1123 @return: list of tuples (name, id, memory, vcpus, stat, times)
1127 for name in os.listdir(self._PIDS_DIR):
1129 info = self.GetInstanceInfo(name)
1130 except errors.HypervisorError:
1131 # Ignore exceptions due to instances being shut down
1137 def _GenerateKVMBlockDevicesOptions(self, instance, block_devices, kvmhelp):
1139 hvp = instance.hvparams
1140 boot_disk = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
1141 kvm_path = hvp[constants.HV_KVM_PATH]
1143 # whether this is an older KVM version that uses the boot=on flag
1145 needs_boot_flag = self._BOOT_RE.search(kvmhelp)
1148 device_driver = None
1149 disk_type = hvp[constants.HV_DISK_TYPE]
1150 if disk_type == constants.HT_DISK_PARAVIRTUAL:
1151 if_val = ",if=%s" % self._VIRTIO
1153 devlist = self._GetKVMOutput(kvm_path, self._KVMOPT_DEVICELIST)
1154 if self._VIRTIO_BLK_RE.search(devlist):
1156 # will be passed in -device option as driver
1157 device_driver = self._VIRTIO_BLK_PCI
1158 except errors.HypervisorError, _:
1161 if_val = ",if=%s" % disk_type
1163 disk_cache = hvp[constants.HV_DISK_CACHE]
1164 if instance.disk_template in constants.DTS_EXT_MIRROR:
1165 if disk_cache != "none":
1166 # TODO: make this a hard error, instead of a silent overwrite
1167 logging.warning("KVM: overriding disk_cache setting '%s' with 'none'"
1168 " to prevent shared storage corruption on migration",
1170 cache_val = ",cache=none"
1171 elif disk_cache != constants.HT_CACHE_DEFAULT:
1172 cache_val = ",cache=%s" % disk_cache
1175 for cfdev, dev_path in block_devices:
1176 if cfdev.mode != constants.DISK_RDWR:
1177 raise errors.HypervisorError("Instance has read-only disks which"
1178 " are not supported by KVM")
1179 # TODO: handle FD_LOOP and FD_BLKTAP (?)
1182 dev_opts.extend(["-boot", "c"])
1184 if needs_boot_flag and disk_type != constants.HT_DISK_IDE:
1185 boot_val = ",boot=on"
1186 drive_val = "file=%s,format=raw%s%s%s" % \
1187 (dev_path, if_val, boot_val, cache_val)
1190 # block_devices are the 4th entry of runtime file that did not exist in
1191 # the past. That means that cfdev should always have pci slot and
1192 # _GenerateDeviceKVMId() will not raise a exception.
1193 kvm_devid = _GenerateDeviceKVMId(constants.HOTPLUG_TARGET_DISK, cfdev)
1194 drive_val += (",id=%s" % kvm_devid)
1195 drive_val += (",bus=0,unit=%d" % cfdev.pci)
1196 dev_val = ("%s,drive=%s,id=%s" %
1197 (device_driver, kvm_devid, kvm_devid))
1198 dev_val += ",bus=pci.0,addr=%s" % hex(cfdev.pci)
1199 dev_opts.extend(["-device", dev_val])
1201 dev_opts.extend(["-drive", drive_val])
1205 def _GenerateKVMRuntime(self, instance, block_devices, startup_paused,
1207 """Generate KVM information to start an instance.
1209 @type kvmhelp: string
1210 @param kvmhelp: output of kvm --help
1211 @attention: this function must not have any side-effects; for
1212 example, it must not write to the filesystem, or read values
1213 from the current system the are expected to differ between
1214 nodes, since it is only run once at instance startup;
1215 actions/kvm arguments that can vary between systems should be
1216 done in L{_ExecuteKVMRuntime}
1219 # pylint: disable=R0912,R0914,R0915
1220 hvp = instance.hvparams
1221 self.ValidateParameters(hvp)
1223 pidfile = self._InstancePidFile(instance.name)
1224 kvm = hvp[constants.HV_KVM_PATH]
1226 # used just by the vnc server, if enabled
1227 kvm_cmd.extend(["-name", instance.name])
1228 kvm_cmd.extend(["-m", instance.beparams[constants.BE_MAXMEM]])
1230 smp_list = ["%s" % instance.beparams[constants.BE_VCPUS]]
1231 if hvp[constants.HV_CPU_CORES]:
1232 smp_list.append("cores=%s" % hvp[constants.HV_CPU_CORES])
1233 if hvp[constants.HV_CPU_THREADS]:
1234 smp_list.append("threads=%s" % hvp[constants.HV_CPU_THREADS])
1235 if hvp[constants.HV_CPU_SOCKETS]:
1236 smp_list.append("sockets=%s" % hvp[constants.HV_CPU_SOCKETS])
1238 kvm_cmd.extend(["-smp", ",".join(smp_list)])
1240 kvm_cmd.extend(["-pidfile", pidfile])
1241 kvm_cmd.extend(["-balloon", "virtio"])
1242 kvm_cmd.extend(["-daemonize"])
1243 if not instance.hvparams[constants.HV_ACPI]:
1244 kvm_cmd.extend(["-no-acpi"])
1245 if instance.hvparams[constants.HV_REBOOT_BEHAVIOR] == \
1246 constants.INSTANCE_REBOOT_EXIT:
1247 kvm_cmd.extend(["-no-reboot"])
1249 mversion = hvp[constants.HV_KVM_MACHINE_VERSION]
1251 mversion = self._GetDefaultMachineVersion(kvm)
1252 if self._MACHINE_RE.search(kvmhelp):
1253 # TODO (2.8): kernel_irqchip and kvm_shadow_mem machine properties, as
1254 # extra hypervisor parameters. We should also investigate whether and how
1255 # shadow_mem should be considered for the resource model.
1256 if (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED):
1257 specprop = ",accel=kvm"
1260 machinespec = "%s%s" % (mversion, specprop)
1261 kvm_cmd.extend(["-machine", machinespec])
1263 kvm_cmd.extend(["-M", mversion])
1264 if (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED and
1265 self._ENABLE_KVM_RE.search(kvmhelp)):
1266 kvm_cmd.extend(["-enable-kvm"])
1267 elif (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_DISABLED and
1268 self._DISABLE_KVM_RE.search(kvmhelp)):
1269 kvm_cmd.extend(["-disable-kvm"])
1271 kernel_path = hvp[constants.HV_KERNEL_PATH]
1273 boot_cdrom = boot_floppy = boot_network = False
1275 boot_cdrom = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_CDROM
1276 boot_floppy = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_FLOPPY
1277 boot_network = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_NETWORK
1280 kvm_cmd.extend([_KVM_START_PAUSED_FLAG])
1283 kvm_cmd.extend(["-boot", "n"])
1285 # whether this is an older KVM version that uses the boot=on flag
1287 needs_boot_flag = self._BOOT_RE.search(kvmhelp)
1289 disk_type = hvp[constants.HV_DISK_TYPE]
1291 #Now we can specify a different device type for CDROM devices.
1292 cdrom_disk_type = hvp[constants.HV_KVM_CDROM_DISK_TYPE]
1293 if not cdrom_disk_type:
1294 cdrom_disk_type = disk_type
1296 iso_image = hvp[constants.HV_CDROM_IMAGE_PATH]
1298 options = ",format=raw,media=cdrom"
1299 # set cdrom 'if' type
1301 actual_cdrom_type = constants.HT_DISK_IDE
1302 elif cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
1303 actual_cdrom_type = "virtio"
1305 actual_cdrom_type = cdrom_disk_type
1306 if_val = ",if=%s" % actual_cdrom_type
1307 # set boot flag, if needed
1310 kvm_cmd.extend(["-boot", "d"])
1312 boot_val = ",boot=on"
1313 # and finally build the entire '-drive' value
1314 drive_val = "file=%s%s%s%s" % (iso_image, options, if_val, boot_val)
1315 kvm_cmd.extend(["-drive", drive_val])
1317 iso_image2 = hvp[constants.HV_KVM_CDROM2_IMAGE_PATH]
1319 options = ",format=raw,media=cdrom"
1320 if cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
1321 if_val = ",if=virtio"
1323 if_val = ",if=%s" % cdrom_disk_type
1324 drive_val = "file=%s%s%s" % (iso_image2, options, if_val)
1325 kvm_cmd.extend(["-drive", drive_val])
1327 floppy_image = hvp[constants.HV_KVM_FLOPPY_IMAGE_PATH]
1329 options = ",format=raw,media=disk"
1331 kvm_cmd.extend(["-boot", "a"])
1332 options = "%s,boot=on" % options
1333 if_val = ",if=floppy"
1334 options = "%s%s" % (options, if_val)
1335 drive_val = "file=%s%s" % (floppy_image, options)
1336 kvm_cmd.extend(["-drive", drive_val])
1339 kvm_cmd.extend(["-kernel", kernel_path])
1340 initrd_path = hvp[constants.HV_INITRD_PATH]
1342 kvm_cmd.extend(["-initrd", initrd_path])
1343 root_append = ["root=%s" % hvp[constants.HV_ROOT_PATH],
1344 hvp[constants.HV_KERNEL_ARGS]]
1345 if hvp[constants.HV_SERIAL_CONSOLE]:
1346 serial_speed = hvp[constants.HV_SERIAL_SPEED]
1347 root_append.append("console=ttyS0,%s" % serial_speed)
1348 kvm_cmd.extend(["-append", " ".join(root_append)])
1350 mem_path = hvp[constants.HV_MEM_PATH]
1352 kvm_cmd.extend(["-mem-path", mem_path, "-mem-prealloc"])
1354 monitor_dev = ("unix:%s,server,nowait" %
1355 self._InstanceMonitor(instance.name))
1356 kvm_cmd.extend(["-monitor", monitor_dev])
1357 if hvp[constants.HV_SERIAL_CONSOLE]:
1358 serial_dev = ("unix:%s,server,nowait" %
1359 self._InstanceSerial(instance.name))
1360 kvm_cmd.extend(["-serial", serial_dev])
1362 kvm_cmd.extend(["-serial", "none"])
1364 mouse_type = hvp[constants.HV_USB_MOUSE]
1365 vnc_bind_address = hvp[constants.HV_VNC_BIND_ADDRESS]
1366 spice_bind = hvp[constants.HV_KVM_SPICE_BIND]
1367 spice_ip_version = None
1369 kvm_cmd.extend(["-usb"])
1372 kvm_cmd.extend(["-usbdevice", mouse_type])
1373 elif vnc_bind_address:
1374 kvm_cmd.extend(["-usbdevice", constants.HT_MOUSE_TABLET])
1376 if vnc_bind_address:
1377 if netutils.IP4Address.IsValid(vnc_bind_address):
1378 if instance.network_port > constants.VNC_BASE_PORT:
1379 display = instance.network_port - constants.VNC_BASE_PORT
1380 if vnc_bind_address == constants.IP4_ADDRESS_ANY:
1381 vnc_arg = ":%d" % (display)
1383 vnc_arg = "%s:%d" % (vnc_bind_address, display)
1385 logging.error("Network port is not a valid VNC display (%d < %d),"
1386 " not starting VNC",
1387 instance.network_port, constants.VNC_BASE_PORT)
1390 # Only allow tls and other option when not binding to a file, for now.
1391 # kvm/qemu gets confused otherwise about the filename to use.
1393 if hvp[constants.HV_VNC_TLS]:
1394 vnc_append = "%s,tls" % vnc_append
1395 if hvp[constants.HV_VNC_X509_VERIFY]:
1396 vnc_append = "%s,x509verify=%s" % (vnc_append,
1397 hvp[constants.HV_VNC_X509])
1398 elif hvp[constants.HV_VNC_X509]:
1399 vnc_append = "%s,x509=%s" % (vnc_append,
1400 hvp[constants.HV_VNC_X509])
1401 if hvp[constants.HV_VNC_PASSWORD_FILE]:
1402 vnc_append = "%s,password" % vnc_append
1404 vnc_arg = "%s%s" % (vnc_arg, vnc_append)
1407 vnc_arg = "unix:%s/%s.vnc" % (vnc_bind_address, instance.name)
1409 kvm_cmd.extend(["-vnc", vnc_arg])
1411 # FIXME: this is wrong here; the iface ip address differs
1412 # between systems, so it should be done in _ExecuteKVMRuntime
1413 if netutils.IsValidInterface(spice_bind):
1414 # The user specified a network interface, we have to figure out the IP
1416 addresses = netutils.GetInterfaceIpAddresses(spice_bind)
1417 spice_ip_version = hvp[constants.HV_KVM_SPICE_IP_VERSION]
1419 # if the user specified an IP version and the interface does not
1420 # have that kind of IP addresses, throw an exception
1421 if spice_ip_version != constants.IFACE_NO_IP_VERSION_SPECIFIED:
1422 if not addresses[spice_ip_version]:
1423 raise errors.HypervisorError("SPICE: Unable to get an IPv%s address"
1424 " for %s" % (spice_ip_version,
1427 # the user did not specify an IP version, we have to figure it out
1428 elif (addresses[constants.IP4_VERSION] and
1429 addresses[constants.IP6_VERSION]):
1430 # we have both ipv4 and ipv6, let's use the cluster default IP
1432 cluster_family = ssconf.SimpleStore().GetPrimaryIPFamily()
1433 spice_ip_version = \
1434 netutils.IPAddress.GetVersionFromAddressFamily(cluster_family)
1435 elif addresses[constants.IP4_VERSION]:
1436 spice_ip_version = constants.IP4_VERSION
1437 elif addresses[constants.IP6_VERSION]:
1438 spice_ip_version = constants.IP6_VERSION
1440 raise errors.HypervisorError("SPICE: Unable to get an IP address"
1441 " for %s" % (spice_bind))
1443 spice_address = addresses[spice_ip_version][0]
1446 # spice_bind is known to be a valid IP address, because
1447 # ValidateParameters checked it.
1448 spice_address = spice_bind
1450 spice_arg = "addr=%s" % spice_address
1451 if hvp[constants.HV_KVM_SPICE_USE_TLS]:
1452 spice_arg = ("%s,tls-port=%s,x509-cacert-file=%s" %
1453 (spice_arg, instance.network_port,
1454 pathutils.SPICE_CACERT_FILE))
1455 spice_arg = ("%s,x509-key-file=%s,x509-cert-file=%s" %
1456 (spice_arg, pathutils.SPICE_CERT_FILE,
1457 pathutils.SPICE_CERT_FILE))
1458 tls_ciphers = hvp[constants.HV_KVM_SPICE_TLS_CIPHERS]
1460 spice_arg = "%s,tls-ciphers=%s" % (spice_arg, tls_ciphers)
1462 spice_arg = "%s,port=%s" % (spice_arg, instance.network_port)
1464 if not hvp[constants.HV_KVM_SPICE_PASSWORD_FILE]:
1465 spice_arg = "%s,disable-ticketing" % spice_arg
1467 if spice_ip_version:
1468 spice_arg = "%s,ipv%s" % (spice_arg, spice_ip_version)
1470 # Image compression options
1471 img_lossless = hvp[constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR]
1472 img_jpeg = hvp[constants.HV_KVM_SPICE_JPEG_IMG_COMPR]
1473 img_zlib_glz = hvp[constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR]
1475 spice_arg = "%s,image-compression=%s" % (spice_arg, img_lossless)
1477 spice_arg = "%s,jpeg-wan-compression=%s" % (spice_arg, img_jpeg)
1479 spice_arg = "%s,zlib-glz-wan-compression=%s" % (spice_arg, img_zlib_glz)
1481 # Video stream detection
1482 video_streaming = hvp[constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION]
1484 spice_arg = "%s,streaming-video=%s" % (spice_arg, video_streaming)
1486 # Audio compression, by default in qemu-kvm it is on
1487 if not hvp[constants.HV_KVM_SPICE_AUDIO_COMPR]:
1488 spice_arg = "%s,playback-compression=off" % spice_arg
1489 if not hvp[constants.HV_KVM_SPICE_USE_VDAGENT]:
1490 spice_arg = "%s,agent-mouse=off" % spice_arg
1492 # Enable the spice agent communication channel between the host and the
1494 kvm_cmd.extend(["-device", "virtio-serial-pci"])
1497 "virtserialport,chardev=spicechannel0,name=com.redhat.spice.0",
1499 kvm_cmd.extend(["-chardev", "spicevmc,id=spicechannel0,name=vdagent"])
1501 logging.info("KVM: SPICE will listen on port %s", instance.network_port)
1502 kvm_cmd.extend(["-spice", spice_arg])
1505 # From qemu 1.4 -nographic is incompatible with -daemonize. The new way
1506 # also works in earlier versions though (tested with 1.1 and 1.3)
1507 if self._DISPLAY_RE.search(kvmhelp):
1508 kvm_cmd.extend(["-display", "none"])
1510 kvm_cmd.extend(["-nographic"])
1512 if hvp[constants.HV_USE_LOCALTIME]:
1513 kvm_cmd.extend(["-localtime"])
1515 if hvp[constants.HV_KVM_USE_CHROOT]:
1516 kvm_cmd.extend(["-chroot", self._InstanceChrootDir(instance.name)])
1518 # Add qemu-KVM -cpu param
1519 if hvp[constants.HV_CPU_TYPE]:
1520 kvm_cmd.extend(["-cpu", hvp[constants.HV_CPU_TYPE]])
1522 # As requested by music lovers
1523 if hvp[constants.HV_SOUNDHW]:
1524 kvm_cmd.extend(["-soundhw", hvp[constants.HV_SOUNDHW]])
1526 # Pass a -vga option if requested, or if spice is used, for backwards
1528 if hvp[constants.HV_VGA]:
1529 kvm_cmd.extend(["-vga", hvp[constants.HV_VGA]])
1531 kvm_cmd.extend(["-vga", "qxl"])
1533 # Various types of usb devices, comma separated
1534 if hvp[constants.HV_USB_DEVICES]:
1535 for dev in hvp[constants.HV_USB_DEVICES].split(","):
1536 kvm_cmd.extend(["-usbdevice", dev])
1538 if hvp[constants.HV_KVM_EXTRA]:
1539 kvm_cmd.extend(hvp[constants.HV_KVM_EXTRA].split(" "))
1541 pci_reservations = bitarray(self._DEFAULT_PCI_RESERVATIONS)
1543 for disk, dev_path in block_devices:
1544 _UpdatePCISlots(disk, pci_reservations)
1545 kvm_disks.append((disk, dev_path))
1548 for nic in instance.nics:
1549 _UpdatePCISlots(nic, pci_reservations)
1550 kvm_nics.append(nic)
1554 return (kvm_cmd, kvm_nics, hvparams, kvm_disks)
1556 def _WriteKVMRuntime(self, instance_name, data):
1557 """Write an instance's KVM runtime
1561 utils.WriteFile(self._InstanceKVMRuntime(instance_name),
1563 except EnvironmentError, err:
1564 raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err)
1566 def _ReadKVMRuntime(self, instance_name):
1567 """Read an instance's KVM runtime
1571 file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name))
1572 except EnvironmentError, err:
1573 raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err)
1576 def _SaveKVMRuntime(self, instance, kvm_runtime):
1577 """Save an instance's KVM runtime
1580 kvm_cmd, kvm_nics, hvparams, block_devices = kvm_runtime
1582 serialized_nics = [nic.ToDict() for nic in kvm_nics]
1583 serialized_blockdevs = [(blk.ToDict(), link) for blk, link in block_devices]
1584 serialized_form = serializer.Dump((kvm_cmd, serialized_nics, hvparams,
1585 serialized_blockdevs))
1587 self._WriteKVMRuntime(instance.name, serialized_form)
1589 def _LoadKVMRuntime(self, instance, serialized_runtime=None):
1590 """Load an instance's KVM runtime
1593 if not serialized_runtime:
1594 serialized_runtime = self._ReadKVMRuntime(instance.name)
1596 loaded_runtime = serializer.Load(serialized_runtime)
1597 if len(loaded_runtime) == 3:
1598 serialized_blockdevs = []
1599 kvm_cmd, serialized_nics, hvparams = loaded_runtime
1601 kvm_cmd, serialized_nics, hvparams, serialized_blockdevs = loaded_runtime
1603 kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics]
1604 block_devices = [(objects.Disk.FromDict(sdisk), link)
1605 for sdisk, link in serialized_blockdevs]
1607 return (kvm_cmd, kvm_nics, hvparams, block_devices)
1609 def _RunKVMCmd(self, name, kvm_cmd, tap_fds=None):
1610 """Run the KVM cmd and check for errors
1613 @param name: instance name
1614 @type kvm_cmd: list of strings
1615 @param kvm_cmd: runcmd input for kvm
1616 @type tap_fds: list of int
1617 @param tap_fds: fds of tap devices opened by Ganeti
1621 result = utils.RunCmd(kvm_cmd, noclose_fds=tap_fds)
1624 utils_wrapper.CloseFdNoError(fd)
1627 raise errors.HypervisorError("Failed to start instance %s: %s (%s)" %
1628 (name, result.fail_reason, result.output))
1629 if not self._InstancePidAlive(name)[2]:
1630 raise errors.HypervisorError("Failed to start instance %s" % name)
1632 # 52/50 local variables
1633 # pylint: disable=R0914
1634 def _ExecuteKVMRuntime(self, instance, kvm_runtime, kvmhelp, incoming=None):
1635 """Execute a KVM cmd, after completing it with some last minute data.
1637 @type incoming: tuple of strings
1638 @param incoming: (target_host_ip, port)
1639 @type kvmhelp: string
1640 @param kvmhelp: output of kvm --help
1643 # Small _ExecuteKVMRuntime hv parameters programming howto:
1644 # - conf_hvp contains the parameters as configured on ganeti. they might
1645 # have changed since the instance started; only use them if the change
1646 # won't affect the inside of the instance (which hasn't been rebooted).
1647 # - up_hvp contains the parameters as they were when the instance was
1648 # started, plus any new parameter which has been added between ganeti
1649 # versions: it is paramount that those default to a value which won't
1650 # affect the inside of the instance as well.
1651 conf_hvp = instance.hvparams
1652 name = instance.name
1653 self._CheckDown(name)
1657 kvm_cmd, kvm_nics, up_hvp, block_devices = kvm_runtime
1658 # the first element of kvm_cmd is always the path to the kvm binary
1659 kvm_path = kvm_cmd[0]
1660 up_hvp = objects.FillDict(conf_hvp, up_hvp)
1662 # We know it's safe to run as a different user upon migration, so we'll use
1663 # the latest conf, from conf_hvp.
1664 security_model = conf_hvp[constants.HV_SECURITY_MODEL]
1665 if security_model == constants.HT_SM_USER:
1666 kvm_cmd.extend(["-runas", conf_hvp[constants.HV_SECURITY_DOMAIN]])
1668 keymap = conf_hvp[constants.HV_KEYMAP]
1670 keymap_path = self._InstanceKeymapFile(name)
1671 # If a keymap file is specified, KVM won't use its internal defaults. By
1672 # first including the "en-us" layout, an error on loading the actual
1673 # layout (e.g. because it can't be found) won't lead to a non-functional
1674 # keyboard. A keyboard with incorrect keys is still better than none.
1675 utils.WriteFile(keymap_path, data="include en-us\ninclude %s\n" % keymap)
1676 kvm_cmd.extend(["-k", keymap_path])
1678 # We have reasons to believe changing something like the nic driver/type
1679 # upon migration won't exactly fly with the instance kernel, so for nic
1680 # related parameters we'll use up_hvp
1684 kvm_cmd.extend(["-net", "none"])
1688 nic_type = up_hvp[constants.HV_NIC_TYPE]
1689 if nic_type == constants.HT_NIC_PARAVIRTUAL:
1690 nic_model = self._VIRTIO
1692 devlist = self._GetKVMOutput(kvm_path, self._KVMOPT_DEVICELIST)
1693 if self._VIRTIO_NET_RE.search(devlist):
1694 nic_model = self._VIRTIO_NET_PCI
1695 vnet_hdr = up_hvp[constants.HV_VNET_HDR]
1696 except errors.HypervisorError, _:
1697 # Older versions of kvm don't support DEVICE_LIST, but they don't
1698 # have new virtio syntax either.
1701 if up_hvp[constants.HV_VHOST_NET]:
1702 # check for vhost_net support
1703 if self._VHOST_RE.search(kvmhelp):
1704 tap_extra = ",vhost=on"
1706 raise errors.HypervisorError("vhost_net is configured"
1707 " but it is not available")
1709 nic_model = nic_type
1711 kvm_supports_netdev = self._NETDEV_RE.search(kvmhelp)
1713 for nic_seq, nic in enumerate(kvm_nics):
1714 tapname, tapfd = _OpenTap(vnet_hdr=vnet_hdr)
1715 tapfds.append(tapfd)
1716 taps.append(tapname)
1717 if kvm_supports_netdev:
1718 nic_val = "%s,mac=%s" % (nic_model, nic.mac)
1720 # kvm_nics already exist in old runtime files and thus there might
1721 # be some entries without pci slot (therefore try: except:)
1722 kvm_devid = _GenerateDeviceKVMId(constants.HOTPLUG_TARGET_NIC, nic)
1724 nic_val += (",id=%s,bus=pci.0,addr=%s" % (kvm_devid, hex(nic.pci)))
1725 except errors.HotplugError:
1726 netdev = "netdev%d" % nic_seq
1727 nic_val += (",netdev=%s" % netdev)
1728 tap_val = ("type=tap,id=%s,fd=%d%s" %
1729 (netdev, tapfd, tap_extra))
1730 kvm_cmd.extend(["-netdev", tap_val, "-device", nic_val])
1732 nic_val = "nic,vlan=%s,macaddr=%s,model=%s" % (nic_seq,
1734 tap_val = "tap,vlan=%s,fd=%d" % (nic_seq, tapfd)
1735 kvm_cmd.extend(["-net", tap_val, "-net", nic_val])
1738 target, port = incoming
1739 kvm_cmd.extend(["-incoming", "tcp:%s:%s" % (target, port)])
1741 # Changing the vnc password doesn't bother the guest that much. At most it
1742 # will surprise people who connect to it. Whether positively or negatively
1744 vnc_pwd_file = conf_hvp[constants.HV_VNC_PASSWORD_FILE]
1748 vnc_pwd = utils.ReadOneLineFile(vnc_pwd_file, strict=True)
1749 except EnvironmentError, err:
1750 raise errors.HypervisorError("Failed to open VNC password file %s: %s"
1751 % (vnc_pwd_file, err))
1753 if conf_hvp[constants.HV_KVM_USE_CHROOT]:
1754 utils.EnsureDirs([(self._InstanceChrootDir(name),
1755 constants.SECURE_DIR_MODE)])
1757 # Automatically enable QMP if version is >= 0.14
1758 if self._QMP_RE.search(kvmhelp):
1759 logging.debug("Enabling QMP")
1760 kvm_cmd.extend(["-qmp", "unix:%s,server,nowait" %
1761 self._InstanceQmpMonitor(instance.name)])
1763 # Configure the network now for starting instances and bridged interfaces,
1764 # during FinalizeMigration for incoming instances' routed interfaces
1765 for nic_seq, nic in enumerate(kvm_nics):
1767 nic.nicparams[constants.NIC_MODE] != constants.NIC_MODE_BRIDGED):
1769 self._ConfigureNIC(instance, nic_seq, nic, taps[nic_seq])
1771 bdev_opts = self._GenerateKVMBlockDevicesOptions(instance,
1774 kvm_cmd.extend(bdev_opts)
1775 # CPU affinity requires kvm to start paused, so we set this flag if the
1776 # instance is not already paused and if we are not going to accept a
1777 # migrating instance. In the latter case, pausing is not needed.
1778 start_kvm_paused = not (_KVM_START_PAUSED_FLAG in kvm_cmd) and not incoming
1779 if start_kvm_paused:
1780 kvm_cmd.extend([_KVM_START_PAUSED_FLAG])
1782 # Note: CPU pinning is using up_hvp since changes take effect
1783 # during instance startup anyway, and to avoid problems when soft
1784 # rebooting the instance.
1786 if up_hvp.get(constants.HV_CPU_MASK, None):
1789 if security_model == constants.HT_SM_POOL:
1790 ss = ssconf.SimpleStore()
1791 uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\n")
1792 all_uids = set(uidpool.ExpandUidPool(uid_pool))
1793 uid = uidpool.RequestUnusedUid(all_uids)
1795 username = pwd.getpwuid(uid.GetUid()).pw_name
1796 kvm_cmd.extend(["-runas", username])
1797 self._RunKVMCmd(name, kvm_cmd, tapfds)
1799 uidpool.ReleaseUid(uid)
1803 utils.WriteFile(self._InstanceUidFile(name), data=uid.AsStr())
1805 self._RunKVMCmd(name, kvm_cmd, tapfds)
1807 utils.EnsureDirs([(self._InstanceNICDir(instance.name),
1808 constants.RUN_DIRS_MODE)])
1809 for nic_seq, tap in enumerate(taps):
1810 utils.WriteFile(self._InstanceNICFile(instance.name, nic_seq),
1814 change_cmd = "change vnc password %s" % vnc_pwd
1815 self._CallMonitorCommand(instance.name, change_cmd)
1817 # Setting SPICE password. We are not vulnerable to malicious passwordless
1818 # connection attempts because SPICE by default does not allow connections
1819 # if neither a password nor the "disable_ticketing" options are specified.
1820 # As soon as we send the password via QMP, that password is a valid ticket
1822 spice_password_file = conf_hvp[constants.HV_KVM_SPICE_PASSWORD_FILE]
1823 if spice_password_file:
1826 spice_pwd = utils.ReadOneLineFile(spice_password_file, strict=True)
1827 except EnvironmentError, err:
1828 raise errors.HypervisorError("Failed to open SPICE password file %s: %s"
1829 % (spice_password_file, err))
1831 qmp = QmpConnection(self._InstanceQmpMonitor(instance.name))
1834 "protocol": "spice",
1835 "password": spice_pwd,
1837 qmp.Execute("set_password", arguments)
1839 for filename in temp_files:
1840 utils.RemoveFile(filename)
1842 # If requested, set CPU affinity and resume instance execution
1844 self._ExecuteCpuAffinity(instance.name, up_hvp[constants.HV_CPU_MASK])
1846 start_memory = self._InstanceStartupMemory(instance)
1847 if start_memory < instance.beparams[constants.BE_MAXMEM]:
1848 self.BalloonInstanceMemory(instance, start_memory)
1850 if start_kvm_paused:
1851 # To control CPU pinning, ballooning, and vnc/spice passwords
1852 # the VM was started in a frozen state. If freezing was not
1853 # explicitly requested resume the vm status.
1854 self._CallMonitorCommand(instance.name, self._CONT_CMD)
1856 def StartInstance(self, instance, block_devices, startup_paused):
1857 """Start an instance.
1860 self._CheckDown(instance.name)
1861 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
1862 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
1863 kvm_runtime = self._GenerateKVMRuntime(instance, block_devices,
1864 startup_paused, kvmhelp)
1865 self._SaveKVMRuntime(instance, kvm_runtime)
1866 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp)
1868 def _CallMonitorCommand(self, instance_name, command):
1869 """Invoke a command on the instance monitor.
1872 # TODO: Replace monitor calls with QMP once KVM >= 0.14 is the minimum
1873 # version. The monitor protocol is designed for human consumption, whereas
1874 # QMP is made for programmatic usage. In the worst case QMP can also
1875 # execute monitor commands. As it is, all calls to socat take at least
1876 # 500ms and likely more: socat can't detect the end of the reply and waits
1877 # for 500ms of no data received before exiting (500 ms is the default for
1878 # the "-t" parameter).
1879 socat = ("echo %s | %s STDIO UNIX-CONNECT:%s" %
1880 (utils.ShellQuote(command),
1881 constants.SOCAT_PATH,
1882 utils.ShellQuote(self._InstanceMonitor(instance_name))))
1883 result = utils.RunCmd(socat)
1885 msg = ("Failed to send command '%s' to instance '%s', reason '%s',"
1887 (command, instance_name, result.fail_reason, result.output))
1888 raise errors.HypervisorError(msg)
1892 def _GetFreePCISlot(self, instance, dev):
1893 """Get the first available pci slot of a runnung instance.
1896 slots = bitarray(32)
1897 slots.setall(False) # pylint: disable=E1101
1898 output = self._CallMonitorCommand(instance.name, self._INFO_PCI_CMD)
1899 for line in output.stdout.splitlines():
1900 match = self._INFO_PCI_RE.search(line)
1902 slot = int(match.group(1))
1905 [free] = slots.search(_AVAILABLE_PCI_SLOT, 1) # pylint: disable=E1101
1907 raise errors.HypervisorError("All PCI slots occupied")
1912 def _ParseKVMVersion(cls, text):
1913 """Parse the KVM version from the --help output.
1916 @param text: output of kvm --help
1917 @return: (version, v_maj, v_min, v_rev)
1918 @raise errors.HypervisorError: when the KVM version cannot be retrieved
1921 match = cls._VERSION_RE.search(text.splitlines()[0])
1923 raise errors.HypervisorError("Unable to get KVM version")
1925 v_all = match.group(0)
1926 v_maj = int(match.group(1))
1927 v_min = int(match.group(2))
1929 v_rev = int(match.group(4))
1932 return (v_all, v_maj, v_min, v_rev)
1935 def _GetKVMOutput(cls, kvm_path, option):
1936 """Return the output of a kvm invocation
1938 @type kvm_path: string
1939 @param kvm_path: path to the kvm executable
1940 @type option: a key of _KVMOPTS_CMDS
1941 @param option: kvm option to fetch the output from
1942 @return: output a supported kvm invocation
1943 @raise errors.HypervisorError: when the KVM help output cannot be retrieved
1946 assert option in cls._KVMOPTS_CMDS, "Invalid output option"
1948 optlist, can_fail = cls._KVMOPTS_CMDS[option]
1950 result = utils.RunCmd([kvm_path] + optlist)
1951 if result.failed and not can_fail:
1952 raise errors.HypervisorError("Unable to get KVM %s output" %
1953 " ".join(cls._KVMOPTS_CMDS[option]))
1954 return result.output
1957 def _GetKVMVersion(cls, kvm_path):
1958 """Return the installed KVM version.
1960 @return: (version, v_maj, v_min, v_rev)
1961 @raise errors.HypervisorError: when the KVM version cannot be retrieved
1964 return cls._ParseKVMVersion(cls._GetKVMOutput(kvm_path, cls._KVMOPT_HELP))
1967 def _GetDefaultMachineVersion(cls, kvm_path):
1968 """Return the default hardware revision (e.g. pc-1.1)
1971 output = cls._GetKVMOutput(kvm_path, cls._KVMOPT_MLIST)
1972 match = cls._DEFAULT_MACHINE_VERSION_RE.search(output)
1974 return match.group(1)
1978 def StopInstance(self, instance, force=False, retry=False, name=None):
1979 """Stop an instance.
1982 if name is not None and not force:
1983 raise errors.HypervisorError("Cannot shutdown cleanly by name only")
1985 name = instance.name
1986 acpi = instance.hvparams[constants.HV_ACPI]
1989 _, pid, alive = self._InstancePidAlive(name)
1990 if pid > 0 and alive:
1991 if force or not acpi:
1992 utils.KillProcess(pid)
1994 self._CallMonitorCommand(name, "system_powerdown")
1996 def CleanupInstance(self, instance_name):
1997 """Cleanup after a stopped instance
2000 pidfile, pid, alive = self._InstancePidAlive(instance_name)
2001 if pid > 0 and alive:
2002 raise errors.HypervisorError("Cannot cleanup a live instance")
2003 self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
2005 def RebootInstance(self, instance):
2006 """Reboot an instance.
2009 # For some reason if we do a 'send-key ctrl-alt-delete' to the control
2010 # socket the instance will stop, but now power up again. So we'll resort
2011 # to shutdown and restart.
2012 _, _, alive = self._InstancePidAlive(instance.name)
2014 raise errors.HypervisorError("Failed to reboot instance %s:"
2015 " not running" % instance.name)
2016 # StopInstance will delete the saved KVM runtime so:
2017 # ...first load it...
2018 kvm_runtime = self._LoadKVMRuntime(instance)
2019 # ...now we can safely call StopInstance...
2020 if not self.StopInstance(instance):
2021 self.StopInstance(instance, force=True)
2022 # ...and finally we can save it again, and execute it...
2023 self._SaveKVMRuntime(instance, kvm_runtime)
2024 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
2025 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
2026 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp)
2028 def MigrationInfo(self, instance):
2029 """Get instance information to perform a migration.
2031 @type instance: L{objects.Instance}
2032 @param instance: instance to be migrated
2034 @return: content of the KVM runtime file
2037 return self._ReadKVMRuntime(instance.name)
2039 def AcceptInstance(self, instance, info, target):
2040 """Prepare to accept an instance.
2042 @type instance: L{objects.Instance}
2043 @param instance: instance to be accepted
2045 @param info: content of the KVM runtime file on the source node
2046 @type target: string
2047 @param target: target host (usually ip), on this node
2050 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
2051 incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT])
2052 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
2053 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
2054 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp,
2055 incoming=incoming_address)
2057 def FinalizeMigrationDst(self, instance, info, success):
2058 """Finalize the instance migration on the target node.
2060 Stop the incoming mode KVM.
2062 @type instance: L{objects.Instance}
2063 @param instance: instance whose migration is being finalized
2067 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
2068 kvm_nics = kvm_runtime[1]
2070 for nic_seq, nic in enumerate(kvm_nics):
2071 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2072 # Bridged interfaces have already been configured
2075 tap = utils.ReadFile(self._InstanceNICFile(instance.name, nic_seq))
2076 except EnvironmentError, err:
2077 logging.warning("Failed to find host interface for %s NIC #%d: %s",
2078 instance.name, nic_seq, str(err))
2081 self._ConfigureNIC(instance, nic_seq, nic, tap)
2082 except errors.HypervisorError, err:
2083 logging.warning(str(err))
2085 self._WriteKVMRuntime(instance.name, info)
2087 self.StopInstance(instance, force=True)
2089 def MigrateInstance(self, instance, target, live):
2090 """Migrate an instance to a target node.
2092 The migration will not be attempted if the instance is not
2095 @type instance: L{objects.Instance}
2096 @param instance: the instance to be migrated
2097 @type target: string
2098 @param target: ip address of the target node
2100 @param live: perform a live migration
2103 instance_name = instance.name
2104 port = instance.hvparams[constants.HV_MIGRATION_PORT]
2105 _, _, alive = self._InstancePidAlive(instance_name)
2107 raise errors.HypervisorError("Instance not running, cannot migrate")
2110 self._CallMonitorCommand(instance_name, "stop")
2112 migrate_command = ("migrate_set_speed %dm" %
2113 instance.hvparams[constants.HV_MIGRATION_BANDWIDTH])
2114 self._CallMonitorCommand(instance_name, migrate_command)
2116 migrate_command = ("migrate_set_downtime %dms" %
2117 instance.hvparams[constants.HV_MIGRATION_DOWNTIME])
2118 self._CallMonitorCommand(instance_name, migrate_command)
2120 migrate_command = "migrate -d tcp:%s:%s" % (target, port)
2121 self._CallMonitorCommand(instance_name, migrate_command)
2123 def FinalizeMigrationSource(self, instance, success, live):
2124 """Finalize the instance migration on the source node.
2126 @type instance: L{objects.Instance}
2127 @param instance: the instance that was migrated
2129 @param success: whether the migration succeeded or not
2131 @param live: whether the user requested a live migration or not
2135 pidfile, pid, _ = self._InstancePidAlive(instance.name)
2136 utils.KillProcess(pid)
2137 self._RemoveInstanceRuntimeFiles(pidfile, instance.name)
2139 self._CallMonitorCommand(instance.name, self._CONT_CMD)
2141 def GetMigrationStatus(self, instance):
2142 """Get the migration status
2144 @type instance: L{objects.Instance}
2145 @param instance: the instance that is being migrated
2146 @rtype: L{objects.MigrationStatus}
2147 @return: the status of the current migration (one of
2148 L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional
2149 progress info that can be retrieved from the hypervisor
2152 info_command = "info migrate"
2153 for _ in range(self._MIGRATION_INFO_MAX_BAD_ANSWERS):
2154 result = self._CallMonitorCommand(instance.name, info_command)
2155 match = self._MIGRATION_STATUS_RE.search(result.stdout)
2157 if not result.stdout:
2158 logging.info("KVM: empty 'info migrate' result")
2160 logging.warning("KVM: unknown 'info migrate' result: %s",
2163 status = match.group(1)
2164 if status in constants.HV_KVM_MIGRATION_VALID_STATUSES:
2165 migration_status = objects.MigrationStatus(status=status)
2166 match = self._MIGRATION_PROGRESS_RE.search(result.stdout)
2168 migration_status.transferred_ram = match.group("transferred")
2169 migration_status.total_ram = match.group("total")
2171 return migration_status
2173 logging.warning("KVM: unknown migration status '%s'", status)
2175 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
2177 return objects.MigrationStatus(status=constants.HV_MIGRATION_FAILED)
2179 def BalloonInstanceMemory(self, instance, mem):
2180 """Balloon an instance memory to a certain value.
2182 @type instance: L{objects.Instance}
2183 @param instance: instance to be accepted
2185 @param mem: actual memory size to use for instance runtime
2188 self._CallMonitorCommand(instance.name, "balloon %d" % mem)
2190 def GetNodeInfo(self):
2191 """Return information about the node.
2193 @return: a dict with the following keys (values in MiB):
2194 - memory_total: the total memory size on the node
2195 - memory_free: the available memory on the node for instances
2196 - memory_dom0: the memory used by the node itself, if available
2197 - hv_version: the hypervisor version in the form (major, minor,
2201 result = self.GetLinuxNodeInfo()
2202 # FIXME: this is the global kvm version, but the actual version can be
2203 # customized as an hv parameter. we should use the nodegroup's default kvm
2204 # path parameter here.
2205 _, v_major, v_min, v_rev = self._GetKVMVersion(constants.KVM_PATH)
2206 result[constants.HV_NODEINFO_KEY_VERSION] = (v_major, v_min, v_rev)
2210 def GetInstanceConsole(cls, instance, hvparams, beparams):
2211 """Return a command for connecting to the console of an instance.
2214 if hvparams[constants.HV_SERIAL_CONSOLE]:
2215 cmd = [pathutils.KVM_CONSOLE_WRAPPER,
2216 constants.SOCAT_PATH, utils.ShellQuote(instance.name),
2217 utils.ShellQuote(cls._InstanceMonitor(instance.name)),
2218 "STDIO,%s" % cls._SocatUnixConsoleParams(),
2219 "UNIX-CONNECT:%s" % cls._InstanceSerial(instance.name)]
2220 return objects.InstanceConsole(instance=instance.name,
2221 kind=constants.CONS_SSH,
2222 host=instance.primary_node,
2223 user=constants.SSH_CONSOLE_USER,
2226 vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS]
2227 if vnc_bind_address and instance.network_port > constants.VNC_BASE_PORT:
2228 display = instance.network_port - constants.VNC_BASE_PORT
2229 return objects.InstanceConsole(instance=instance.name,
2230 kind=constants.CONS_VNC,
2231 host=vnc_bind_address,
2232 port=instance.network_port,
2235 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2237 return objects.InstanceConsole(instance=instance.name,
2238 kind=constants.CONS_SPICE,
2240 port=instance.network_port)
2242 return objects.InstanceConsole(instance=instance.name,
2243 kind=constants.CONS_MESSAGE,
2244 message=("No serial shell for instance %s" %
2248 """Verify the hypervisor.
2250 Check that the required binaries exist.
2252 @return: Problem description if something is wrong, C{None} otherwise
2256 # FIXME: this is the global kvm binary, but the actual path can be
2257 # customized as an hv parameter; we should use the nodegroup's
2258 # default kvm path parameter here.
2259 if not os.path.exists(constants.KVM_PATH):
2260 msgs.append("The KVM binary ('%s') does not exist" % constants.KVM_PATH)
2261 if not os.path.exists(constants.SOCAT_PATH):
2262 msgs.append("The socat binary ('%s') does not exist" %
2263 constants.SOCAT_PATH)
2265 return self._FormatVerifyResults(msgs)
2268 def CheckParameterSyntax(cls, hvparams):
2269 """Check the given parameters for validity.
2271 @type hvparams: dict
2272 @param hvparams: dictionary with parameter names/value
2273 @raise errors.HypervisorError: when a parameter is not valid
2276 super(KVMHypervisor, cls).CheckParameterSyntax(hvparams)
2278 kernel_path = hvparams[constants.HV_KERNEL_PATH]
2280 if not hvparams[constants.HV_ROOT_PATH]:
2281 raise errors.HypervisorError("Need a root partition for the instance,"
2282 " if a kernel is defined")
2284 if (hvparams[constants.HV_VNC_X509_VERIFY] and
2285 not hvparams[constants.HV_VNC_X509]):
2286 raise errors.HypervisorError("%s must be defined, if %s is" %
2287 (constants.HV_VNC_X509,
2288 constants.HV_VNC_X509_VERIFY))
2290 if hvparams[constants.HV_SERIAL_CONSOLE]:
2291 serial_speed = hvparams[constants.HV_SERIAL_SPEED]
2292 valid_speeds = constants.VALID_SERIAL_SPEEDS
2293 if not serial_speed or serial_speed not in valid_speeds:
2294 raise errors.HypervisorError("Invalid serial console speed, must be"
2296 utils.CommaJoin(valid_speeds))
2298 boot_order = hvparams[constants.HV_BOOT_ORDER]
2299 if (boot_order == constants.HT_BO_CDROM and
2300 not hvparams[constants.HV_CDROM_IMAGE_PATH]):
2301 raise errors.HypervisorError("Cannot boot from cdrom without an"
2304 security_model = hvparams[constants.HV_SECURITY_MODEL]
2305 if security_model == constants.HT_SM_USER:
2306 if not hvparams[constants.HV_SECURITY_DOMAIN]:
2307 raise errors.HypervisorError("A security domain (user to run kvm as)"
2308 " must be specified")
2309 elif (security_model == constants.HT_SM_NONE or
2310 security_model == constants.HT_SM_POOL):
2311 if hvparams[constants.HV_SECURITY_DOMAIN]:
2312 raise errors.HypervisorError("Cannot have a security domain when the"
2313 " security model is 'none' or 'pool'")
2315 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2316 spice_ip_version = hvparams[constants.HV_KVM_SPICE_IP_VERSION]
2318 if spice_ip_version != constants.IFACE_NO_IP_VERSION_SPECIFIED:
2319 # if an IP version is specified, the spice_bind parameter must be an
2321 if (netutils.IP4Address.IsValid(spice_bind) and
2322 spice_ip_version != constants.IP4_VERSION):
2323 raise errors.HypervisorError("SPICE: Got an IPv4 address (%s), but"
2324 " the specified IP version is %s" %
2325 (spice_bind, spice_ip_version))
2327 if (netutils.IP6Address.IsValid(spice_bind) and
2328 spice_ip_version != constants.IP6_VERSION):
2329 raise errors.HypervisorError("SPICE: Got an IPv6 address (%s), but"
2330 " the specified IP version is %s" %
2331 (spice_bind, spice_ip_version))
2333 # All the other SPICE parameters depend on spice_bind being set. Raise an
2334 # error if any of them is set without it.
2335 for param in _SPICE_ADDITIONAL_PARAMS:
2337 raise errors.HypervisorError("SPICE: %s requires %s to be set" %
2338 (param, constants.HV_KVM_SPICE_BIND))
2341 def ValidateParameters(cls, hvparams):
2342 """Check the given parameters for validity.
2344 @type hvparams: dict
2345 @param hvparams: dictionary with parameter names/value
2346 @raise errors.HypervisorError: when a parameter is not valid
2349 super(KVMHypervisor, cls).ValidateParameters(hvparams)
2351 kvm_path = hvparams[constants.HV_KVM_PATH]
2353 security_model = hvparams[constants.HV_SECURITY_MODEL]
2354 if security_model == constants.HT_SM_USER:
2355 username = hvparams[constants.HV_SECURITY_DOMAIN]
2357 pwd.getpwnam(username)
2359 raise errors.HypervisorError("Unknown security domain user %s"
2362 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2364 # only one of VNC and SPICE can be used currently.
2365 if hvparams[constants.HV_VNC_BIND_ADDRESS]:
2366 raise errors.HypervisorError("Both SPICE and VNC are configured, but"
2367 " only one of them can be used at a"
2370 # check that KVM supports SPICE
2371 kvmhelp = cls._GetKVMOutput(kvm_path, cls._KVMOPT_HELP)
2372 if not cls._SPICE_RE.search(kvmhelp):
2373 raise errors.HypervisorError("SPICE is configured, but it is not"
2374 " supported according to 'kvm --help'")
2376 # if spice_bind is not an IP address, it must be a valid interface
2377 bound_to_addr = (netutils.IP4Address.IsValid(spice_bind) or
2378 netutils.IP6Address.IsValid(spice_bind))
2379 if not bound_to_addr and not netutils.IsValidInterface(spice_bind):
2380 raise errors.HypervisorError("SPICE: The %s parameter must be either"
2381 " a valid IP address or interface name" %
2382 constants.HV_KVM_SPICE_BIND)
2384 machine_version = hvparams[constants.HV_KVM_MACHINE_VERSION]
2386 output = cls._GetKVMOutput(kvm_path, cls._KVMOPT_MLIST)
2387 if not cls._CHECK_MACHINE_VERSION_RE(machine_version).search(output):
2388 raise errors.HypervisorError("Unsupported machine version: %s" %
2392 def PowercycleNode(cls):
2393 """KVM powercycle, just a wrapper over Linux powercycle.
2396 cls.LinuxPowercycle()