4 # Copyright (C) 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
40 from bitarray import bitarray
42 import affinity # pylint: disable=F0401
46 from ganeti import utils
47 from ganeti import constants
48 from ganeti import errors
49 from ganeti import serializer
50 from ganeti import objects
51 from ganeti import uidpool
52 from ganeti import ssconf
53 from ganeti import netutils
54 from ganeti import pathutils
55 from ganeti.hypervisor import hv_base
56 from ganeti.utils import wrapper as utils_wrapper
59 _KVM_NETWORK_SCRIPT = pathutils.CONF_DIR + "/kvm-vif-bridge"
60 _KVM_START_PAUSED_FLAG = "-S"
62 # TUN/TAP driver constants, taken from <linux/if_tun.h>
63 # They are architecture-independent and already hardcoded in qemu-kvm source,
64 # so we can safely include them here.
65 TUNSETIFF = 0x400454ca
66 TUNGETIFF = 0x800454d2
67 TUNGETFEATURES = 0x800454cf
72 #: SPICE parameters which depend on L{constants.HV_KVM_SPICE_BIND}
73 _SPICE_ADDITIONAL_PARAMS = frozenset([
74 constants.HV_KVM_SPICE_IP_VERSION,
75 constants.HV_KVM_SPICE_PASSWORD_FILE,
76 constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR,
77 constants.HV_KVM_SPICE_JPEG_IMG_COMPR,
78 constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR,
79 constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION,
80 constants.HV_KVM_SPICE_USE_TLS,
83 # Constant bitarray that reflects to a free pci slot
84 # Use it with bitarray.search()
85 _AVAILABLE_PCI_SLOT = bitarray("0")
87 # below constants show the format of runtime file
88 # the nics are in second possition, while the disks in 4th (last)
89 # moreover disk entries are stored in tupples of L{objects.Disk}, dev_path
90 _KVM_NICS_RUNTIME_INDEX = 1
91 _KVM_DISKS_RUNTIME_INDEX = 3
92 _DEVICE_RUNTIME_INDEX = {
93 constants.HOTPLUG_TARGET_DISK: _KVM_DISKS_RUNTIME_INDEX,
94 constants.HOTPLUG_TARGET_NIC: _KVM_NICS_RUNTIME_INDEX
96 _FIND_RUNTIME_ENTRY = {
97 constants.HOTPLUG_TARGET_NIC:
98 lambda nic, kvm_nics: [n for n in kvm_nics if n.uuid == nic.uuid],
99 constants.HOTPLUG_TARGET_DISK:
100 lambda disk, kvm_disks: [(d, l) for (d, l) in kvm_disks
101 if d.uuid == disk.uuid]
104 constants.HOTPLUG_TARGET_NIC: lambda d: d,
105 constants.HOTPLUG_TARGET_DISK: lambda (d, e): d
108 constants.HOTPLUG_TARGET_NIC: lambda d, e: d,
109 constants.HOTPLUG_TARGET_DISK: lambda d, e: (d, e)
113 def _GenerateDeviceKVMId(dev_type, dev):
114 """Helper function to generate a unique device name used by KVM
116 QEMU monitor commands use names to identify devices. Here we use their pci
117 slot and a part of their UUID to name them. dev.pci might be None for old
118 devices in the cluster.
120 @type dev_type: sting
121 @param dev_type: device type of param dev
122 @type dev: L{objects.Disk} or L{objects.NIC}
123 @param dev: the device object for which we generate a kvm name
124 @raise errors.HotplugError: in case a device has no pci slot (old devices)
129 raise errors.HotplugError("Hotplug is not supported for %s with UUID %s" %
130 (dev_type, dev.uuid))
132 return "%s-%s-pci-%d" % (dev_type.lower(), dev.uuid.split("-")[0], dev.pci)
135 def _UpdatePCISlots(dev, pci_reservations):
136 """Update pci configuration for a stopped instance
138 If dev has a pci slot then reserve it, else find first available
139 in pci_reservations bitarray. It acts on the same objects passed
140 as params so there is no need to return anything.
142 @type dev: L{objects.Disk} or L{objects.NIC}
143 @param dev: the device object for which we update its pci slot
144 @type pci_reservations: bitarray
145 @param pci_reservations: existing pci reservations for an instance
146 @raise errors.HotplugError: in case an instance has all its slot occupied
151 else: # pylint: disable=E1103
152 [free] = pci_reservations.search(_AVAILABLE_PCI_SLOT, 1)
154 raise errors.HypervisorError("All PCI slots occupied")
157 pci_reservations[free] = True
160 def _GetExistingDeviceInfo(dev_type, device, runtime):
161 """Helper function to get an existing device inside the runtime file
163 Used when an instance is running. Load kvm runtime file and search
164 for a device based on its type and uuid.
166 @type dev_type: sting
167 @param dev_type: device type of param dev
168 @type device: L{objects.Disk} or L{objects.NIC}
169 @param device: the device object for which we generate a kvm name
170 @type runtime: tuple (cmd, nics, hvparams, disks)
171 @param runtime: the runtime data to search for the device
172 @raise errors.HotplugError: in case the requested device does not
173 exist (e.g. device has been added without --hotplug option) or
174 device info has not pci slot (e.g. old devices in the cluster)
177 index = _DEVICE_RUNTIME_INDEX[dev_type]
178 found = _FIND_RUNTIME_ENTRY[dev_type](device, runtime[index])
180 raise errors.HotplugError("Cannot find runtime info for %s with UUID %s" %
181 (dev_type, device.uuid))
186 def _AnalyzeSerializedRuntime(serialized_runtime):
187 """Return runtime entries for a serialized runtime file
189 @type serialized_runtime: string
190 @param serialized_runtime: raw text data read from actual runtime file
191 @return: (cmd, nics, hvparams, bdevs)
195 loaded_runtime = serializer.Load(serialized_runtime)
196 if len(loaded_runtime) == 3:
197 serialized_blockdevs = []
198 kvm_cmd, serialized_nics, hvparams = loaded_runtime
200 kvm_cmd, serialized_nics, hvparams, serialized_blockdevs = loaded_runtime
202 kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics]
203 block_devices = [(objects.Disk.FromDict(sdisk), link)
204 for sdisk, link in serialized_blockdevs]
206 return (kvm_cmd, kvm_nics, hvparams, block_devices)
209 def _GetTunFeatures(fd, _ioctl=fcntl.ioctl):
210 """Retrieves supported TUN features from file descriptor.
212 @see: L{_ProbeTapVnetHdr}
215 req = struct.pack("I", 0)
217 buf = _ioctl(fd, TUNGETFEATURES, req)
218 except EnvironmentError, err:
219 logging.warning("ioctl(TUNGETFEATURES) failed: %s", err)
222 (flags, ) = struct.unpack("I", buf)
226 def _ProbeTapVnetHdr(fd, _features_fn=_GetTunFeatures):
227 """Check whether to enable the IFF_VNET_HDR flag.
229 To do this, _all_ of the following conditions must be met:
230 1. TUNGETFEATURES ioctl() *must* be implemented
231 2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag
232 3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in
233 drivers/net/tun.c there is no way to test this until after the tap device
234 has been created using TUNSETIFF, and there is no way to change the
235 IFF_VNET_HDR flag after creating the interface, catch-22! However both
236 TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27,
237 thus we can expect TUNGETIFF to be present if TUNGETFEATURES is.
240 @param fd: the file descriptor of /dev/net/tun
243 flags = _features_fn(fd)
249 result = bool(flags & IFF_VNET_HDR)
252 logging.warning("Kernel does not support IFF_VNET_HDR, not enabling")
257 def _OpenTap(vnet_hdr=True):
258 """Open a new tap device and return its file descriptor.
260 This is intended to be used by a qemu-type hypervisor together with the -net
261 tap,fd=<fd> command line parameter.
263 @type vnet_hdr: boolean
264 @param vnet_hdr: Enable the VNET Header
265 @return: (ifname, tapfd)
270 tapfd = os.open("/dev/net/tun", os.O_RDWR)
271 except EnvironmentError:
272 raise errors.HypervisorError("Failed to open /dev/net/tun")
274 flags = IFF_TAP | IFF_NO_PI
276 if vnet_hdr and _ProbeTapVnetHdr(tapfd):
277 flags |= IFF_VNET_HDR
279 # The struct ifreq ioctl request (see netdevice(7))
280 ifr = struct.pack("16sh", "", flags)
283 res = fcntl.ioctl(tapfd, TUNSETIFF, ifr)
284 except EnvironmentError, err:
285 raise errors.HypervisorError("Failed to allocate a new TAP device: %s" %
288 # Get the interface name from the ioctl
289 ifname = struct.unpack("16sh", res)[0].strip("\x00")
290 return (ifname, tapfd)
294 """QEMU Messaging Protocol (QMP) message.
297 def __init__(self, data):
298 """Creates a new QMP message based on the passed data.
301 if not isinstance(data, dict):
302 raise TypeError("QmpMessage must be initialized with a dict")
306 def __getitem__(self, field_name):
307 """Get the value of the required field if present, or None.
309 Overrides the [] operator to provide access to the message data,
310 returning None if the required item is not in the message
311 @return: the value of the field_name field, or None if field_name
312 is not contained in the message
315 return self.data.get(field_name, None)
317 def __setitem__(self, field_name, field_value):
318 """Set the value of the required field_name to field_value.
321 self.data[field_name] = field_value
324 def BuildFromJsonString(json_string):
325 """Build a QmpMessage from a JSON encoded string.
327 @type json_string: str
328 @param json_string: JSON string representing the message
329 @rtype: L{QmpMessage}
330 @return: a L{QmpMessage} built from json_string
334 data = serializer.LoadJson(json_string)
335 return QmpMessage(data)
338 # The protocol expects the JSON object to be sent as a single line.
339 return serializer.DumpJson(self.data)
341 def __eq__(self, other):
342 # When comparing two QmpMessages, we are interested in comparing
343 # their internal representation of the message data
344 return self.data == other.data
348 """Connection to the QEMU Monitor using the QEMU Monitor Protocol (QMP).
351 _FIRST_MESSAGE_KEY = "QMP"
354 _RETURN_KEY = RETURN_KEY = "return"
355 _ACTUAL_KEY = ACTUAL_KEY = "actual"
356 _ERROR_CLASS_KEY = "class"
357 _ERROR_DATA_KEY = "data"
358 _ERROR_DESC_KEY = "desc"
359 _EXECUTE_KEY = "execute"
360 _ARGUMENTS_KEY = "arguments"
361 _CAPABILITIES_COMMAND = "qmp_capabilities"
362 _MESSAGE_END_TOKEN = "\r\n"
365 def __init__(self, monitor_filename):
366 """Instantiates the QmpConnection object.
368 @type monitor_filename: string
369 @param monitor_filename: the filename of the UNIX raw socket on which the
370 QMP monitor is listening
373 self.monitor_filename = monitor_filename
374 self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
375 # We want to fail if the server doesn't send a complete message
376 # in a reasonable amount of time
377 self.sock.settimeout(self._SOCKET_TIMEOUT)
378 self._connected = False
381 def _check_socket(self):
384 sock_stat = os.stat(self.monitor_filename)
385 except EnvironmentError, err:
386 if err.errno == errno.ENOENT:
387 raise errors.HypervisorError("No qmp socket found")
389 raise errors.HypervisorError("Error checking qmp socket: %s",
390 utils.ErrnoOrStr(err))
391 if not stat.S_ISSOCK(sock_stat.st_mode):
392 raise errors.HypervisorError("Qmp socket is not a socket")
394 def _check_connection(self):
395 """Make sure that the connection is established.
398 if not self._connected:
399 raise errors.ProgrammerError("To use a QmpConnection you need to first"
400 " invoke connect() on it")
403 """Connects to the QMP monitor.
405 Connects to the UNIX socket and makes sure that we can actually send and
406 receive data to the kvm instance via QMP.
408 @raise errors.HypervisorError: when there are communication errors
409 @raise errors.ProgrammerError: when there are data serialization errors
413 raise errors.ProgrammerError("Cannot connect twice")
417 # Check file existance/stuff
419 self.sock.connect(self.monitor_filename)
420 except EnvironmentError:
421 raise errors.HypervisorError("Can't connect to qmp socket")
422 self._connected = True
424 # Check if we receive a correct greeting message from the server
425 # (As per the QEMU Protocol Specification 0.1 - section 2.2)
426 greeting = self._Recv()
427 if not greeting[self._FIRST_MESSAGE_KEY]:
428 self._connected = False
429 raise errors.HypervisorError("kvm: QMP communication error (wrong"
432 # Let's put the monitor in command mode using the qmp_capabilities
433 # command, or else no command will be executable.
434 # (As per the QEMU Protocol Specification 0.1 - section 4)
435 self.Execute(self._CAPABILITIES_COMMAND)
437 def _ParseMessage(self, buf):
438 """Extract and parse a QMP message from the given buffer.
440 Seeks for a QMP message in the given buf. If found, it parses it and
441 returns it together with the rest of the characters in the buf.
442 If no message is found, returns None and the whole buffer.
444 @raise errors.ProgrammerError: when there are data serialization errors
448 # Check if we got the message end token (CRLF, as per the QEMU Protocol
449 # Specification 0.1 - Section 2.1.1)
450 pos = buf.find(self._MESSAGE_END_TOKEN)
453 message = QmpMessage.BuildFromJsonString(buf[:pos + 1])
454 except Exception, err:
455 raise errors.ProgrammerError("QMP data serialization error: %s" % err)
458 return (message, buf)
461 """Receives a message from QMP and decodes the received JSON object.
464 @return: the received message
465 @raise errors.HypervisorError: when there are communication errors
466 @raise errors.ProgrammerError: when there are data serialization errors
469 self._check_connection()
471 # Check if there is already a message in the buffer
472 (message, self._buf) = self._ParseMessage(self._buf)
476 recv_buffer = StringIO.StringIO(self._buf)
477 recv_buffer.seek(len(self._buf))
480 data = self.sock.recv(4096)
483 recv_buffer.write(data)
485 (message, self._buf) = self._ParseMessage(recv_buffer.getvalue())
489 except socket.timeout, err:
490 raise errors.HypervisorError("Timeout while receiving a QMP message: "
492 except socket.error, err:
493 raise errors.HypervisorError("Unable to receive data from KVM using the"
494 " QMP protocol: %s" % err)
496 def _Send(self, message):
497 """Encodes and sends a message to KVM using QMP.
499 @type message: QmpMessage
500 @param message: message to send to KVM
501 @raise errors.HypervisorError: when there are communication errors
502 @raise errors.ProgrammerError: when there are data serialization errors
505 self._check_connection()
507 message_str = str(message)
508 except Exception, err:
509 raise errors.ProgrammerError("QMP data deserialization error: %s" % err)
512 self.sock.sendall(message_str)
513 except socket.timeout, err:
514 raise errors.HypervisorError("Timeout while sending a QMP message: "
515 "%s (%s)" % (err.string, err.errno))
516 except socket.error, err:
517 raise errors.HypervisorError("Unable to send data from KVM using the"
518 " QMP protocol: %s" % err)
520 def Execute(self, command, arguments=None):
521 """Executes a QMP command and returns the response of the server.
524 @param command: the command to execute
525 @type arguments: dict
526 @param arguments: dictionary of arguments to be passed to the command
528 @return: dictionary representing the received JSON object
529 @raise errors.HypervisorError: when there are communication errors
530 @raise errors.ProgrammerError: when there are data serialization errors
533 self._check_connection()
534 message = QmpMessage({self._EXECUTE_KEY: command})
536 message[self._ARGUMENTS_KEY] = arguments
539 # Events can occur between the sending of the command and the reception
540 # of the response, so we need to filter out messages with the event key.
542 response = self._Recv()
543 err = response[self._ERROR_KEY]
545 raise errors.HypervisorError("kvm: error executing the %s"
546 " command: %s (%s, %s):" %
548 err[self._ERROR_DESC_KEY],
549 err[self._ERROR_CLASS_KEY],
550 err[self._ERROR_DATA_KEY]))
552 elif not response[self._EVENT_KEY]:
556 class KVMHypervisor(hv_base.BaseHypervisor):
557 """KVM hypervisor interface
562 _ROOT_DIR = pathutils.RUN_DIR + "/kvm-hypervisor"
563 _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids
564 _UIDS_DIR = _ROOT_DIR + "/uid" # contains instances reserved uids
565 _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets
566 _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data
567 _NICS_DIR = _ROOT_DIR + "/nic" # contains instances nic <-> tap associations
568 _KEYMAP_DIR = _ROOT_DIR + "/keymap" # contains instances keymaps
569 # KVM instances with chroot enabled are started in empty chroot directories.
570 _CHROOT_DIR = _ROOT_DIR + "/chroot" # for empty chroot directories
571 # After an instance is stopped, its chroot directory is removed.
572 # If the chroot directory is not empty, it can't be removed.
573 # A non-empty chroot directory indicates a possible security incident.
574 # To support forensics, the non-empty chroot directory is quarantined in
575 # a separate directory, called 'chroot-quarantine'.
576 _CHROOT_QUARANTINE_DIR = _ROOT_DIR + "/chroot-quarantine"
577 _DIRS = [_ROOT_DIR, _PIDS_DIR, _UIDS_DIR, _CTRL_DIR, _CONF_DIR, _NICS_DIR,
578 _CHROOT_DIR, _CHROOT_QUARANTINE_DIR, _KEYMAP_DIR]
581 constants.HV_KVM_PATH: hv_base.REQ_FILE_CHECK,
582 constants.HV_KERNEL_PATH: hv_base.OPT_FILE_CHECK,
583 constants.HV_INITRD_PATH: hv_base.OPT_FILE_CHECK,
584 constants.HV_ROOT_PATH: hv_base.NO_CHECK,
585 constants.HV_KERNEL_ARGS: hv_base.NO_CHECK,
586 constants.HV_ACPI: hv_base.NO_CHECK,
587 constants.HV_SERIAL_CONSOLE: hv_base.NO_CHECK,
588 constants.HV_SERIAL_SPEED: hv_base.NO_CHECK,
589 constants.HV_VNC_BIND_ADDRESS:
590 (False, lambda x: (netutils.IP4Address.IsValid(x) or
591 utils.IsNormAbsPath(x)),
592 "The VNC bind address must be either a valid IP address or an absolute"
593 " pathname", None, None),
594 constants.HV_VNC_TLS: hv_base.NO_CHECK,
595 constants.HV_VNC_X509: hv_base.OPT_DIR_CHECK,
596 constants.HV_VNC_X509_VERIFY: hv_base.NO_CHECK,
597 constants.HV_VNC_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
598 constants.HV_KVM_SPICE_BIND: hv_base.NO_CHECK, # will be checked later
599 constants.HV_KVM_SPICE_IP_VERSION:
600 (False, lambda x: (x == constants.IFACE_NO_IP_VERSION_SPECIFIED or
601 x in constants.VALID_IP_VERSIONS),
602 "The SPICE IP version should be 4 or 6",
604 constants.HV_KVM_SPICE_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
605 constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR:
607 False, constants.HT_KVM_SPICE_VALID_LOSSLESS_IMG_COMPR_OPTIONS),
608 constants.HV_KVM_SPICE_JPEG_IMG_COMPR:
610 False, constants.HT_KVM_SPICE_VALID_LOSSY_IMG_COMPR_OPTIONS),
611 constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR:
613 False, constants.HT_KVM_SPICE_VALID_LOSSY_IMG_COMPR_OPTIONS),
614 constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION:
616 False, constants.HT_KVM_SPICE_VALID_VIDEO_STREAM_DETECTION_OPTIONS),
617 constants.HV_KVM_SPICE_AUDIO_COMPR: hv_base.NO_CHECK,
618 constants.HV_KVM_SPICE_USE_TLS: hv_base.NO_CHECK,
619 constants.HV_KVM_SPICE_TLS_CIPHERS: hv_base.NO_CHECK,
620 constants.HV_KVM_SPICE_USE_VDAGENT: hv_base.NO_CHECK,
621 constants.HV_KVM_FLOPPY_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
622 constants.HV_CDROM_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
623 constants.HV_KVM_CDROM2_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
624 constants.HV_BOOT_ORDER:
625 hv_base.ParamInSet(True, constants.HT_KVM_VALID_BO_TYPES),
626 constants.HV_NIC_TYPE:
627 hv_base.ParamInSet(True, constants.HT_KVM_VALID_NIC_TYPES),
628 constants.HV_DISK_TYPE:
629 hv_base.ParamInSet(True, constants.HT_KVM_VALID_DISK_TYPES),
630 constants.HV_KVM_CDROM_DISK_TYPE:
631 hv_base.ParamInSet(False, constants.HT_KVM_VALID_DISK_TYPES),
632 constants.HV_USB_MOUSE:
633 hv_base.ParamInSet(False, constants.HT_KVM_VALID_MOUSE_TYPES),
634 constants.HV_KEYMAP: hv_base.NO_CHECK,
635 constants.HV_MIGRATION_PORT: hv_base.REQ_NET_PORT_CHECK,
636 constants.HV_MIGRATION_BANDWIDTH: hv_base.REQ_NONNEGATIVE_INT_CHECK,
637 constants.HV_MIGRATION_DOWNTIME: hv_base.REQ_NONNEGATIVE_INT_CHECK,
638 constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK,
639 constants.HV_USE_LOCALTIME: hv_base.NO_CHECK,
640 constants.HV_DISK_CACHE:
641 hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES),
642 constants.HV_SECURITY_MODEL:
643 hv_base.ParamInSet(True, constants.HT_KVM_VALID_SM_TYPES),
644 constants.HV_SECURITY_DOMAIN: hv_base.NO_CHECK,
645 constants.HV_KVM_FLAG:
646 hv_base.ParamInSet(False, constants.HT_KVM_FLAG_VALUES),
647 constants.HV_VHOST_NET: hv_base.NO_CHECK,
648 constants.HV_KVM_USE_CHROOT: hv_base.NO_CHECK,
649 constants.HV_MEM_PATH: hv_base.OPT_DIR_CHECK,
650 constants.HV_REBOOT_BEHAVIOR:
651 hv_base.ParamInSet(True, constants.REBOOT_BEHAVIORS),
652 constants.HV_CPU_MASK: hv_base.OPT_MULTI_CPU_MASK_CHECK,
653 constants.HV_CPU_TYPE: hv_base.NO_CHECK,
654 constants.HV_CPU_CORES: hv_base.OPT_NONNEGATIVE_INT_CHECK,
655 constants.HV_CPU_THREADS: hv_base.OPT_NONNEGATIVE_INT_CHECK,
656 constants.HV_CPU_SOCKETS: hv_base.OPT_NONNEGATIVE_INT_CHECK,
657 constants.HV_SOUNDHW: hv_base.NO_CHECK,
658 constants.HV_USB_DEVICES: hv_base.NO_CHECK,
659 constants.HV_VGA: hv_base.NO_CHECK,
660 constants.HV_KVM_EXTRA: hv_base.NO_CHECK,
661 constants.HV_KVM_MACHINE_VERSION: hv_base.NO_CHECK,
662 constants.HV_VNET_HDR: hv_base.NO_CHECK,
666 _VIRTIO_NET_PCI = "virtio-net-pci"
667 _VIRTIO_BLK_PCI = "virtio-blk-pci"
669 _MIGRATION_STATUS_RE = re.compile("Migration\s+status:\s+(\w+)",
671 _MIGRATION_PROGRESS_RE = \
672 re.compile(r"\s*transferred\s+ram:\s+(?P<transferred>\d+)\s+kbytes\s*\n"
673 r"\s*remaining\s+ram:\s+(?P<remaining>\d+)\s+kbytes\s*\n"
674 r"\s*total\s+ram:\s+(?P<total>\d+)\s+kbytes\s*\n", re.I)
676 _MIGRATION_INFO_MAX_BAD_ANSWERS = 5
677 _MIGRATION_INFO_RETRY_DELAY = 2
679 _VERSION_RE = re.compile(r"\b(\d+)\.(\d+)(\.(\d+))?\b")
681 _CPU_INFO_RE = re.compile(r"cpu\s+\#(\d+).*thread_id\s*=\s*(\d+)", re.I)
682 _CPU_INFO_CMD = "info cpus"
685 _DEFAULT_MACHINE_VERSION_RE = re.compile(r"^(\S+).*\(default\)", re.M)
686 _CHECK_MACHINE_VERSION_RE = \
687 staticmethod(lambda x: re.compile(r"^(%s)[ ]+.*PC" % x, re.M))
689 _QMP_RE = re.compile(r"^-qmp\s", re.M)
690 _SPICE_RE = re.compile(r"^-spice\s", re.M)
691 _VHOST_RE = re.compile(r"^-net\s.*,vhost=on|off", re.M)
692 _ENABLE_KVM_RE = re.compile(r"^-enable-kvm\s", re.M)
693 _DISABLE_KVM_RE = re.compile(r"^-disable-kvm\s", re.M)
694 _NETDEV_RE = re.compile(r"^-netdev\s", re.M)
695 _DISPLAY_RE = re.compile(r"^-display\s", re.M)
696 _MACHINE_RE = re.compile(r"^-machine\s", re.M)
697 _VIRTIO_NET_RE = re.compile(r"^name \"%s\"" % _VIRTIO_NET_PCI, re.M)
698 _VIRTIO_BLK_RE = re.compile(r"^name \"%s\"" % _VIRTIO_BLK_PCI, re.M)
699 # match -drive.*boot=on|off on different lines, but in between accept only
700 # dashes not preceeded by a new line (which would mean another option
701 # different than -drive is starting)
702 _BOOT_RE = re.compile(r"^-drive\s([^-]|(?<!^)-)*,boot=on\|off", re.M | re.S)
704 _INFO_PCI_RE = re.compile(r'Bus.*device[ ]*(\d+).*')
705 _INFO_PCI_CMD = "info pci"
707 re.compile(r'^QEMU (\d+)\.(\d+)(\.(\d+))?.*monitor.*', re.M)
708 _INFO_VERSION_CMD = "info version"
710 _DEFAULT_PCI_RESERVATIONS = "11110000000000000000000000000000"
715 ANCILLARY_FILES_OPT = [
719 # Supported kvm options to get output from
720 _KVMOPT_HELP = "help"
721 _KVMOPT_MLIST = "mlist"
722 _KVMOPT_DEVICELIST = "devicelist"
724 # Command to execute to get the output from kvm, and whether to
725 # accept the output even on failure.
727 _KVMOPT_HELP: (["--help"], False),
728 _KVMOPT_MLIST: (["-M", "?"], False),
729 _KVMOPT_DEVICELIST: (["-device", "?"], True),
733 hv_base.BaseHypervisor.__init__(self)
734 # Let's make sure the directories we need exist, even if the RUN_DIR lives
735 # in a tmpfs filesystem or has been otherwise wiped out.
736 dirs = [(dname, constants.RUN_DIRS_MODE) for dname in self._DIRS]
737 utils.EnsureDirs(dirs)
740 def _InstancePidFile(cls, instance_name):
741 """Returns the instance pidfile.
744 return utils.PathJoin(cls._PIDS_DIR, instance_name)
747 def _InstanceUidFile(cls, instance_name):
748 """Returns the instance uidfile.
751 return utils.PathJoin(cls._UIDS_DIR, instance_name)
754 def _InstancePidInfo(cls, pid):
755 """Check pid file for instance information.
757 Check that a pid file is associated with an instance, and retrieve
758 information from its command line.
760 @type pid: string or int
761 @param pid: process id of the instance to check
763 @return: (instance_name, memory, vcpus)
764 @raise errors.HypervisorError: when an instance cannot be found
767 alive = utils.IsProcessAlive(pid)
769 raise errors.HypervisorError("Cannot get info for pid %s" % pid)
771 cmdline_file = utils.PathJoin("/proc", str(pid), "cmdline")
773 cmdline = utils.ReadFile(cmdline_file)
774 except EnvironmentError, err:
775 raise errors.HypervisorError("Can't open cmdline file for pid %s: %s" %
782 arg_list = cmdline.split("\x00")
784 arg = arg_list.pop(0)
786 instance = arg_list.pop(0)
788 memory = int(arg_list.pop(0))
790 vcpus = int(arg_list.pop(0).split(",")[0])
793 raise errors.HypervisorError("Pid %s doesn't contain a ganeti kvm"
796 return (instance, memory, vcpus)
798 def _InstancePidAlive(self, instance_name):
799 """Returns the instance pidfile, pid, and liveness.
801 @type instance_name: string
802 @param instance_name: instance name
804 @return: (pid file name, pid, liveness)
807 pidfile = self._InstancePidFile(instance_name)
808 pid = utils.ReadPidFile(pidfile)
812 cmd_instance = self._InstancePidInfo(pid)[0]
813 alive = (cmd_instance == instance_name)
814 except errors.HypervisorError:
817 return (pidfile, pid, alive)
819 def _CheckDown(self, instance_name):
820 """Raises an error unless the given instance is down.
823 alive = self._InstancePidAlive(instance_name)[2]
825 raise errors.HypervisorError("Failed to start instance %s: %s" %
826 (instance_name, "already running"))
829 def _InstanceMonitor(cls, instance_name):
830 """Returns the instance monitor socket name
833 return utils.PathJoin(cls._CTRL_DIR, "%s.monitor" % instance_name)
836 def _InstanceSerial(cls, instance_name):
837 """Returns the instance serial socket name
840 return utils.PathJoin(cls._CTRL_DIR, "%s.serial" % instance_name)
843 def _InstanceQmpMonitor(cls, instance_name):
844 """Returns the instance serial QMP socket name
847 return utils.PathJoin(cls._CTRL_DIR, "%s.qmp" % instance_name)
850 def _SocatUnixConsoleParams():
851 """Returns the correct parameters for socat
853 If we have a new-enough socat we can use raw mode with an escape character.
856 if constants.SOCAT_USE_ESCAPE:
857 return "raw,echo=0,escape=%s" % constants.SOCAT_ESCAPE_CODE
859 return "echo=0,icanon=0"
862 def _InstanceKVMRuntime(cls, instance_name):
863 """Returns the instance KVM runtime filename
866 return utils.PathJoin(cls._CONF_DIR, "%s.runtime" % instance_name)
869 def _InstanceChrootDir(cls, instance_name):
870 """Returns the name of the KVM chroot dir of the instance
873 return utils.PathJoin(cls._CHROOT_DIR, instance_name)
876 def _InstanceNICDir(cls, instance_name):
877 """Returns the name of the directory holding the tap device files for a
881 return utils.PathJoin(cls._NICS_DIR, instance_name)
884 def _InstanceNICFile(cls, instance_name, seq):
885 """Returns the name of the file containing the tap device for a given NIC
888 return utils.PathJoin(cls._InstanceNICDir(instance_name), str(seq))
891 def _InstanceKeymapFile(cls, instance_name):
892 """Returns the name of the file containing the keymap for a given instance
895 return utils.PathJoin(cls._KEYMAP_DIR, instance_name)
898 def _TryReadUidFile(cls, uid_file):
899 """Try to read a uid file
902 if os.path.exists(uid_file):
904 uid = int(utils.ReadOneLineFile(uid_file))
906 except EnvironmentError:
907 logging.warning("Can't read uid file", exc_info=True)
908 except (TypeError, ValueError):
909 logging.warning("Can't parse uid file contents", exc_info=True)
913 def _RemoveInstanceRuntimeFiles(cls, pidfile, instance_name):
914 """Removes an instance's rutime sockets/files/dirs.
917 utils.RemoveFile(pidfile)
918 utils.RemoveFile(cls._InstanceMonitor(instance_name))
919 utils.RemoveFile(cls._InstanceSerial(instance_name))
920 utils.RemoveFile(cls._InstanceQmpMonitor(instance_name))
921 utils.RemoveFile(cls._InstanceKVMRuntime(instance_name))
922 utils.RemoveFile(cls._InstanceKeymapFile(instance_name))
923 uid_file = cls._InstanceUidFile(instance_name)
924 uid = cls._TryReadUidFile(uid_file)
925 utils.RemoveFile(uid_file)
927 uidpool.ReleaseUid(uid)
929 shutil.rmtree(cls._InstanceNICDir(instance_name))
931 if err.errno != errno.ENOENT:
934 chroot_dir = cls._InstanceChrootDir(instance_name)
935 utils.RemoveDir(chroot_dir)
937 if err.errno == errno.ENOTEMPTY:
938 # The chroot directory is expected to be empty, but it isn't.
939 new_chroot_dir = tempfile.mkdtemp(dir=cls._CHROOT_QUARANTINE_DIR,
942 utils.TimestampForFilename()))
943 logging.warning("The chroot directory of instance %s can not be"
944 " removed as it is not empty. Moving it to the"
945 " quarantine instead. Please investigate the"
946 " contents (%s) and clean up manually",
947 instance_name, new_chroot_dir)
948 utils.RenameFile(chroot_dir, new_chroot_dir)
953 def _ConfigureNIC(instance, seq, nic, tap):
954 """Run the network configuration script for a specified NIC
956 @param instance: instance we're acting on
957 @type instance: instance object
958 @param seq: nic sequence number
960 @param nic: nic we're acting on
961 @type nic: nic object
962 @param tap: the host's tap interface this NIC corresponds to
967 tags = " ".join(instance.tags)
972 "PATH": "%s:/sbin:/usr/sbin" % os.environ["PATH"],
973 "INSTANCE": instance.name,
975 "MODE": nic.nicparams[constants.NIC_MODE],
977 "INTERFACE_INDEX": str(seq),
984 if nic.nicparams[constants.NIC_LINK]:
985 env["LINK"] = nic.nicparams[constants.NIC_LINK]
988 n = objects.Network.FromDict(nic.netinfo)
989 env.update(n.HooksDict())
991 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
992 env["BRIDGE"] = nic.nicparams[constants.NIC_LINK]
994 result = utils.RunCmd([pathutils.KVM_IFUP, tap], env=env)
996 raise errors.HypervisorError("Failed to configure interface %s: %s;"
997 " network configuration script output: %s" %
998 (tap, result.fail_reason, result.output))
1001 def _VerifyAffinityPackage():
1002 if affinity is None:
1003 raise errors.HypervisorError("affinity Python package not"
1004 " found; cannot use CPU pinning under KVM")
1007 def _BuildAffinityCpuMask(cpu_list):
1008 """Create a CPU mask suitable for sched_setaffinity from a list of
1011 See man taskset for more info on sched_setaffinity masks.
1012 For example: [ 0, 2, 5, 6 ] will return 101 (0x65, 0..01100101).
1014 @type cpu_list: list of int
1015 @param cpu_list: list of physical CPU numbers to map to vCPUs in order
1017 @return: a bit mask of CPU affinities
1020 if cpu_list == constants.CPU_PINNING_OFF:
1021 return constants.CPU_PINNING_ALL_KVM
1023 return sum(2 ** cpu for cpu in cpu_list)
1026 def _AssignCpuAffinity(cls, cpu_mask, process_id, thread_dict):
1027 """Change CPU affinity for running VM according to given CPU mask.
1029 @param cpu_mask: CPU mask as given by the user. e.g. "0-2,4:all:1,3"
1030 @type cpu_mask: string
1031 @param process_id: process ID of KVM process. Used to pin entire VM
1033 @type process_id: int
1034 @param thread_dict: map of virtual CPUs to KVM thread IDs
1035 @type thread_dict: dict int:int
1038 # Convert the string CPU mask to a list of list of int's
1039 cpu_list = utils.ParseMultiCpuMask(cpu_mask)
1041 if len(cpu_list) == 1:
1042 all_cpu_mapping = cpu_list[0]
1043 if all_cpu_mapping == constants.CPU_PINNING_OFF:
1044 # If CPU pinning has 1 entry that's "all", then do nothing
1047 # If CPU pinning has one non-all entry, map the entire VM to
1048 # one set of physical CPUs
1049 cls._VerifyAffinityPackage()
1050 affinity.set_process_affinity_mask(
1051 process_id, cls._BuildAffinityCpuMask(all_cpu_mapping))
1053 # The number of vCPUs mapped should match the number of vCPUs
1054 # reported by KVM. This was already verified earlier, so
1055 # here only as a sanity check.
1056 assert len(thread_dict) == len(cpu_list)
1057 cls._VerifyAffinityPackage()
1059 # For each vCPU, map it to the proper list of physical CPUs
1060 for vcpu, i in zip(cpu_list, range(len(cpu_list))):
1061 affinity.set_process_affinity_mask(thread_dict[i],
1062 cls._BuildAffinityCpuMask(vcpu))
1064 def _GetVcpuThreadIds(self, instance_name):
1065 """Get a mapping of vCPU no. to thread IDs for the instance
1067 @type instance_name: string
1068 @param instance_name: instance in question
1069 @rtype: dictionary of int:int
1070 @return: a dictionary mapping vCPU numbers to thread IDs
1074 output = self._CallMonitorCommand(instance_name, self._CPU_INFO_CMD)
1075 for line in output.stdout.splitlines():
1076 match = self._CPU_INFO_RE.search(line)
1079 grp = map(int, match.groups())
1080 result[grp[0]] = grp[1]
1084 def _ExecuteCpuAffinity(self, instance_name, cpu_mask):
1085 """Complete CPU pinning.
1087 @type instance_name: string
1088 @param instance_name: name of instance
1089 @type cpu_mask: string
1090 @param cpu_mask: CPU pinning mask as entered by user
1093 # Get KVM process ID, to be used if need to pin entire VM
1094 _, pid, _ = self._InstancePidAlive(instance_name)
1095 # Get vCPU thread IDs, to be used if need to pin vCPUs separately
1096 thread_dict = self._GetVcpuThreadIds(instance_name)
1097 # Run CPU pinning, based on configured mask
1098 self._AssignCpuAffinity(cpu_mask, pid, thread_dict)
1100 def ListInstances(self):
1101 """Get the list of running instances.
1103 We can do this by listing our live instances directory and
1104 checking whether the associated kvm process is still alive.
1108 for name in os.listdir(self._PIDS_DIR):
1109 if self._InstancePidAlive(name)[2]:
1113 def GetInstanceInfo(self, instance_name):
1114 """Get instance properties.
1116 @type instance_name: string
1117 @param instance_name: the instance name
1118 @rtype: tuple of strings
1119 @return: (name, id, memory, vcpus, stat, times)
1122 _, pid, alive = self._InstancePidAlive(instance_name)
1126 _, memory, vcpus = self._InstancePidInfo(pid)
1131 qmp = QmpConnection(self._InstanceQmpMonitor(instance_name))
1133 vcpus = len(qmp.Execute("query-cpus")[qmp.RETURN_KEY])
1134 # Will fail if ballooning is not enabled, but we can then just resort to
1136 mem_bytes = qmp.Execute("query-balloon")[qmp.RETURN_KEY][qmp.ACTUAL_KEY]
1137 memory = mem_bytes / 1048576
1138 except errors.HypervisorError:
1141 return (instance_name, pid, memory, vcpus, istat, times)
1143 def GetAllInstancesInfo(self):
1144 """Get properties of all instances.
1146 @return: list of tuples (name, id, memory, vcpus, stat, times)
1150 for name in os.listdir(self._PIDS_DIR):
1152 info = self.GetInstanceInfo(name)
1153 except errors.HypervisorError:
1154 # Ignore exceptions due to instances being shut down
1160 def _GenerateKVMBlockDevicesOptions(self, instance, block_devices, kvmhelp):
1162 hvp = instance.hvparams
1163 boot_disk = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
1164 kvm_path = hvp[constants.HV_KVM_PATH]
1166 # whether this is an older KVM version that uses the boot=on flag
1168 needs_boot_flag = self._BOOT_RE.search(kvmhelp)
1171 device_driver = None
1172 disk_type = hvp[constants.HV_DISK_TYPE]
1173 if disk_type == constants.HT_DISK_PARAVIRTUAL:
1174 if_val = ",if=%s" % self._VIRTIO
1176 devlist = self._GetKVMOutput(kvm_path, self._KVMOPT_DEVICELIST)
1177 if self._VIRTIO_BLK_RE.search(devlist):
1179 # will be passed in -device option as driver
1180 device_driver = self._VIRTIO_BLK_PCI
1181 except errors.HypervisorError, _:
1184 if_val = ",if=%s" % disk_type
1186 disk_cache = hvp[constants.HV_DISK_CACHE]
1187 if instance.disk_template in constants.DTS_EXT_MIRROR:
1188 if disk_cache != "none":
1189 # TODO: make this a hard error, instead of a silent overwrite
1190 logging.warning("KVM: overriding disk_cache setting '%s' with 'none'"
1191 " to prevent shared storage corruption on migration",
1193 cache_val = ",cache=none"
1194 elif disk_cache != constants.HT_CACHE_DEFAULT:
1195 cache_val = ",cache=%s" % disk_cache
1198 for cfdev, dev_path in block_devices:
1199 if cfdev.mode != constants.DISK_RDWR:
1200 raise errors.HypervisorError("Instance has read-only disks which"
1201 " are not supported by KVM")
1202 # TODO: handle FD_LOOP and FD_BLKTAP (?)
1205 dev_opts.extend(["-boot", "c"])
1207 if needs_boot_flag and disk_type != constants.HT_DISK_IDE:
1208 boot_val = ",boot=on"
1209 drive_val = "file=%s,format=raw%s%s%s" % \
1210 (dev_path, if_val, boot_val, cache_val)
1213 # block_devices are the 4th entry of runtime file that did not exist in
1214 # the past. That means that cfdev should always have pci slot and
1215 # _GenerateDeviceKVMId() will not raise a exception.
1216 kvm_devid = _GenerateDeviceKVMId(constants.HOTPLUG_TARGET_DISK, cfdev)
1217 drive_val += (",id=%s" % kvm_devid)
1218 drive_val += (",bus=0,unit=%d" % cfdev.pci)
1219 dev_val = ("%s,drive=%s,id=%s" %
1220 (device_driver, kvm_devid, kvm_devid))
1221 dev_val += ",bus=pci.0,addr=%s" % hex(cfdev.pci)
1222 dev_opts.extend(["-device", dev_val])
1224 dev_opts.extend(["-drive", drive_val])
1228 def _GenerateKVMRuntime(self, instance, block_devices, startup_paused,
1230 """Generate KVM information to start an instance.
1232 @type kvmhelp: string
1233 @param kvmhelp: output of kvm --help
1234 @attention: this function must not have any side-effects; for
1235 example, it must not write to the filesystem, or read values
1236 from the current system the are expected to differ between
1237 nodes, since it is only run once at instance startup;
1238 actions/kvm arguments that can vary between systems should be
1239 done in L{_ExecuteKVMRuntime}
1242 # pylint: disable=R0912,R0914,R0915
1243 hvp = instance.hvparams
1244 self.ValidateParameters(hvp)
1246 pidfile = self._InstancePidFile(instance.name)
1247 kvm = hvp[constants.HV_KVM_PATH]
1249 # used just by the vnc server, if enabled
1250 kvm_cmd.extend(["-name", instance.name])
1251 kvm_cmd.extend(["-m", instance.beparams[constants.BE_MAXMEM]])
1253 smp_list = ["%s" % instance.beparams[constants.BE_VCPUS]]
1254 if hvp[constants.HV_CPU_CORES]:
1255 smp_list.append("cores=%s" % hvp[constants.HV_CPU_CORES])
1256 if hvp[constants.HV_CPU_THREADS]:
1257 smp_list.append("threads=%s" % hvp[constants.HV_CPU_THREADS])
1258 if hvp[constants.HV_CPU_SOCKETS]:
1259 smp_list.append("sockets=%s" % hvp[constants.HV_CPU_SOCKETS])
1261 kvm_cmd.extend(["-smp", ",".join(smp_list)])
1263 kvm_cmd.extend(["-pidfile", pidfile])
1264 kvm_cmd.extend(["-balloon", "virtio"])
1265 kvm_cmd.extend(["-daemonize"])
1266 if not instance.hvparams[constants.HV_ACPI]:
1267 kvm_cmd.extend(["-no-acpi"])
1268 if instance.hvparams[constants.HV_REBOOT_BEHAVIOR] == \
1269 constants.INSTANCE_REBOOT_EXIT:
1270 kvm_cmd.extend(["-no-reboot"])
1272 mversion = hvp[constants.HV_KVM_MACHINE_VERSION]
1274 mversion = self._GetDefaultMachineVersion(kvm)
1275 if self._MACHINE_RE.search(kvmhelp):
1276 # TODO (2.8): kernel_irqchip and kvm_shadow_mem machine properties, as
1277 # extra hypervisor parameters. We should also investigate whether and how
1278 # shadow_mem should be considered for the resource model.
1279 if (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED):
1280 specprop = ",accel=kvm"
1283 machinespec = "%s%s" % (mversion, specprop)
1284 kvm_cmd.extend(["-machine", machinespec])
1286 kvm_cmd.extend(["-M", mversion])
1287 if (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED and
1288 self._ENABLE_KVM_RE.search(kvmhelp)):
1289 kvm_cmd.extend(["-enable-kvm"])
1290 elif (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_DISABLED and
1291 self._DISABLE_KVM_RE.search(kvmhelp)):
1292 kvm_cmd.extend(["-disable-kvm"])
1294 kernel_path = hvp[constants.HV_KERNEL_PATH]
1296 boot_cdrom = boot_floppy = boot_network = False
1298 boot_cdrom = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_CDROM
1299 boot_floppy = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_FLOPPY
1300 boot_network = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_NETWORK
1303 kvm_cmd.extend([_KVM_START_PAUSED_FLAG])
1306 kvm_cmd.extend(["-boot", "n"])
1308 # whether this is an older KVM version that uses the boot=on flag
1310 needs_boot_flag = self._BOOT_RE.search(kvmhelp)
1312 disk_type = hvp[constants.HV_DISK_TYPE]
1314 #Now we can specify a different device type for CDROM devices.
1315 cdrom_disk_type = hvp[constants.HV_KVM_CDROM_DISK_TYPE]
1316 if not cdrom_disk_type:
1317 cdrom_disk_type = disk_type
1319 iso_image = hvp[constants.HV_CDROM_IMAGE_PATH]
1321 options = ",format=raw,media=cdrom"
1322 # set cdrom 'if' type
1324 actual_cdrom_type = constants.HT_DISK_IDE
1325 elif cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
1326 actual_cdrom_type = "virtio"
1328 actual_cdrom_type = cdrom_disk_type
1329 if_val = ",if=%s" % actual_cdrom_type
1330 # set boot flag, if needed
1333 kvm_cmd.extend(["-boot", "d"])
1335 boot_val = ",boot=on"
1336 # and finally build the entire '-drive' value
1337 drive_val = "file=%s%s%s%s" % (iso_image, options, if_val, boot_val)
1338 kvm_cmd.extend(["-drive", drive_val])
1340 iso_image2 = hvp[constants.HV_KVM_CDROM2_IMAGE_PATH]
1342 options = ",format=raw,media=cdrom"
1343 if cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
1344 if_val = ",if=virtio"
1346 if_val = ",if=%s" % cdrom_disk_type
1347 drive_val = "file=%s%s%s" % (iso_image2, options, if_val)
1348 kvm_cmd.extend(["-drive", drive_val])
1350 floppy_image = hvp[constants.HV_KVM_FLOPPY_IMAGE_PATH]
1352 options = ",format=raw,media=disk"
1354 kvm_cmd.extend(["-boot", "a"])
1355 options = "%s,boot=on" % options
1356 if_val = ",if=floppy"
1357 options = "%s%s" % (options, if_val)
1358 drive_val = "file=%s%s" % (floppy_image, options)
1359 kvm_cmd.extend(["-drive", drive_val])
1362 kvm_cmd.extend(["-kernel", kernel_path])
1363 initrd_path = hvp[constants.HV_INITRD_PATH]
1365 kvm_cmd.extend(["-initrd", initrd_path])
1366 root_append = ["root=%s" % hvp[constants.HV_ROOT_PATH],
1367 hvp[constants.HV_KERNEL_ARGS]]
1368 if hvp[constants.HV_SERIAL_CONSOLE]:
1369 serial_speed = hvp[constants.HV_SERIAL_SPEED]
1370 root_append.append("console=ttyS0,%s" % serial_speed)
1371 kvm_cmd.extend(["-append", " ".join(root_append)])
1373 mem_path = hvp[constants.HV_MEM_PATH]
1375 kvm_cmd.extend(["-mem-path", mem_path, "-mem-prealloc"])
1377 monitor_dev = ("unix:%s,server,nowait" %
1378 self._InstanceMonitor(instance.name))
1379 kvm_cmd.extend(["-monitor", monitor_dev])
1380 if hvp[constants.HV_SERIAL_CONSOLE]:
1381 serial_dev = ("unix:%s,server,nowait" %
1382 self._InstanceSerial(instance.name))
1383 kvm_cmd.extend(["-serial", serial_dev])
1385 kvm_cmd.extend(["-serial", "none"])
1387 mouse_type = hvp[constants.HV_USB_MOUSE]
1388 vnc_bind_address = hvp[constants.HV_VNC_BIND_ADDRESS]
1389 spice_bind = hvp[constants.HV_KVM_SPICE_BIND]
1390 spice_ip_version = None
1392 kvm_cmd.extend(["-usb"])
1395 kvm_cmd.extend(["-usbdevice", mouse_type])
1396 elif vnc_bind_address:
1397 kvm_cmd.extend(["-usbdevice", constants.HT_MOUSE_TABLET])
1399 if vnc_bind_address:
1400 if netutils.IP4Address.IsValid(vnc_bind_address):
1401 if instance.network_port > constants.VNC_BASE_PORT:
1402 display = instance.network_port - constants.VNC_BASE_PORT
1403 if vnc_bind_address == constants.IP4_ADDRESS_ANY:
1404 vnc_arg = ":%d" % (display)
1406 vnc_arg = "%s:%d" % (vnc_bind_address, display)
1408 logging.error("Network port is not a valid VNC display (%d < %d),"
1409 " not starting VNC",
1410 instance.network_port, constants.VNC_BASE_PORT)
1413 # Only allow tls and other option when not binding to a file, for now.
1414 # kvm/qemu gets confused otherwise about the filename to use.
1416 if hvp[constants.HV_VNC_TLS]:
1417 vnc_append = "%s,tls" % vnc_append
1418 if hvp[constants.HV_VNC_X509_VERIFY]:
1419 vnc_append = "%s,x509verify=%s" % (vnc_append,
1420 hvp[constants.HV_VNC_X509])
1421 elif hvp[constants.HV_VNC_X509]:
1422 vnc_append = "%s,x509=%s" % (vnc_append,
1423 hvp[constants.HV_VNC_X509])
1424 if hvp[constants.HV_VNC_PASSWORD_FILE]:
1425 vnc_append = "%s,password" % vnc_append
1427 vnc_arg = "%s%s" % (vnc_arg, vnc_append)
1430 vnc_arg = "unix:%s/%s.vnc" % (vnc_bind_address, instance.name)
1432 kvm_cmd.extend(["-vnc", vnc_arg])
1434 # FIXME: this is wrong here; the iface ip address differs
1435 # between systems, so it should be done in _ExecuteKVMRuntime
1436 if netutils.IsValidInterface(spice_bind):
1437 # The user specified a network interface, we have to figure out the IP
1439 addresses = netutils.GetInterfaceIpAddresses(spice_bind)
1440 spice_ip_version = hvp[constants.HV_KVM_SPICE_IP_VERSION]
1442 # if the user specified an IP version and the interface does not
1443 # have that kind of IP addresses, throw an exception
1444 if spice_ip_version != constants.IFACE_NO_IP_VERSION_SPECIFIED:
1445 if not addresses[spice_ip_version]:
1446 raise errors.HypervisorError("SPICE: Unable to get an IPv%s address"
1447 " for %s" % (spice_ip_version,
1450 # the user did not specify an IP version, we have to figure it out
1451 elif (addresses[constants.IP4_VERSION] and
1452 addresses[constants.IP6_VERSION]):
1453 # we have both ipv4 and ipv6, let's use the cluster default IP
1455 cluster_family = ssconf.SimpleStore().GetPrimaryIPFamily()
1456 spice_ip_version = \
1457 netutils.IPAddress.GetVersionFromAddressFamily(cluster_family)
1458 elif addresses[constants.IP4_VERSION]:
1459 spice_ip_version = constants.IP4_VERSION
1460 elif addresses[constants.IP6_VERSION]:
1461 spice_ip_version = constants.IP6_VERSION
1463 raise errors.HypervisorError("SPICE: Unable to get an IP address"
1464 " for %s" % (spice_bind))
1466 spice_address = addresses[spice_ip_version][0]
1469 # spice_bind is known to be a valid IP address, because
1470 # ValidateParameters checked it.
1471 spice_address = spice_bind
1473 spice_arg = "addr=%s" % spice_address
1474 if hvp[constants.HV_KVM_SPICE_USE_TLS]:
1475 spice_arg = ("%s,tls-port=%s,x509-cacert-file=%s" %
1476 (spice_arg, instance.network_port,
1477 pathutils.SPICE_CACERT_FILE))
1478 spice_arg = ("%s,x509-key-file=%s,x509-cert-file=%s" %
1479 (spice_arg, pathutils.SPICE_CERT_FILE,
1480 pathutils.SPICE_CERT_FILE))
1481 tls_ciphers = hvp[constants.HV_KVM_SPICE_TLS_CIPHERS]
1483 spice_arg = "%s,tls-ciphers=%s" % (spice_arg, tls_ciphers)
1485 spice_arg = "%s,port=%s" % (spice_arg, instance.network_port)
1487 if not hvp[constants.HV_KVM_SPICE_PASSWORD_FILE]:
1488 spice_arg = "%s,disable-ticketing" % spice_arg
1490 if spice_ip_version:
1491 spice_arg = "%s,ipv%s" % (spice_arg, spice_ip_version)
1493 # Image compression options
1494 img_lossless = hvp[constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR]
1495 img_jpeg = hvp[constants.HV_KVM_SPICE_JPEG_IMG_COMPR]
1496 img_zlib_glz = hvp[constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR]
1498 spice_arg = "%s,image-compression=%s" % (spice_arg, img_lossless)
1500 spice_arg = "%s,jpeg-wan-compression=%s" % (spice_arg, img_jpeg)
1502 spice_arg = "%s,zlib-glz-wan-compression=%s" % (spice_arg, img_zlib_glz)
1504 # Video stream detection
1505 video_streaming = hvp[constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION]
1507 spice_arg = "%s,streaming-video=%s" % (spice_arg, video_streaming)
1509 # Audio compression, by default in qemu-kvm it is on
1510 if not hvp[constants.HV_KVM_SPICE_AUDIO_COMPR]:
1511 spice_arg = "%s,playback-compression=off" % spice_arg
1512 if not hvp[constants.HV_KVM_SPICE_USE_VDAGENT]:
1513 spice_arg = "%s,agent-mouse=off" % spice_arg
1515 # Enable the spice agent communication channel between the host and the
1517 kvm_cmd.extend(["-device", "virtio-serial-pci"])
1520 "virtserialport,chardev=spicechannel0,name=com.redhat.spice.0",
1522 kvm_cmd.extend(["-chardev", "spicevmc,id=spicechannel0,name=vdagent"])
1524 logging.info("KVM: SPICE will listen on port %s", instance.network_port)
1525 kvm_cmd.extend(["-spice", spice_arg])
1528 # From qemu 1.4 -nographic is incompatible with -daemonize. The new way
1529 # also works in earlier versions though (tested with 1.1 and 1.3)
1530 if self._DISPLAY_RE.search(kvmhelp):
1531 kvm_cmd.extend(["-display", "none"])
1533 kvm_cmd.extend(["-nographic"])
1535 if hvp[constants.HV_USE_LOCALTIME]:
1536 kvm_cmd.extend(["-localtime"])
1538 if hvp[constants.HV_KVM_USE_CHROOT]:
1539 kvm_cmd.extend(["-chroot", self._InstanceChrootDir(instance.name)])
1541 # Add qemu-KVM -cpu param
1542 if hvp[constants.HV_CPU_TYPE]:
1543 kvm_cmd.extend(["-cpu", hvp[constants.HV_CPU_TYPE]])
1545 # As requested by music lovers
1546 if hvp[constants.HV_SOUNDHW]:
1547 kvm_cmd.extend(["-soundhw", hvp[constants.HV_SOUNDHW]])
1549 # Pass a -vga option if requested, or if spice is used, for backwards
1551 if hvp[constants.HV_VGA]:
1552 kvm_cmd.extend(["-vga", hvp[constants.HV_VGA]])
1554 kvm_cmd.extend(["-vga", "qxl"])
1556 # Various types of usb devices, comma separated
1557 if hvp[constants.HV_USB_DEVICES]:
1558 for dev in hvp[constants.HV_USB_DEVICES].split(","):
1559 kvm_cmd.extend(["-usbdevice", dev])
1561 if hvp[constants.HV_KVM_EXTRA]:
1562 kvm_cmd.extend(hvp[constants.HV_KVM_EXTRA].split(" "))
1564 pci_reservations = bitarray(self._DEFAULT_PCI_RESERVATIONS)
1566 for disk, link_name in block_devices:
1567 _UpdatePCISlots(disk, pci_reservations)
1568 kvm_disks.append((disk, link_name))
1571 for nic in instance.nics:
1572 _UpdatePCISlots(nic, pci_reservations)
1573 kvm_nics.append(nic)
1577 return (kvm_cmd, kvm_nics, hvparams, kvm_disks)
1579 def _WriteKVMRuntime(self, instance_name, data):
1580 """Write an instance's KVM runtime
1584 utils.WriteFile(self._InstanceKVMRuntime(instance_name),
1586 except EnvironmentError, err:
1587 raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err)
1589 def _ReadKVMRuntime(self, instance_name):
1590 """Read an instance's KVM runtime
1594 file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name))
1595 except EnvironmentError, err:
1596 raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err)
1599 def _SaveKVMRuntime(self, instance, kvm_runtime):
1600 """Save an instance's KVM runtime
1603 kvm_cmd, kvm_nics, hvparams, block_devices = kvm_runtime
1605 serialized_nics = [nic.ToDict() for nic in kvm_nics]
1606 serialized_blockdevs = [(blk.ToDict(), link)
1607 for blk, link in block_devices]
1608 serialized_form = serializer.Dump((kvm_cmd, serialized_nics, hvparams,
1609 serialized_blockdevs))
1611 self._WriteKVMRuntime(instance.name, serialized_form)
1613 def _LoadKVMRuntime(self, instance, serialized_runtime=None):
1614 """Load an instance's KVM runtime
1617 if not serialized_runtime:
1618 serialized_runtime = self._ReadKVMRuntime(instance.name)
1620 return _AnalyzeSerializedRuntime(serialized_runtime)
1622 def _RunKVMCmd(self, name, kvm_cmd, tap_fds=None):
1623 """Run the KVM cmd and check for errors
1626 @param name: instance name
1627 @type kvm_cmd: list of strings
1628 @param kvm_cmd: runcmd input for kvm
1629 @type tap_fds: list of int
1630 @param tap_fds: fds of tap devices opened by Ganeti
1634 result = utils.RunCmd(kvm_cmd, noclose_fds=tap_fds)
1637 utils_wrapper.CloseFdNoError(fd)
1640 raise errors.HypervisorError("Failed to start instance %s: %s (%s)" %
1641 (name, result.fail_reason, result.output))
1642 if not self._InstancePidAlive(name)[2]:
1643 raise errors.HypervisorError("Failed to start instance %s" % name)
1645 # 52/50 local variables
1646 # pylint: disable=R0914
1647 def _ExecuteKVMRuntime(self, instance, kvm_runtime, kvmhelp, incoming=None):
1648 """Execute a KVM cmd, after completing it with some last minute data.
1650 @type incoming: tuple of strings
1651 @param incoming: (target_host_ip, port)
1652 @type kvmhelp: string
1653 @param kvmhelp: output of kvm --help
1656 # Small _ExecuteKVMRuntime hv parameters programming howto:
1657 # - conf_hvp contains the parameters as configured on ganeti. they might
1658 # have changed since the instance started; only use them if the change
1659 # won't affect the inside of the instance (which hasn't been rebooted).
1660 # - up_hvp contains the parameters as they were when the instance was
1661 # started, plus any new parameter which has been added between ganeti
1662 # versions: it is paramount that those default to a value which won't
1663 # affect the inside of the instance as well.
1664 conf_hvp = instance.hvparams
1665 name = instance.name
1666 self._CheckDown(name)
1670 kvm_cmd, kvm_nics, up_hvp, block_devices = kvm_runtime
1671 # the first element of kvm_cmd is always the path to the kvm binary
1672 kvm_path = kvm_cmd[0]
1673 up_hvp = objects.FillDict(conf_hvp, up_hvp)
1675 # We know it's safe to run as a different user upon migration, so we'll use
1676 # the latest conf, from conf_hvp.
1677 security_model = conf_hvp[constants.HV_SECURITY_MODEL]
1678 if security_model == constants.HT_SM_USER:
1679 kvm_cmd.extend(["-runas", conf_hvp[constants.HV_SECURITY_DOMAIN]])
1681 keymap = conf_hvp[constants.HV_KEYMAP]
1683 keymap_path = self._InstanceKeymapFile(name)
1684 # If a keymap file is specified, KVM won't use its internal defaults. By
1685 # first including the "en-us" layout, an error on loading the actual
1686 # layout (e.g. because it can't be found) won't lead to a non-functional
1687 # keyboard. A keyboard with incorrect keys is still better than none.
1688 utils.WriteFile(keymap_path, data="include en-us\ninclude %s\n" % keymap)
1689 kvm_cmd.extend(["-k", keymap_path])
1691 # We have reasons to believe changing something like the nic driver/type
1692 # upon migration won't exactly fly with the instance kernel, so for nic
1693 # related parameters we'll use up_hvp
1697 kvm_cmd.extend(["-net", "none"])
1701 nic_type = up_hvp[constants.HV_NIC_TYPE]
1702 if nic_type == constants.HT_NIC_PARAVIRTUAL:
1703 nic_model = self._VIRTIO
1705 devlist = self._GetKVMOutput(kvm_path, self._KVMOPT_DEVICELIST)
1706 if self._VIRTIO_NET_RE.search(devlist):
1707 nic_model = self._VIRTIO_NET_PCI
1708 vnet_hdr = up_hvp[constants.HV_VNET_HDR]
1709 except errors.HypervisorError, _:
1710 # Older versions of kvm don't support DEVICE_LIST, but they don't
1711 # have new virtio syntax either.
1714 if up_hvp[constants.HV_VHOST_NET]:
1715 # check for vhost_net support
1716 if self._VHOST_RE.search(kvmhelp):
1717 tap_extra = ",vhost=on"
1719 raise errors.HypervisorError("vhost_net is configured"
1720 " but it is not available")
1722 nic_model = nic_type
1724 kvm_supports_netdev = self._NETDEV_RE.search(kvmhelp)
1726 for nic_seq, nic in enumerate(kvm_nics):
1727 tapname, tapfd = _OpenTap(vnet_hdr=vnet_hdr)
1728 tapfds.append(tapfd)
1729 taps.append(tapname)
1730 if kvm_supports_netdev:
1731 nic_val = "%s,mac=%s" % (nic_model, nic.mac)
1733 # kvm_nics already exist in old runtime files and thus there might
1734 # be some entries without pci slot (therefore try: except:)
1735 kvm_devid = _GenerateDeviceKVMId(constants.HOTPLUG_TARGET_NIC, nic)
1737 nic_val += (",id=%s,bus=pci.0,addr=%s" % (kvm_devid, hex(nic.pci)))
1738 except errors.HotplugError:
1739 netdev = "netdev%d" % nic_seq
1740 nic_val += (",netdev=%s" % netdev)
1741 tap_val = ("type=tap,id=%s,fd=%d%s" %
1742 (netdev, tapfd, tap_extra))
1743 kvm_cmd.extend(["-netdev", tap_val, "-device", nic_val])
1745 nic_val = "nic,vlan=%s,macaddr=%s,model=%s" % (nic_seq,
1747 tap_val = "tap,vlan=%s,fd=%d" % (nic_seq, tapfd)
1748 kvm_cmd.extend(["-net", tap_val, "-net", nic_val])
1751 target, port = incoming
1752 kvm_cmd.extend(["-incoming", "tcp:%s:%s" % (target, port)])
1754 # Changing the vnc password doesn't bother the guest that much. At most it
1755 # will surprise people who connect to it. Whether positively or negatively
1757 vnc_pwd_file = conf_hvp[constants.HV_VNC_PASSWORD_FILE]
1761 vnc_pwd = utils.ReadOneLineFile(vnc_pwd_file, strict=True)
1762 except EnvironmentError, err:
1763 raise errors.HypervisorError("Failed to open VNC password file %s: %s"
1764 % (vnc_pwd_file, err))
1766 if conf_hvp[constants.HV_KVM_USE_CHROOT]:
1767 utils.EnsureDirs([(self._InstanceChrootDir(name),
1768 constants.SECURE_DIR_MODE)])
1770 # Automatically enable QMP if version is >= 0.14
1771 if self._QMP_RE.search(kvmhelp):
1772 logging.debug("Enabling QMP")
1773 kvm_cmd.extend(["-qmp", "unix:%s,server,nowait" %
1774 self._InstanceQmpMonitor(instance.name)])
1776 # Configure the network now for starting instances and bridged interfaces,
1777 # during FinalizeMigration for incoming instances' routed interfaces
1778 for nic_seq, nic in enumerate(kvm_nics):
1780 nic.nicparams[constants.NIC_MODE] != constants.NIC_MODE_BRIDGED):
1782 self._ConfigureNIC(instance, nic_seq, nic, taps[nic_seq])
1784 bdev_opts = self._GenerateKVMBlockDevicesOptions(instance,
1787 kvm_cmd.extend(bdev_opts)
1788 # CPU affinity requires kvm to start paused, so we set this flag if the
1789 # instance is not already paused and if we are not going to accept a
1790 # migrating instance. In the latter case, pausing is not needed.
1791 start_kvm_paused = not (_KVM_START_PAUSED_FLAG in kvm_cmd) and not incoming
1792 if start_kvm_paused:
1793 kvm_cmd.extend([_KVM_START_PAUSED_FLAG])
1795 # Note: CPU pinning is using up_hvp since changes take effect
1796 # during instance startup anyway, and to avoid problems when soft
1797 # rebooting the instance.
1799 if up_hvp.get(constants.HV_CPU_MASK, None):
1802 if security_model == constants.HT_SM_POOL:
1803 ss = ssconf.SimpleStore()
1804 uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\n")
1805 all_uids = set(uidpool.ExpandUidPool(uid_pool))
1806 uid = uidpool.RequestUnusedUid(all_uids)
1808 username = pwd.getpwuid(uid.GetUid()).pw_name
1809 kvm_cmd.extend(["-runas", username])
1810 self._RunKVMCmd(name, kvm_cmd, tapfds)
1812 uidpool.ReleaseUid(uid)
1816 utils.WriteFile(self._InstanceUidFile(name), data=uid.AsStr())
1818 self._RunKVMCmd(name, kvm_cmd, tapfds)
1820 utils.EnsureDirs([(self._InstanceNICDir(instance.name),
1821 constants.RUN_DIRS_MODE)])
1822 for nic_seq, tap in enumerate(taps):
1823 utils.WriteFile(self._InstanceNICFile(instance.name, nic_seq),
1827 change_cmd = "change vnc password %s" % vnc_pwd
1828 self._CallMonitorCommand(instance.name, change_cmd)
1830 # Setting SPICE password. We are not vulnerable to malicious passwordless
1831 # connection attempts because SPICE by default does not allow connections
1832 # if neither a password nor the "disable_ticketing" options are specified.
1833 # As soon as we send the password via QMP, that password is a valid ticket
1835 spice_password_file = conf_hvp[constants.HV_KVM_SPICE_PASSWORD_FILE]
1836 if spice_password_file:
1839 spice_pwd = utils.ReadOneLineFile(spice_password_file, strict=True)
1840 except EnvironmentError, err:
1841 raise errors.HypervisorError("Failed to open SPICE password file %s: %s"
1842 % (spice_password_file, err))
1844 qmp = QmpConnection(self._InstanceQmpMonitor(instance.name))
1847 "protocol": "spice",
1848 "password": spice_pwd,
1850 qmp.Execute("set_password", arguments)
1852 for filename in temp_files:
1853 utils.RemoveFile(filename)
1855 # If requested, set CPU affinity and resume instance execution
1857 self._ExecuteCpuAffinity(instance.name, up_hvp[constants.HV_CPU_MASK])
1859 start_memory = self._InstanceStartupMemory(instance)
1860 if start_memory < instance.beparams[constants.BE_MAXMEM]:
1861 self.BalloonInstanceMemory(instance, start_memory)
1863 if start_kvm_paused:
1864 # To control CPU pinning, ballooning, and vnc/spice passwords
1865 # the VM was started in a frozen state. If freezing was not
1866 # explicitly requested resume the vm status.
1867 self._CallMonitorCommand(instance.name, self._CONT_CMD)
1869 def StartInstance(self, instance, block_devices, startup_paused):
1870 """Start an instance.
1873 self._CheckDown(instance.name)
1874 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
1875 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
1876 kvm_runtime = self._GenerateKVMRuntime(instance, block_devices,
1877 startup_paused, kvmhelp)
1878 self._SaveKVMRuntime(instance, kvm_runtime)
1879 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp)
1881 def _CallMonitorCommand(self, instance_name, command, timeout=None):
1882 """Invoke a command on the instance monitor.
1885 if timeout is not None:
1886 timeout_cmd = "timeout %s" % (timeout, )
1890 # TODO: Replace monitor calls with QMP once KVM >= 0.14 is the minimum
1891 # version. The monitor protocol is designed for human consumption, whereas
1892 # QMP is made for programmatic usage. In the worst case QMP can also
1893 # execute monitor commands. As it is, all calls to socat take at least
1894 # 500ms and likely more: socat can't detect the end of the reply and waits
1895 # for 500ms of no data received before exiting (500 ms is the default for
1896 # the "-t" parameter).
1897 socat = ("echo %s | %s %s STDIO UNIX-CONNECT:%s" %
1898 (utils.ShellQuote(command),
1900 constants.SOCAT_PATH,
1901 utils.ShellQuote(self._InstanceMonitor(instance_name))))
1903 result = utils.RunCmd(socat)
1905 msg = ("Failed to send command '%s' to instance '%s', reason '%s',"
1907 (command, instance_name, result.fail_reason, result.output))
1908 raise errors.HypervisorError(msg)
1912 def _GetFreePCISlot(self, instance, dev):
1913 """Get the first available pci slot of a runnung instance.
1916 slots = bitarray(32)
1917 slots.setall(False) # pylint: disable=E1101
1918 output = self._CallMonitorCommand(instance.name, self._INFO_PCI_CMD)
1919 for line in output.stdout.splitlines():
1920 match = self._INFO_PCI_RE.search(line)
1922 slot = int(match.group(1))
1925 [free] = slots.search(_AVAILABLE_PCI_SLOT, 1) # pylint: disable=E1101
1927 raise errors.HypervisorError("All PCI slots occupied")
1932 def _ParseKVMVersion(cls, text):
1933 """Parse the KVM version from the --help output.
1936 @param text: output of kvm --help
1937 @return: (version, v_maj, v_min, v_rev)
1938 @raise errors.HypervisorError: when the KVM version cannot be retrieved
1941 match = cls._VERSION_RE.search(text.splitlines()[0])
1943 raise errors.HypervisorError("Unable to get KVM version")
1945 v_all = match.group(0)
1946 v_maj = int(match.group(1))
1947 v_min = int(match.group(2))
1949 v_rev = int(match.group(4))
1952 return (v_all, v_maj, v_min, v_rev)
1955 def _GetKVMOutput(cls, kvm_path, option):
1956 """Return the output of a kvm invocation
1958 @type kvm_path: string
1959 @param kvm_path: path to the kvm executable
1960 @type option: a key of _KVMOPTS_CMDS
1961 @param option: kvm option to fetch the output from
1962 @return: output a supported kvm invocation
1963 @raise errors.HypervisorError: when the KVM help output cannot be retrieved
1966 assert option in cls._KVMOPTS_CMDS, "Invalid output option"
1968 optlist, can_fail = cls._KVMOPTS_CMDS[option]
1970 result = utils.RunCmd([kvm_path] + optlist)
1971 if result.failed and not can_fail:
1972 raise errors.HypervisorError("Unable to get KVM %s output" %
1974 return result.output
1977 def _GetKVMVersion(cls, kvm_path):
1978 """Return the installed KVM version.
1980 @return: (version, v_maj, v_min, v_rev)
1981 @raise errors.HypervisorError: when the KVM version cannot be retrieved
1984 return cls._ParseKVMVersion(cls._GetKVMOutput(kvm_path, cls._KVMOPT_HELP))
1987 def _GetDefaultMachineVersion(cls, kvm_path):
1988 """Return the default hardware revision (e.g. pc-1.1)
1991 output = cls._GetKVMOutput(kvm_path, cls._KVMOPT_MLIST)
1992 match = cls._DEFAULT_MACHINE_VERSION_RE.search(output)
1994 return match.group(1)
1998 def StopInstance(self, instance, force=False, retry=False, name=None,
2000 """Stop an instance.
2003 assert(timeout is None or force is not None)
2005 if name is not None and not force:
2006 raise errors.HypervisorError("Cannot shutdown cleanly by name only")
2008 name = instance.name
2009 acpi = instance.hvparams[constants.HV_ACPI]
2012 _, pid, alive = self._InstancePidAlive(name)
2013 if pid > 0 and alive:
2014 if force or not acpi:
2015 utils.KillProcess(pid)
2017 self._CallMonitorCommand(name, "system_powerdown", timeout)
2019 def CleanupInstance(self, instance_name):
2020 """Cleanup after a stopped instance
2023 pidfile, pid, alive = self._InstancePidAlive(instance_name)
2024 if pid > 0 and alive:
2025 raise errors.HypervisorError("Cannot cleanup a live instance")
2026 self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
2028 def RebootInstance(self, instance):
2029 """Reboot an instance.
2032 # For some reason if we do a 'send-key ctrl-alt-delete' to the control
2033 # socket the instance will stop, but now power up again. So we'll resort
2034 # to shutdown and restart.
2035 _, _, alive = self._InstancePidAlive(instance.name)
2037 raise errors.HypervisorError("Failed to reboot instance %s:"
2038 " not running" % instance.name)
2039 # StopInstance will delete the saved KVM runtime so:
2040 # ...first load it...
2041 kvm_runtime = self._LoadKVMRuntime(instance)
2042 # ...now we can safely call StopInstance...
2043 if not self.StopInstance(instance):
2044 self.StopInstance(instance, force=True)
2045 # ...and finally we can save it again, and execute it...
2046 self._SaveKVMRuntime(instance, kvm_runtime)
2047 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
2048 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
2049 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp)
2051 def MigrationInfo(self, instance):
2052 """Get instance information to perform a migration.
2054 @type instance: L{objects.Instance}
2055 @param instance: instance to be migrated
2057 @return: content of the KVM runtime file
2060 return self._ReadKVMRuntime(instance.name)
2062 def AcceptInstance(self, instance, info, target):
2063 """Prepare to accept an instance.
2065 @type instance: L{objects.Instance}
2066 @param instance: instance to be accepted
2068 @param info: content of the KVM runtime file on the source node
2069 @type target: string
2070 @param target: target host (usually ip), on this node
2073 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
2074 incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT])
2075 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
2076 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
2077 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp,
2078 incoming=incoming_address)
2080 def FinalizeMigrationDst(self, instance, info, success):
2081 """Finalize the instance migration on the target node.
2083 Stop the incoming mode KVM.
2085 @type instance: L{objects.Instance}
2086 @param instance: instance whose migration is being finalized
2090 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
2091 kvm_nics = kvm_runtime[1]
2093 for nic_seq, nic in enumerate(kvm_nics):
2094 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2095 # Bridged interfaces have already been configured
2098 tap = utils.ReadFile(self._InstanceNICFile(instance.name, nic_seq))
2099 except EnvironmentError, err:
2100 logging.warning("Failed to find host interface for %s NIC #%d: %s",
2101 instance.name, nic_seq, str(err))
2104 self._ConfigureNIC(instance, nic_seq, nic, tap)
2105 except errors.HypervisorError, err:
2106 logging.warning(str(err))
2108 self._WriteKVMRuntime(instance.name, info)
2110 self.StopInstance(instance, force=True)
2112 def MigrateInstance(self, instance, target, live):
2113 """Migrate an instance to a target node.
2115 The migration will not be attempted if the instance is not
2118 @type instance: L{objects.Instance}
2119 @param instance: the instance to be migrated
2120 @type target: string
2121 @param target: ip address of the target node
2123 @param live: perform a live migration
2126 instance_name = instance.name
2127 port = instance.hvparams[constants.HV_MIGRATION_PORT]
2128 _, _, alive = self._InstancePidAlive(instance_name)
2130 raise errors.HypervisorError("Instance not running, cannot migrate")
2133 self._CallMonitorCommand(instance_name, "stop")
2135 migrate_command = ("migrate_set_speed %dm" %
2136 instance.hvparams[constants.HV_MIGRATION_BANDWIDTH])
2137 self._CallMonitorCommand(instance_name, migrate_command)
2139 migrate_command = ("migrate_set_downtime %dms" %
2140 instance.hvparams[constants.HV_MIGRATION_DOWNTIME])
2141 self._CallMonitorCommand(instance_name, migrate_command)
2143 migrate_command = "migrate -d tcp:%s:%s" % (target, port)
2144 self._CallMonitorCommand(instance_name, migrate_command)
2146 def FinalizeMigrationSource(self, instance, success, live):
2147 """Finalize the instance migration on the source node.
2149 @type instance: L{objects.Instance}
2150 @param instance: the instance that was migrated
2152 @param success: whether the migration succeeded or not
2154 @param live: whether the user requested a live migration or not
2158 pidfile, pid, _ = self._InstancePidAlive(instance.name)
2159 utils.KillProcess(pid)
2160 self._RemoveInstanceRuntimeFiles(pidfile, instance.name)
2162 self._CallMonitorCommand(instance.name, self._CONT_CMD)
2164 def GetMigrationStatus(self, instance):
2165 """Get the migration status
2167 @type instance: L{objects.Instance}
2168 @param instance: the instance that is being migrated
2169 @rtype: L{objects.MigrationStatus}
2170 @return: the status of the current migration (one of
2171 L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional
2172 progress info that can be retrieved from the hypervisor
2175 info_command = "info migrate"
2176 for _ in range(self._MIGRATION_INFO_MAX_BAD_ANSWERS):
2177 result = self._CallMonitorCommand(instance.name, info_command)
2178 match = self._MIGRATION_STATUS_RE.search(result.stdout)
2180 if not result.stdout:
2181 logging.info("KVM: empty 'info migrate' result")
2183 logging.warning("KVM: unknown 'info migrate' result: %s",
2186 status = match.group(1)
2187 if status in constants.HV_KVM_MIGRATION_VALID_STATUSES:
2188 migration_status = objects.MigrationStatus(status=status)
2189 match = self._MIGRATION_PROGRESS_RE.search(result.stdout)
2191 migration_status.transferred_ram = match.group("transferred")
2192 migration_status.total_ram = match.group("total")
2194 return migration_status
2196 logging.warning("KVM: unknown migration status '%s'", status)
2198 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
2200 return objects.MigrationStatus(status=constants.HV_MIGRATION_FAILED)
2202 def BalloonInstanceMemory(self, instance, mem):
2203 """Balloon an instance memory to a certain value.
2205 @type instance: L{objects.Instance}
2206 @param instance: instance to be accepted
2208 @param mem: actual memory size to use for instance runtime
2211 self._CallMonitorCommand(instance.name, "balloon %d" % mem)
2213 def GetNodeInfo(self):
2214 """Return information about the node.
2216 @return: a dict with the following keys (values in MiB):
2217 - memory_total: the total memory size on the node
2218 - memory_free: the available memory on the node for instances
2219 - memory_dom0: the memory used by the node itself, if available
2220 - hv_version: the hypervisor version in the form (major, minor,
2224 result = self.GetLinuxNodeInfo()
2225 # FIXME: this is the global kvm version, but the actual version can be
2226 # customized as an hv parameter. we should use the nodegroup's default kvm
2227 # path parameter here.
2228 _, v_major, v_min, v_rev = self._GetKVMVersion(constants.KVM_PATH)
2229 result[constants.HV_NODEINFO_KEY_VERSION] = (v_major, v_min, v_rev)
2233 def GetInstanceConsole(cls, instance, hvparams, beparams):
2234 """Return a command for connecting to the console of an instance.
2237 if hvparams[constants.HV_SERIAL_CONSOLE]:
2238 cmd = [pathutils.KVM_CONSOLE_WRAPPER,
2239 constants.SOCAT_PATH, utils.ShellQuote(instance.name),
2240 utils.ShellQuote(cls._InstanceMonitor(instance.name)),
2241 "STDIO,%s" % cls._SocatUnixConsoleParams(),
2242 "UNIX-CONNECT:%s" % cls._InstanceSerial(instance.name)]
2243 return objects.InstanceConsole(instance=instance.name,
2244 kind=constants.CONS_SSH,
2245 host=instance.primary_node,
2246 user=constants.SSH_CONSOLE_USER,
2249 vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS]
2250 if vnc_bind_address and instance.network_port > constants.VNC_BASE_PORT:
2251 display = instance.network_port - constants.VNC_BASE_PORT
2252 return objects.InstanceConsole(instance=instance.name,
2253 kind=constants.CONS_VNC,
2254 host=vnc_bind_address,
2255 port=instance.network_port,
2258 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2260 return objects.InstanceConsole(instance=instance.name,
2261 kind=constants.CONS_SPICE,
2263 port=instance.network_port)
2265 return objects.InstanceConsole(instance=instance.name,
2266 kind=constants.CONS_MESSAGE,
2267 message=("No serial shell for instance %s" %
2271 """Verify the hypervisor.
2273 Check that the required binaries exist.
2275 @return: Problem description if something is wrong, C{None} otherwise
2279 # FIXME: this is the global kvm binary, but the actual path can be
2280 # customized as an hv parameter; we should use the nodegroup's
2281 # default kvm path parameter here.
2282 if not os.path.exists(constants.KVM_PATH):
2283 msgs.append("The KVM binary ('%s') does not exist" % constants.KVM_PATH)
2284 if not os.path.exists(constants.SOCAT_PATH):
2285 msgs.append("The socat binary ('%s') does not exist" %
2286 constants.SOCAT_PATH)
2288 return self._FormatVerifyResults(msgs)
2291 def CheckParameterSyntax(cls, hvparams):
2292 """Check the given parameters for validity.
2294 @type hvparams: dict
2295 @param hvparams: dictionary with parameter names/value
2296 @raise errors.HypervisorError: when a parameter is not valid
2299 super(KVMHypervisor, cls).CheckParameterSyntax(hvparams)
2301 kernel_path = hvparams[constants.HV_KERNEL_PATH]
2303 if not hvparams[constants.HV_ROOT_PATH]:
2304 raise errors.HypervisorError("Need a root partition for the instance,"
2305 " if a kernel is defined")
2307 if (hvparams[constants.HV_VNC_X509_VERIFY] and
2308 not hvparams[constants.HV_VNC_X509]):
2309 raise errors.HypervisorError("%s must be defined, if %s is" %
2310 (constants.HV_VNC_X509,
2311 constants.HV_VNC_X509_VERIFY))
2313 if hvparams[constants.HV_SERIAL_CONSOLE]:
2314 serial_speed = hvparams[constants.HV_SERIAL_SPEED]
2315 valid_speeds = constants.VALID_SERIAL_SPEEDS
2316 if not serial_speed or serial_speed not in valid_speeds:
2317 raise errors.HypervisorError("Invalid serial console speed, must be"
2319 utils.CommaJoin(valid_speeds))
2321 boot_order = hvparams[constants.HV_BOOT_ORDER]
2322 if (boot_order == constants.HT_BO_CDROM and
2323 not hvparams[constants.HV_CDROM_IMAGE_PATH]):
2324 raise errors.HypervisorError("Cannot boot from cdrom without an"
2327 security_model = hvparams[constants.HV_SECURITY_MODEL]
2328 if security_model == constants.HT_SM_USER:
2329 if not hvparams[constants.HV_SECURITY_DOMAIN]:
2330 raise errors.HypervisorError("A security domain (user to run kvm as)"
2331 " must be specified")
2332 elif (security_model == constants.HT_SM_NONE or
2333 security_model == constants.HT_SM_POOL):
2334 if hvparams[constants.HV_SECURITY_DOMAIN]:
2335 raise errors.HypervisorError("Cannot have a security domain when the"
2336 " security model is 'none' or 'pool'")
2338 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2339 spice_ip_version = hvparams[constants.HV_KVM_SPICE_IP_VERSION]
2341 if spice_ip_version != constants.IFACE_NO_IP_VERSION_SPECIFIED:
2342 # if an IP version is specified, the spice_bind parameter must be an
2344 if (netutils.IP4Address.IsValid(spice_bind) and
2345 spice_ip_version != constants.IP4_VERSION):
2346 raise errors.HypervisorError("SPICE: Got an IPv4 address (%s), but"
2347 " the specified IP version is %s" %
2348 (spice_bind, spice_ip_version))
2350 if (netutils.IP6Address.IsValid(spice_bind) and
2351 spice_ip_version != constants.IP6_VERSION):
2352 raise errors.HypervisorError("SPICE: Got an IPv6 address (%s), but"
2353 " the specified IP version is %s" %
2354 (spice_bind, spice_ip_version))
2356 # All the other SPICE parameters depend on spice_bind being set. Raise an
2357 # error if any of them is set without it.
2358 for param in _SPICE_ADDITIONAL_PARAMS:
2360 raise errors.HypervisorError("SPICE: %s requires %s to be set" %
2361 (param, constants.HV_KVM_SPICE_BIND))
2364 def ValidateParameters(cls, hvparams):
2365 """Check the given parameters for validity.
2367 @type hvparams: dict
2368 @param hvparams: dictionary with parameter names/value
2369 @raise errors.HypervisorError: when a parameter is not valid
2372 super(KVMHypervisor, cls).ValidateParameters(hvparams)
2374 kvm_path = hvparams[constants.HV_KVM_PATH]
2376 security_model = hvparams[constants.HV_SECURITY_MODEL]
2377 if security_model == constants.HT_SM_USER:
2378 username = hvparams[constants.HV_SECURITY_DOMAIN]
2380 pwd.getpwnam(username)
2382 raise errors.HypervisorError("Unknown security domain user %s"
2385 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2387 # only one of VNC and SPICE can be used currently.
2388 if hvparams[constants.HV_VNC_BIND_ADDRESS]:
2389 raise errors.HypervisorError("Both SPICE and VNC are configured, but"
2390 " only one of them can be used at a"
2393 # check that KVM supports SPICE
2394 kvmhelp = cls._GetKVMOutput(kvm_path, cls._KVMOPT_HELP)
2395 if not cls._SPICE_RE.search(kvmhelp):
2396 raise errors.HypervisorError("SPICE is configured, but it is not"
2397 " supported according to 'kvm --help'")
2399 # if spice_bind is not an IP address, it must be a valid interface
2400 bound_to_addr = (netutils.IP4Address.IsValid(spice_bind) or
2401 netutils.IP6Address.IsValid(spice_bind))
2402 if not bound_to_addr and not netutils.IsValidInterface(spice_bind):
2403 raise errors.HypervisorError("SPICE: The %s parameter must be either"
2404 " a valid IP address or interface name" %
2405 constants.HV_KVM_SPICE_BIND)
2407 machine_version = hvparams[constants.HV_KVM_MACHINE_VERSION]
2409 output = cls._GetKVMOutput(kvm_path, cls._KVMOPT_MLIST)
2410 if not cls._CHECK_MACHINE_VERSION_RE(machine_version).search(output):
2411 raise errors.HypervisorError("Unsupported machine version: %s" %
2415 def PowercycleNode(cls):
2416 """KVM powercycle, just a wrapper over Linux powercycle.
2419 cls.LinuxPowercycle()