4 # Copyright (C) 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
40 from bitarray import bitarray
42 import affinity # pylint: disable=F0401
46 import fdsend # pylint: disable=F0401
50 from ganeti import utils
51 from ganeti import constants
52 from ganeti import errors
53 from ganeti import serializer
54 from ganeti import objects
55 from ganeti import uidpool
56 from ganeti import ssconf
57 from ganeti import netutils
58 from ganeti import pathutils
59 from ganeti.hypervisor import hv_base
60 from ganeti.utils import wrapper as utils_wrapper
63 _KVM_NETWORK_SCRIPT = pathutils.CONF_DIR + "/kvm-vif-bridge"
64 _KVM_START_PAUSED_FLAG = "-S"
66 # TUN/TAP driver constants, taken from <linux/if_tun.h>
67 # They are architecture-independent and already hardcoded in qemu-kvm source,
68 # so we can safely include them here.
69 TUNSETIFF = 0x400454ca
70 TUNGETIFF = 0x800454d2
71 TUNGETFEATURES = 0x800454cf
76 #: SPICE parameters which depend on L{constants.HV_KVM_SPICE_BIND}
77 _SPICE_ADDITIONAL_PARAMS = frozenset([
78 constants.HV_KVM_SPICE_IP_VERSION,
79 constants.HV_KVM_SPICE_PASSWORD_FILE,
80 constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR,
81 constants.HV_KVM_SPICE_JPEG_IMG_COMPR,
82 constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR,
83 constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION,
84 constants.HV_KVM_SPICE_USE_TLS,
87 # Constant bitarray that reflects to a free pci slot
88 # Use it with bitarray.search()
89 _AVAILABLE_PCI_SLOT = bitarray("0")
91 # below constants show the format of runtime file
92 # the nics are in second possition, while the disks in 4th (last)
93 # moreover disk entries are stored in tupples of L{objects.Disk}, dev_path
94 _KVM_NICS_RUNTIME_INDEX = 1
95 _KVM_DISKS_RUNTIME_INDEX = 3
96 _DEVICE_RUNTIME_INDEX = {
97 constants.HOTPLUG_TARGET_DISK: _KVM_DISKS_RUNTIME_INDEX,
98 constants.HOTPLUG_TARGET_NIC: _KVM_NICS_RUNTIME_INDEX
100 _FIND_RUNTIME_ENTRY = {
101 constants.HOTPLUG_TARGET_NIC:
102 lambda nic, kvm_nics: [n for n in kvm_nics if n.uuid == nic.uuid],
103 constants.HOTPLUG_TARGET_DISK:
104 lambda disk, kvm_disks: [(d, l) for (d, l) in kvm_disks
105 if d.uuid == disk.uuid]
108 constants.HOTPLUG_TARGET_NIC: lambda d: d,
109 constants.HOTPLUG_TARGET_DISK: lambda (d, e): d
112 constants.HOTPLUG_TARGET_NIC: lambda d, e: d,
113 constants.HOTPLUG_TARGET_DISK: lambda d, e: (d, e)
117 def _GenerateDeviceKVMId(dev_type, dev):
118 """Helper function to generate a unique device name used by KVM
120 QEMU monitor commands use names to identify devices. Here we use their pci
121 slot and a part of their UUID to name them. dev.pci might be None for old
122 devices in the cluster.
124 @type dev_type: sting
125 @param dev_type: device type of param dev
126 @type dev: L{objects.Disk} or L{objects.NIC}
127 @param dev: the device object for which we generate a kvm name
128 @raise errors.HotplugError: in case a device has no pci slot (old devices)
133 raise errors.HotplugError("Hotplug is not supported for %s with UUID %s" %
134 (dev_type, dev.uuid))
136 return "%s-%s-pci-%d" % (dev_type.lower(), dev.uuid.split("-")[0], dev.pci)
139 def _UpdatePCISlots(dev, pci_reservations):
140 """Update pci configuration for a stopped instance
142 If dev has a pci slot then reserve it, else find first available
143 in pci_reservations bitarray. It acts on the same objects passed
144 as params so there is no need to return anything.
146 @type dev: L{objects.Disk} or L{objects.NIC}
147 @param dev: the device object for which we update its pci slot
148 @type pci_reservations: bitarray
149 @param pci_reservations: existing pci reservations for an instance
150 @raise errors.HotplugError: in case an instance has all its slot occupied
155 else: # pylint: disable=E1103
156 [free] = pci_reservations.search(_AVAILABLE_PCI_SLOT, 1)
158 raise errors.HypervisorError("All PCI slots occupied")
161 pci_reservations[free] = True
164 def _GetExistingDeviceInfo(dev_type, device, runtime):
165 """Helper function to get an existing device inside the runtime file
167 Used when an instance is running. Load kvm runtime file and search
168 for a device based on its type and uuid.
170 @type dev_type: sting
171 @param dev_type: device type of param dev
172 @type device: L{objects.Disk} or L{objects.NIC}
173 @param device: the device object for which we generate a kvm name
174 @type runtime: tuple (cmd, nics, hvparams, disks)
175 @param runtime: the runtime data to search for the device
176 @raise errors.HotplugError: in case the requested device does not
177 exist (e.g. device has been added without --hotplug option) or
178 device info has not pci slot (e.g. old devices in the cluster)
181 index = _DEVICE_RUNTIME_INDEX[dev_type]
182 found = _FIND_RUNTIME_ENTRY[dev_type](device, runtime[index])
184 raise errors.HotplugError("Cannot find runtime info for %s with UUID %s" %
185 (dev_type, device.uuid))
190 def _AnalyzeSerializedRuntime(serialized_runtime):
191 """Return runtime entries for a serialized runtime file
193 @type serialized_runtime: string
194 @param serialized_runtime: raw text data read from actual runtime file
195 @return: (cmd, nics, hvparams, bdevs)
199 loaded_runtime = serializer.Load(serialized_runtime)
200 if len(loaded_runtime) == 3:
201 serialized_blockdevs = []
202 kvm_cmd, serialized_nics, hvparams = loaded_runtime
204 kvm_cmd, serialized_nics, hvparams, serialized_blockdevs = loaded_runtime
206 kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics]
207 block_devices = [(objects.Disk.FromDict(sdisk), link)
208 for sdisk, link in serialized_blockdevs]
210 return (kvm_cmd, kvm_nics, hvparams, block_devices)
213 def _GetTunFeatures(fd, _ioctl=fcntl.ioctl):
214 """Retrieves supported TUN features from file descriptor.
216 @see: L{_ProbeTapVnetHdr}
219 req = struct.pack("I", 0)
221 buf = _ioctl(fd, TUNGETFEATURES, req)
222 except EnvironmentError, err:
223 logging.warning("ioctl(TUNGETFEATURES) failed: %s", err)
226 (flags, ) = struct.unpack("I", buf)
230 def _ProbeTapVnetHdr(fd, _features_fn=_GetTunFeatures):
231 """Check whether to enable the IFF_VNET_HDR flag.
233 To do this, _all_ of the following conditions must be met:
234 1. TUNGETFEATURES ioctl() *must* be implemented
235 2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag
236 3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in
237 drivers/net/tun.c there is no way to test this until after the tap device
238 has been created using TUNSETIFF, and there is no way to change the
239 IFF_VNET_HDR flag after creating the interface, catch-22! However both
240 TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27,
241 thus we can expect TUNGETIFF to be present if TUNGETFEATURES is.
244 @param fd: the file descriptor of /dev/net/tun
247 flags = _features_fn(fd)
253 result = bool(flags & IFF_VNET_HDR)
256 logging.warning("Kernel does not support IFF_VNET_HDR, not enabling")
261 def _OpenTap(vnet_hdr=True):
262 """Open a new tap device and return its file descriptor.
264 This is intended to be used by a qemu-type hypervisor together with the -net
265 tap,fd=<fd> command line parameter.
267 @type vnet_hdr: boolean
268 @param vnet_hdr: Enable the VNET Header
269 @return: (ifname, tapfd)
274 tapfd = os.open("/dev/net/tun", os.O_RDWR)
275 except EnvironmentError:
276 raise errors.HypervisorError("Failed to open /dev/net/tun")
278 flags = IFF_TAP | IFF_NO_PI
280 if vnet_hdr and _ProbeTapVnetHdr(tapfd):
281 flags |= IFF_VNET_HDR
283 # The struct ifreq ioctl request (see netdevice(7))
284 ifr = struct.pack("16sh", "", flags)
287 res = fcntl.ioctl(tapfd, TUNSETIFF, ifr)
288 except EnvironmentError, err:
289 raise errors.HypervisorError("Failed to allocate a new TAP device: %s" %
292 # Get the interface name from the ioctl
293 ifname = struct.unpack("16sh", res)[0].strip("\x00")
294 return (ifname, tapfd)
298 """QEMU Messaging Protocol (QMP) message.
301 def __init__(self, data):
302 """Creates a new QMP message based on the passed data.
305 if not isinstance(data, dict):
306 raise TypeError("QmpMessage must be initialized with a dict")
310 def __getitem__(self, field_name):
311 """Get the value of the required field if present, or None.
313 Overrides the [] operator to provide access to the message data,
314 returning None if the required item is not in the message
315 @return: the value of the field_name field, or None if field_name
316 is not contained in the message
319 return self.data.get(field_name, None)
321 def __setitem__(self, field_name, field_value):
322 """Set the value of the required field_name to field_value.
325 self.data[field_name] = field_value
328 def BuildFromJsonString(json_string):
329 """Build a QmpMessage from a JSON encoded string.
331 @type json_string: str
332 @param json_string: JSON string representing the message
333 @rtype: L{QmpMessage}
334 @return: a L{QmpMessage} built from json_string
338 data = serializer.LoadJson(json_string)
339 return QmpMessage(data)
342 # The protocol expects the JSON object to be sent as a single line.
343 return serializer.DumpJson(self.data)
345 def __eq__(self, other):
346 # When comparing two QmpMessages, we are interested in comparing
347 # their internal representation of the message data
348 return self.data == other.data
352 """Connection to the QEMU Monitor using the QEMU Monitor Protocol (QMP).
355 _FIRST_MESSAGE_KEY = "QMP"
358 _RETURN_KEY = RETURN_KEY = "return"
359 _ACTUAL_KEY = ACTUAL_KEY = "actual"
360 _ERROR_CLASS_KEY = "class"
361 _ERROR_DATA_KEY = "data"
362 _ERROR_DESC_KEY = "desc"
363 _EXECUTE_KEY = "execute"
364 _ARGUMENTS_KEY = "arguments"
365 _CAPABILITIES_COMMAND = "qmp_capabilities"
366 _MESSAGE_END_TOKEN = "\r\n"
369 def __init__(self, monitor_filename):
370 """Instantiates the QmpConnection object.
372 @type monitor_filename: string
373 @param monitor_filename: the filename of the UNIX raw socket on which the
374 QMP monitor is listening
377 self.monitor_filename = monitor_filename
378 self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
379 # We want to fail if the server doesn't send a complete message
380 # in a reasonable amount of time
381 self.sock.settimeout(self._SOCKET_TIMEOUT)
382 self._connected = False
385 def _check_socket(self):
388 sock_stat = os.stat(self.monitor_filename)
389 except EnvironmentError, err:
390 if err.errno == errno.ENOENT:
391 raise errors.HypervisorError("No qmp socket found")
393 raise errors.HypervisorError("Error checking qmp socket: %s",
394 utils.ErrnoOrStr(err))
395 if not stat.S_ISSOCK(sock_stat.st_mode):
396 raise errors.HypervisorError("Qmp socket is not a socket")
398 def _check_connection(self):
399 """Make sure that the connection is established.
402 if not self._connected:
403 raise errors.ProgrammerError("To use a QmpConnection you need to first"
404 " invoke connect() on it")
407 """Connects to the QMP monitor.
409 Connects to the UNIX socket and makes sure that we can actually send and
410 receive data to the kvm instance via QMP.
412 @raise errors.HypervisorError: when there are communication errors
413 @raise errors.ProgrammerError: when there are data serialization errors
417 raise errors.ProgrammerError("Cannot connect twice")
421 # Check file existance/stuff
423 self.sock.connect(self.monitor_filename)
424 except EnvironmentError:
425 raise errors.HypervisorError("Can't connect to qmp socket")
426 self._connected = True
428 # Check if we receive a correct greeting message from the server
429 # (As per the QEMU Protocol Specification 0.1 - section 2.2)
430 greeting = self._Recv()
431 if not greeting[self._FIRST_MESSAGE_KEY]:
432 self._connected = False
433 raise errors.HypervisorError("kvm: QMP communication error (wrong"
436 # Let's put the monitor in command mode using the qmp_capabilities
437 # command, or else no command will be executable.
438 # (As per the QEMU Protocol Specification 0.1 - section 4)
439 self.Execute(self._CAPABILITIES_COMMAND)
441 def _ParseMessage(self, buf):
442 """Extract and parse a QMP message from the given buffer.
444 Seeks for a QMP message in the given buf. If found, it parses it and
445 returns it together with the rest of the characters in the buf.
446 If no message is found, returns None and the whole buffer.
448 @raise errors.ProgrammerError: when there are data serialization errors
452 # Check if we got the message end token (CRLF, as per the QEMU Protocol
453 # Specification 0.1 - Section 2.1.1)
454 pos = buf.find(self._MESSAGE_END_TOKEN)
457 message = QmpMessage.BuildFromJsonString(buf[:pos + 1])
458 except Exception, err:
459 raise errors.ProgrammerError("QMP data serialization error: %s" % err)
462 return (message, buf)
465 """Receives a message from QMP and decodes the received JSON object.
468 @return: the received message
469 @raise errors.HypervisorError: when there are communication errors
470 @raise errors.ProgrammerError: when there are data serialization errors
473 self._check_connection()
475 # Check if there is already a message in the buffer
476 (message, self._buf) = self._ParseMessage(self._buf)
480 recv_buffer = StringIO.StringIO(self._buf)
481 recv_buffer.seek(len(self._buf))
484 data = self.sock.recv(4096)
487 recv_buffer.write(data)
489 (message, self._buf) = self._ParseMessage(recv_buffer.getvalue())
493 except socket.timeout, err:
494 raise errors.HypervisorError("Timeout while receiving a QMP message: "
496 except socket.error, err:
497 raise errors.HypervisorError("Unable to receive data from KVM using the"
498 " QMP protocol: %s" % err)
500 def _Send(self, message):
501 """Encodes and sends a message to KVM using QMP.
503 @type message: QmpMessage
504 @param message: message to send to KVM
505 @raise errors.HypervisorError: when there are communication errors
506 @raise errors.ProgrammerError: when there are data serialization errors
509 self._check_connection()
511 message_str = str(message)
512 except Exception, err:
513 raise errors.ProgrammerError("QMP data deserialization error: %s" % err)
516 self.sock.sendall(message_str)
517 except socket.timeout, err:
518 raise errors.HypervisorError("Timeout while sending a QMP message: "
519 "%s (%s)" % (err.string, err.errno))
520 except socket.error, err:
521 raise errors.HypervisorError("Unable to send data from KVM using the"
522 " QMP protocol: %s" % err)
524 def Execute(self, command, arguments=None):
525 """Executes a QMP command and returns the response of the server.
528 @param command: the command to execute
529 @type arguments: dict
530 @param arguments: dictionary of arguments to be passed to the command
532 @return: dictionary representing the received JSON object
533 @raise errors.HypervisorError: when there are communication errors
534 @raise errors.ProgrammerError: when there are data serialization errors
537 self._check_connection()
538 message = QmpMessage({self._EXECUTE_KEY: command})
540 message[self._ARGUMENTS_KEY] = arguments
543 # Events can occur between the sending of the command and the reception
544 # of the response, so we need to filter out messages with the event key.
546 response = self._Recv()
547 err = response[self._ERROR_KEY]
549 raise errors.HypervisorError("kvm: error executing the %s"
550 " command: %s (%s, %s):" %
552 err[self._ERROR_DESC_KEY],
553 err[self._ERROR_CLASS_KEY],
554 err[self._ERROR_DATA_KEY]))
556 elif not response[self._EVENT_KEY]:
560 class KVMHypervisor(hv_base.BaseHypervisor):
561 """KVM hypervisor interface
566 _ROOT_DIR = pathutils.RUN_DIR + "/kvm-hypervisor"
567 _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids
568 _UIDS_DIR = _ROOT_DIR + "/uid" # contains instances reserved uids
569 _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets
570 _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data
571 _NICS_DIR = _ROOT_DIR + "/nic" # contains instances nic <-> tap associations
572 _KEYMAP_DIR = _ROOT_DIR + "/keymap" # contains instances keymaps
573 # KVM instances with chroot enabled are started in empty chroot directories.
574 _CHROOT_DIR = _ROOT_DIR + "/chroot" # for empty chroot directories
575 # After an instance is stopped, its chroot directory is removed.
576 # If the chroot directory is not empty, it can't be removed.
577 # A non-empty chroot directory indicates a possible security incident.
578 # To support forensics, the non-empty chroot directory is quarantined in
579 # a separate directory, called 'chroot-quarantine'.
580 _CHROOT_QUARANTINE_DIR = _ROOT_DIR + "/chroot-quarantine"
581 _DIRS = [_ROOT_DIR, _PIDS_DIR, _UIDS_DIR, _CTRL_DIR, _CONF_DIR, _NICS_DIR,
582 _CHROOT_DIR, _CHROOT_QUARANTINE_DIR, _KEYMAP_DIR]
585 constants.HV_KVM_PATH: hv_base.REQ_FILE_CHECK,
586 constants.HV_KERNEL_PATH: hv_base.OPT_FILE_CHECK,
587 constants.HV_INITRD_PATH: hv_base.OPT_FILE_CHECK,
588 constants.HV_ROOT_PATH: hv_base.NO_CHECK,
589 constants.HV_KERNEL_ARGS: hv_base.NO_CHECK,
590 constants.HV_ACPI: hv_base.NO_CHECK,
591 constants.HV_SERIAL_CONSOLE: hv_base.NO_CHECK,
592 constants.HV_SERIAL_SPEED: hv_base.NO_CHECK,
593 constants.HV_VNC_BIND_ADDRESS:
594 (False, lambda x: (netutils.IP4Address.IsValid(x) or
595 utils.IsNormAbsPath(x)),
596 "The VNC bind address must be either a valid IP address or an absolute"
597 " pathname", None, None),
598 constants.HV_VNC_TLS: hv_base.NO_CHECK,
599 constants.HV_VNC_X509: hv_base.OPT_DIR_CHECK,
600 constants.HV_VNC_X509_VERIFY: hv_base.NO_CHECK,
601 constants.HV_VNC_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
602 constants.HV_KVM_SPICE_BIND: hv_base.NO_CHECK, # will be checked later
603 constants.HV_KVM_SPICE_IP_VERSION:
604 (False, lambda x: (x == constants.IFACE_NO_IP_VERSION_SPECIFIED or
605 x in constants.VALID_IP_VERSIONS),
606 "The SPICE IP version should be 4 or 6",
608 constants.HV_KVM_SPICE_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
609 constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR:
611 False, constants.HT_KVM_SPICE_VALID_LOSSLESS_IMG_COMPR_OPTIONS),
612 constants.HV_KVM_SPICE_JPEG_IMG_COMPR:
614 False, constants.HT_KVM_SPICE_VALID_LOSSY_IMG_COMPR_OPTIONS),
615 constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR:
617 False, constants.HT_KVM_SPICE_VALID_LOSSY_IMG_COMPR_OPTIONS),
618 constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION:
620 False, constants.HT_KVM_SPICE_VALID_VIDEO_STREAM_DETECTION_OPTIONS),
621 constants.HV_KVM_SPICE_AUDIO_COMPR: hv_base.NO_CHECK,
622 constants.HV_KVM_SPICE_USE_TLS: hv_base.NO_CHECK,
623 constants.HV_KVM_SPICE_TLS_CIPHERS: hv_base.NO_CHECK,
624 constants.HV_KVM_SPICE_USE_VDAGENT: hv_base.NO_CHECK,
625 constants.HV_KVM_FLOPPY_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
626 constants.HV_CDROM_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
627 constants.HV_KVM_CDROM2_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
628 constants.HV_BOOT_ORDER:
629 hv_base.ParamInSet(True, constants.HT_KVM_VALID_BO_TYPES),
630 constants.HV_NIC_TYPE:
631 hv_base.ParamInSet(True, constants.HT_KVM_VALID_NIC_TYPES),
632 constants.HV_DISK_TYPE:
633 hv_base.ParamInSet(True, constants.HT_KVM_VALID_DISK_TYPES),
634 constants.HV_KVM_CDROM_DISK_TYPE:
635 hv_base.ParamInSet(False, constants.HT_KVM_VALID_DISK_TYPES),
636 constants.HV_USB_MOUSE:
637 hv_base.ParamInSet(False, constants.HT_KVM_VALID_MOUSE_TYPES),
638 constants.HV_KEYMAP: hv_base.NO_CHECK,
639 constants.HV_MIGRATION_PORT: hv_base.REQ_NET_PORT_CHECK,
640 constants.HV_MIGRATION_BANDWIDTH: hv_base.REQ_NONNEGATIVE_INT_CHECK,
641 constants.HV_MIGRATION_DOWNTIME: hv_base.REQ_NONNEGATIVE_INT_CHECK,
642 constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK,
643 constants.HV_USE_LOCALTIME: hv_base.NO_CHECK,
644 constants.HV_DISK_CACHE:
645 hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES),
646 constants.HV_SECURITY_MODEL:
647 hv_base.ParamInSet(True, constants.HT_KVM_VALID_SM_TYPES),
648 constants.HV_SECURITY_DOMAIN: hv_base.NO_CHECK,
649 constants.HV_KVM_FLAG:
650 hv_base.ParamInSet(False, constants.HT_KVM_FLAG_VALUES),
651 constants.HV_VHOST_NET: hv_base.NO_CHECK,
652 constants.HV_KVM_USE_CHROOT: hv_base.NO_CHECK,
653 constants.HV_MEM_PATH: hv_base.OPT_DIR_CHECK,
654 constants.HV_REBOOT_BEHAVIOR:
655 hv_base.ParamInSet(True, constants.REBOOT_BEHAVIORS),
656 constants.HV_CPU_MASK: hv_base.OPT_MULTI_CPU_MASK_CHECK,
657 constants.HV_CPU_TYPE: hv_base.NO_CHECK,
658 constants.HV_CPU_CORES: hv_base.OPT_NONNEGATIVE_INT_CHECK,
659 constants.HV_CPU_THREADS: hv_base.OPT_NONNEGATIVE_INT_CHECK,
660 constants.HV_CPU_SOCKETS: hv_base.OPT_NONNEGATIVE_INT_CHECK,
661 constants.HV_SOUNDHW: hv_base.NO_CHECK,
662 constants.HV_USB_DEVICES: hv_base.NO_CHECK,
663 constants.HV_VGA: hv_base.NO_CHECK,
664 constants.HV_KVM_EXTRA: hv_base.NO_CHECK,
665 constants.HV_KVM_MACHINE_VERSION: hv_base.NO_CHECK,
666 constants.HV_VNET_HDR: hv_base.NO_CHECK,
670 _VIRTIO_NET_PCI = "virtio-net-pci"
671 _VIRTIO_BLK_PCI = "virtio-blk-pci"
673 _MIGRATION_STATUS_RE = re.compile("Migration\s+status:\s+(\w+)",
675 _MIGRATION_PROGRESS_RE = \
676 re.compile(r"\s*transferred\s+ram:\s+(?P<transferred>\d+)\s+kbytes\s*\n"
677 r"\s*remaining\s+ram:\s+(?P<remaining>\d+)\s+kbytes\s*\n"
678 r"\s*total\s+ram:\s+(?P<total>\d+)\s+kbytes\s*\n", re.I)
680 _MIGRATION_INFO_MAX_BAD_ANSWERS = 5
681 _MIGRATION_INFO_RETRY_DELAY = 2
683 _VERSION_RE = re.compile(r"\b(\d+)\.(\d+)(\.(\d+))?\b")
685 _CPU_INFO_RE = re.compile(r"cpu\s+\#(\d+).*thread_id\s*=\s*(\d+)", re.I)
686 _CPU_INFO_CMD = "info cpus"
689 _DEFAULT_MACHINE_VERSION_RE = re.compile(r"^(\S+).*\(default\)", re.M)
690 _CHECK_MACHINE_VERSION_RE = \
691 staticmethod(lambda x: re.compile(r"^(%s)[ ]+.*PC" % x, re.M))
693 _QMP_RE = re.compile(r"^-qmp\s", re.M)
694 _SPICE_RE = re.compile(r"^-spice\s", re.M)
695 _VHOST_RE = re.compile(r"^-net\s.*,vhost=on|off", re.M)
696 _ENABLE_KVM_RE = re.compile(r"^-enable-kvm\s", re.M)
697 _DISABLE_KVM_RE = re.compile(r"^-disable-kvm\s", re.M)
698 _NETDEV_RE = re.compile(r"^-netdev\s", re.M)
699 _DISPLAY_RE = re.compile(r"^-display\s", re.M)
700 _MACHINE_RE = re.compile(r"^-machine\s", re.M)
701 _VIRTIO_NET_RE = re.compile(r"^name \"%s\"" % _VIRTIO_NET_PCI, re.M)
702 _VIRTIO_BLK_RE = re.compile(r"^name \"%s\"" % _VIRTIO_BLK_PCI, re.M)
703 # match -drive.*boot=on|off on different lines, but in between accept only
704 # dashes not preceeded by a new line (which would mean another option
705 # different than -drive is starting)
706 _BOOT_RE = re.compile(r"^-drive\s([^-]|(?<!^)-)*,boot=on\|off", re.M | re.S)
708 _INFO_PCI_RE = re.compile(r'Bus.*device[ ]*(\d+).*')
709 _INFO_PCI_CMD = "info pci"
711 re.compile(r'^QEMU (\d+)\.(\d+)(\.(\d+))?.*monitor.*', re.M)
712 _INFO_VERSION_CMD = "info version"
714 _DEFAULT_PCI_RESERVATIONS = "11110000000000000000000000000000"
719 ANCILLARY_FILES_OPT = [
723 # Supported kvm options to get output from
724 _KVMOPT_HELP = "help"
725 _KVMOPT_MLIST = "mlist"
726 _KVMOPT_DEVICELIST = "devicelist"
728 # Command to execute to get the output from kvm, and whether to
729 # accept the output even on failure.
731 _KVMOPT_HELP: (["--help"], False),
732 _KVMOPT_MLIST: (["-M", "?"], False),
733 _KVMOPT_DEVICELIST: (["-device", "?"], True),
737 hv_base.BaseHypervisor.__init__(self)
738 # Let's make sure the directories we need exist, even if the RUN_DIR lives
739 # in a tmpfs filesystem or has been otherwise wiped out.
740 dirs = [(dname, constants.RUN_DIRS_MODE) for dname in self._DIRS]
741 utils.EnsureDirs(dirs)
744 def _InstancePidFile(cls, instance_name):
745 """Returns the instance pidfile.
748 return utils.PathJoin(cls._PIDS_DIR, instance_name)
751 def _InstanceUidFile(cls, instance_name):
752 """Returns the instance uidfile.
755 return utils.PathJoin(cls._UIDS_DIR, instance_name)
758 def _InstancePidInfo(cls, pid):
759 """Check pid file for instance information.
761 Check that a pid file is associated with an instance, and retrieve
762 information from its command line.
764 @type pid: string or int
765 @param pid: process id of the instance to check
767 @return: (instance_name, memory, vcpus)
768 @raise errors.HypervisorError: when an instance cannot be found
771 alive = utils.IsProcessAlive(pid)
773 raise errors.HypervisorError("Cannot get info for pid %s" % pid)
775 cmdline_file = utils.PathJoin("/proc", str(pid), "cmdline")
777 cmdline = utils.ReadFile(cmdline_file)
778 except EnvironmentError, err:
779 raise errors.HypervisorError("Can't open cmdline file for pid %s: %s" %
786 arg_list = cmdline.split("\x00")
788 arg = arg_list.pop(0)
790 instance = arg_list.pop(0)
792 memory = int(arg_list.pop(0))
794 vcpus = int(arg_list.pop(0).split(",")[0])
797 raise errors.HypervisorError("Pid %s doesn't contain a ganeti kvm"
800 return (instance, memory, vcpus)
802 def _InstancePidAlive(self, instance_name):
803 """Returns the instance pidfile, pid, and liveness.
805 @type instance_name: string
806 @param instance_name: instance name
808 @return: (pid file name, pid, liveness)
811 pidfile = self._InstancePidFile(instance_name)
812 pid = utils.ReadPidFile(pidfile)
816 cmd_instance = self._InstancePidInfo(pid)[0]
817 alive = (cmd_instance == instance_name)
818 except errors.HypervisorError:
821 return (pidfile, pid, alive)
823 def _CheckDown(self, instance_name):
824 """Raises an error unless the given instance is down.
827 alive = self._InstancePidAlive(instance_name)[2]
829 raise errors.HypervisorError("Failed to start instance %s: %s" %
830 (instance_name, "already running"))
833 def _InstanceMonitor(cls, instance_name):
834 """Returns the instance monitor socket name
837 return utils.PathJoin(cls._CTRL_DIR, "%s.monitor" % instance_name)
840 def _InstanceSerial(cls, instance_name):
841 """Returns the instance serial socket name
844 return utils.PathJoin(cls._CTRL_DIR, "%s.serial" % instance_name)
847 def _InstanceQmpMonitor(cls, instance_name):
848 """Returns the instance serial QMP socket name
851 return utils.PathJoin(cls._CTRL_DIR, "%s.qmp" % instance_name)
854 def _SocatUnixConsoleParams():
855 """Returns the correct parameters for socat
857 If we have a new-enough socat we can use raw mode with an escape character.
860 if constants.SOCAT_USE_ESCAPE:
861 return "raw,echo=0,escape=%s" % constants.SOCAT_ESCAPE_CODE
863 return "echo=0,icanon=0"
866 def _InstanceKVMRuntime(cls, instance_name):
867 """Returns the instance KVM runtime filename
870 return utils.PathJoin(cls._CONF_DIR, "%s.runtime" % instance_name)
873 def _InstanceChrootDir(cls, instance_name):
874 """Returns the name of the KVM chroot dir of the instance
877 return utils.PathJoin(cls._CHROOT_DIR, instance_name)
880 def _InstanceNICDir(cls, instance_name):
881 """Returns the name of the directory holding the tap device files for a
885 return utils.PathJoin(cls._NICS_DIR, instance_name)
888 def _InstanceNICFile(cls, instance_name, seq):
889 """Returns the name of the file containing the tap device for a given NIC
892 return utils.PathJoin(cls._InstanceNICDir(instance_name), str(seq))
895 def _InstanceKeymapFile(cls, instance_name):
896 """Returns the name of the file containing the keymap for a given instance
899 return utils.PathJoin(cls._KEYMAP_DIR, instance_name)
902 def _TryReadUidFile(cls, uid_file):
903 """Try to read a uid file
906 if os.path.exists(uid_file):
908 uid = int(utils.ReadOneLineFile(uid_file))
910 except EnvironmentError:
911 logging.warning("Can't read uid file", exc_info=True)
912 except (TypeError, ValueError):
913 logging.warning("Can't parse uid file contents", exc_info=True)
917 def _RemoveInstanceRuntimeFiles(cls, pidfile, instance_name):
918 """Removes an instance's rutime sockets/files/dirs.
921 utils.RemoveFile(pidfile)
922 utils.RemoveFile(cls._InstanceMonitor(instance_name))
923 utils.RemoveFile(cls._InstanceSerial(instance_name))
924 utils.RemoveFile(cls._InstanceQmpMonitor(instance_name))
925 utils.RemoveFile(cls._InstanceKVMRuntime(instance_name))
926 utils.RemoveFile(cls._InstanceKeymapFile(instance_name))
927 uid_file = cls._InstanceUidFile(instance_name)
928 uid = cls._TryReadUidFile(uid_file)
929 utils.RemoveFile(uid_file)
931 uidpool.ReleaseUid(uid)
933 shutil.rmtree(cls._InstanceNICDir(instance_name))
935 if err.errno != errno.ENOENT:
938 chroot_dir = cls._InstanceChrootDir(instance_name)
939 utils.RemoveDir(chroot_dir)
941 if err.errno == errno.ENOTEMPTY:
942 # The chroot directory is expected to be empty, but it isn't.
943 new_chroot_dir = tempfile.mkdtemp(dir=cls._CHROOT_QUARANTINE_DIR,
946 utils.TimestampForFilename()))
947 logging.warning("The chroot directory of instance %s can not be"
948 " removed as it is not empty. Moving it to the"
949 " quarantine instead. Please investigate the"
950 " contents (%s) and clean up manually",
951 instance_name, new_chroot_dir)
952 utils.RenameFile(chroot_dir, new_chroot_dir)
957 def _ConfigureNIC(instance, seq, nic, tap):
958 """Run the network configuration script for a specified NIC
960 @param instance: instance we're acting on
961 @type instance: instance object
962 @param seq: nic sequence number
964 @param nic: nic we're acting on
965 @type nic: nic object
966 @param tap: the host's tap interface this NIC corresponds to
971 tags = " ".join(instance.tags)
976 "PATH": "%s:/sbin:/usr/sbin" % os.environ["PATH"],
977 "INSTANCE": instance.name,
979 "MODE": nic.nicparams[constants.NIC_MODE],
981 "INTERFACE_INDEX": str(seq),
988 if nic.nicparams[constants.NIC_LINK]:
989 env["LINK"] = nic.nicparams[constants.NIC_LINK]
992 n = objects.Network.FromDict(nic.netinfo)
993 env.update(n.HooksDict())
995 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
996 env["BRIDGE"] = nic.nicparams[constants.NIC_LINK]
998 result = utils.RunCmd([pathutils.KVM_IFUP, tap], env=env)
1000 raise errors.HypervisorError("Failed to configure interface %s: %s;"
1001 " network configuration script output: %s" %
1002 (tap, result.fail_reason, result.output))
1005 def _VerifyAffinityPackage():
1006 if affinity is None:
1007 raise errors.HypervisorError("affinity Python package not"
1008 " found; cannot use CPU pinning under KVM")
1011 def _BuildAffinityCpuMask(cpu_list):
1012 """Create a CPU mask suitable for sched_setaffinity from a list of
1015 See man taskset for more info on sched_setaffinity masks.
1016 For example: [ 0, 2, 5, 6 ] will return 101 (0x65, 0..01100101).
1018 @type cpu_list: list of int
1019 @param cpu_list: list of physical CPU numbers to map to vCPUs in order
1021 @return: a bit mask of CPU affinities
1024 if cpu_list == constants.CPU_PINNING_OFF:
1025 return constants.CPU_PINNING_ALL_KVM
1027 return sum(2 ** cpu for cpu in cpu_list)
1030 def _AssignCpuAffinity(cls, cpu_mask, process_id, thread_dict):
1031 """Change CPU affinity for running VM according to given CPU mask.
1033 @param cpu_mask: CPU mask as given by the user. e.g. "0-2,4:all:1,3"
1034 @type cpu_mask: string
1035 @param process_id: process ID of KVM process. Used to pin entire VM
1037 @type process_id: int
1038 @param thread_dict: map of virtual CPUs to KVM thread IDs
1039 @type thread_dict: dict int:int
1042 # Convert the string CPU mask to a list of list of int's
1043 cpu_list = utils.ParseMultiCpuMask(cpu_mask)
1045 if len(cpu_list) == 1:
1046 all_cpu_mapping = cpu_list[0]
1047 if all_cpu_mapping == constants.CPU_PINNING_OFF:
1048 # If CPU pinning has 1 entry that's "all", then do nothing
1051 # If CPU pinning has one non-all entry, map the entire VM to
1052 # one set of physical CPUs
1053 cls._VerifyAffinityPackage()
1054 affinity.set_process_affinity_mask(
1055 process_id, cls._BuildAffinityCpuMask(all_cpu_mapping))
1057 # The number of vCPUs mapped should match the number of vCPUs
1058 # reported by KVM. This was already verified earlier, so
1059 # here only as a sanity check.
1060 assert len(thread_dict) == len(cpu_list)
1061 cls._VerifyAffinityPackage()
1063 # For each vCPU, map it to the proper list of physical CPUs
1064 for vcpu, i in zip(cpu_list, range(len(cpu_list))):
1065 affinity.set_process_affinity_mask(thread_dict[i],
1066 cls._BuildAffinityCpuMask(vcpu))
1068 def _GetVcpuThreadIds(self, instance_name):
1069 """Get a mapping of vCPU no. to thread IDs for the instance
1071 @type instance_name: string
1072 @param instance_name: instance in question
1073 @rtype: dictionary of int:int
1074 @return: a dictionary mapping vCPU numbers to thread IDs
1078 output = self._CallMonitorCommand(instance_name, self._CPU_INFO_CMD)
1079 for line in output.stdout.splitlines():
1080 match = self._CPU_INFO_RE.search(line)
1083 grp = map(int, match.groups())
1084 result[grp[0]] = grp[1]
1088 def _ExecuteCpuAffinity(self, instance_name, cpu_mask):
1089 """Complete CPU pinning.
1091 @type instance_name: string
1092 @param instance_name: name of instance
1093 @type cpu_mask: string
1094 @param cpu_mask: CPU pinning mask as entered by user
1097 # Get KVM process ID, to be used if need to pin entire VM
1098 _, pid, _ = self._InstancePidAlive(instance_name)
1099 # Get vCPU thread IDs, to be used if need to pin vCPUs separately
1100 thread_dict = self._GetVcpuThreadIds(instance_name)
1101 # Run CPU pinning, based on configured mask
1102 self._AssignCpuAffinity(cpu_mask, pid, thread_dict)
1104 def ListInstances(self):
1105 """Get the list of running instances.
1107 We can do this by listing our live instances directory and
1108 checking whether the associated kvm process is still alive.
1112 for name in os.listdir(self._PIDS_DIR):
1113 if self._InstancePidAlive(name)[2]:
1117 def GetInstanceInfo(self, instance_name):
1118 """Get instance properties.
1120 @type instance_name: string
1121 @param instance_name: the instance name
1122 @rtype: tuple of strings
1123 @return: (name, id, memory, vcpus, stat, times)
1126 _, pid, alive = self._InstancePidAlive(instance_name)
1130 _, memory, vcpus = self._InstancePidInfo(pid)
1135 qmp = QmpConnection(self._InstanceQmpMonitor(instance_name))
1137 vcpus = len(qmp.Execute("query-cpus")[qmp.RETURN_KEY])
1138 # Will fail if ballooning is not enabled, but we can then just resort to
1140 mem_bytes = qmp.Execute("query-balloon")[qmp.RETURN_KEY][qmp.ACTUAL_KEY]
1141 memory = mem_bytes / 1048576
1142 except errors.HypervisorError:
1145 return (instance_name, pid, memory, vcpus, istat, times)
1147 def GetAllInstancesInfo(self):
1148 """Get properties of all instances.
1150 @return: list of tuples (name, id, memory, vcpus, stat, times)
1154 for name in os.listdir(self._PIDS_DIR):
1156 info = self.GetInstanceInfo(name)
1157 except errors.HypervisorError:
1158 # Ignore exceptions due to instances being shut down
1164 def _GenerateKVMBlockDevicesOptions(self, instance, block_devices, kvmhelp):
1166 hvp = instance.hvparams
1167 boot_disk = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
1168 kvm_path = hvp[constants.HV_KVM_PATH]
1170 # whether this is an older KVM version that uses the boot=on flag
1172 needs_boot_flag = self._BOOT_RE.search(kvmhelp)
1175 device_driver = None
1176 disk_type = hvp[constants.HV_DISK_TYPE]
1177 if disk_type == constants.HT_DISK_PARAVIRTUAL:
1178 if_val = ",if=%s" % self._VIRTIO
1180 devlist = self._GetKVMOutput(kvm_path, self._KVMOPT_DEVICELIST)
1181 if self._VIRTIO_BLK_RE.search(devlist):
1183 # will be passed in -device option as driver
1184 device_driver = self._VIRTIO_BLK_PCI
1185 except errors.HypervisorError, _:
1188 if_val = ",if=%s" % disk_type
1190 disk_cache = hvp[constants.HV_DISK_CACHE]
1191 if instance.disk_template in constants.DTS_EXT_MIRROR:
1192 if disk_cache != "none":
1193 # TODO: make this a hard error, instead of a silent overwrite
1194 logging.warning("KVM: overriding disk_cache setting '%s' with 'none'"
1195 " to prevent shared storage corruption on migration",
1197 cache_val = ",cache=none"
1198 elif disk_cache != constants.HT_CACHE_DEFAULT:
1199 cache_val = ",cache=%s" % disk_cache
1202 for cfdev, dev_path in block_devices:
1203 if cfdev.mode != constants.DISK_RDWR:
1204 raise errors.HypervisorError("Instance has read-only disks which"
1205 " are not supported by KVM")
1206 # TODO: handle FD_LOOP and FD_BLKTAP (?)
1209 dev_opts.extend(["-boot", "c"])
1211 if needs_boot_flag and disk_type != constants.HT_DISK_IDE:
1212 boot_val = ",boot=on"
1213 drive_val = "file=%s,format=raw%s%s%s" % \
1214 (dev_path, if_val, boot_val, cache_val)
1217 # block_devices are the 4th entry of runtime file that did not exist in
1218 # the past. That means that cfdev should always have pci slot and
1219 # _GenerateDeviceKVMId() will not raise a exception.
1220 kvm_devid = _GenerateDeviceKVMId(constants.HOTPLUG_TARGET_DISK, cfdev)
1221 drive_val += (",id=%s" % kvm_devid)
1222 drive_val += (",bus=0,unit=%d" % cfdev.pci)
1223 dev_val = ("%s,drive=%s,id=%s" %
1224 (device_driver, kvm_devid, kvm_devid))
1225 dev_val += ",bus=pci.0,addr=%s" % hex(cfdev.pci)
1226 dev_opts.extend(["-device", dev_val])
1228 dev_opts.extend(["-drive", drive_val])
1232 def _GenerateKVMRuntime(self, instance, block_devices, startup_paused,
1234 """Generate KVM information to start an instance.
1236 @type kvmhelp: string
1237 @param kvmhelp: output of kvm --help
1238 @attention: this function must not have any side-effects; for
1239 example, it must not write to the filesystem, or read values
1240 from the current system the are expected to differ between
1241 nodes, since it is only run once at instance startup;
1242 actions/kvm arguments that can vary between systems should be
1243 done in L{_ExecuteKVMRuntime}
1246 # pylint: disable=R0912,R0914,R0915
1247 hvp = instance.hvparams
1248 self.ValidateParameters(hvp)
1250 pidfile = self._InstancePidFile(instance.name)
1251 kvm = hvp[constants.HV_KVM_PATH]
1253 # used just by the vnc server, if enabled
1254 kvm_cmd.extend(["-name", instance.name])
1255 kvm_cmd.extend(["-m", instance.beparams[constants.BE_MAXMEM]])
1257 smp_list = ["%s" % instance.beparams[constants.BE_VCPUS]]
1258 if hvp[constants.HV_CPU_CORES]:
1259 smp_list.append("cores=%s" % hvp[constants.HV_CPU_CORES])
1260 if hvp[constants.HV_CPU_THREADS]:
1261 smp_list.append("threads=%s" % hvp[constants.HV_CPU_THREADS])
1262 if hvp[constants.HV_CPU_SOCKETS]:
1263 smp_list.append("sockets=%s" % hvp[constants.HV_CPU_SOCKETS])
1265 kvm_cmd.extend(["-smp", ",".join(smp_list)])
1267 kvm_cmd.extend(["-pidfile", pidfile])
1268 kvm_cmd.extend(["-balloon", "virtio"])
1269 kvm_cmd.extend(["-daemonize"])
1270 if not instance.hvparams[constants.HV_ACPI]:
1271 kvm_cmd.extend(["-no-acpi"])
1272 if instance.hvparams[constants.HV_REBOOT_BEHAVIOR] == \
1273 constants.INSTANCE_REBOOT_EXIT:
1274 kvm_cmd.extend(["-no-reboot"])
1276 mversion = hvp[constants.HV_KVM_MACHINE_VERSION]
1278 mversion = self._GetDefaultMachineVersion(kvm)
1279 if self._MACHINE_RE.search(kvmhelp):
1280 # TODO (2.8): kernel_irqchip and kvm_shadow_mem machine properties, as
1281 # extra hypervisor parameters. We should also investigate whether and how
1282 # shadow_mem should be considered for the resource model.
1283 if (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED):
1284 specprop = ",accel=kvm"
1287 machinespec = "%s%s" % (mversion, specprop)
1288 kvm_cmd.extend(["-machine", machinespec])
1290 kvm_cmd.extend(["-M", mversion])
1291 if (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED and
1292 self._ENABLE_KVM_RE.search(kvmhelp)):
1293 kvm_cmd.extend(["-enable-kvm"])
1294 elif (hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_DISABLED and
1295 self._DISABLE_KVM_RE.search(kvmhelp)):
1296 kvm_cmd.extend(["-disable-kvm"])
1298 kernel_path = hvp[constants.HV_KERNEL_PATH]
1300 boot_cdrom = boot_floppy = boot_network = False
1302 boot_cdrom = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_CDROM
1303 boot_floppy = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_FLOPPY
1304 boot_network = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_NETWORK
1307 kvm_cmd.extend([_KVM_START_PAUSED_FLAG])
1310 kvm_cmd.extend(["-boot", "n"])
1312 # whether this is an older KVM version that uses the boot=on flag
1314 needs_boot_flag = self._BOOT_RE.search(kvmhelp)
1316 disk_type = hvp[constants.HV_DISK_TYPE]
1318 #Now we can specify a different device type for CDROM devices.
1319 cdrom_disk_type = hvp[constants.HV_KVM_CDROM_DISK_TYPE]
1320 if not cdrom_disk_type:
1321 cdrom_disk_type = disk_type
1323 iso_image = hvp[constants.HV_CDROM_IMAGE_PATH]
1325 options = ",format=raw,media=cdrom"
1326 # set cdrom 'if' type
1328 actual_cdrom_type = constants.HT_DISK_IDE
1329 elif cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
1330 actual_cdrom_type = "virtio"
1332 actual_cdrom_type = cdrom_disk_type
1333 if_val = ",if=%s" % actual_cdrom_type
1334 # set boot flag, if needed
1337 kvm_cmd.extend(["-boot", "d"])
1339 boot_val = ",boot=on"
1340 # and finally build the entire '-drive' value
1341 drive_val = "file=%s%s%s%s" % (iso_image, options, if_val, boot_val)
1342 kvm_cmd.extend(["-drive", drive_val])
1344 iso_image2 = hvp[constants.HV_KVM_CDROM2_IMAGE_PATH]
1346 options = ",format=raw,media=cdrom"
1347 if cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
1348 if_val = ",if=virtio"
1350 if_val = ",if=%s" % cdrom_disk_type
1351 drive_val = "file=%s%s%s" % (iso_image2, options, if_val)
1352 kvm_cmd.extend(["-drive", drive_val])
1354 floppy_image = hvp[constants.HV_KVM_FLOPPY_IMAGE_PATH]
1356 options = ",format=raw,media=disk"
1358 kvm_cmd.extend(["-boot", "a"])
1359 options = "%s,boot=on" % options
1360 if_val = ",if=floppy"
1361 options = "%s%s" % (options, if_val)
1362 drive_val = "file=%s%s" % (floppy_image, options)
1363 kvm_cmd.extend(["-drive", drive_val])
1366 kvm_cmd.extend(["-kernel", kernel_path])
1367 initrd_path = hvp[constants.HV_INITRD_PATH]
1369 kvm_cmd.extend(["-initrd", initrd_path])
1370 root_append = ["root=%s" % hvp[constants.HV_ROOT_PATH],
1371 hvp[constants.HV_KERNEL_ARGS]]
1372 if hvp[constants.HV_SERIAL_CONSOLE]:
1373 serial_speed = hvp[constants.HV_SERIAL_SPEED]
1374 root_append.append("console=ttyS0,%s" % serial_speed)
1375 kvm_cmd.extend(["-append", " ".join(root_append)])
1377 mem_path = hvp[constants.HV_MEM_PATH]
1379 kvm_cmd.extend(["-mem-path", mem_path, "-mem-prealloc"])
1381 monitor_dev = ("unix:%s,server,nowait" %
1382 self._InstanceMonitor(instance.name))
1383 kvm_cmd.extend(["-monitor", monitor_dev])
1384 if hvp[constants.HV_SERIAL_CONSOLE]:
1385 serial_dev = ("unix:%s,server,nowait" %
1386 self._InstanceSerial(instance.name))
1387 kvm_cmd.extend(["-serial", serial_dev])
1389 kvm_cmd.extend(["-serial", "none"])
1391 mouse_type = hvp[constants.HV_USB_MOUSE]
1392 vnc_bind_address = hvp[constants.HV_VNC_BIND_ADDRESS]
1393 spice_bind = hvp[constants.HV_KVM_SPICE_BIND]
1394 spice_ip_version = None
1396 kvm_cmd.extend(["-usb"])
1399 kvm_cmd.extend(["-usbdevice", mouse_type])
1400 elif vnc_bind_address:
1401 kvm_cmd.extend(["-usbdevice", constants.HT_MOUSE_TABLET])
1403 if vnc_bind_address:
1404 if netutils.IP4Address.IsValid(vnc_bind_address):
1405 if instance.network_port > constants.VNC_BASE_PORT:
1406 display = instance.network_port - constants.VNC_BASE_PORT
1407 if vnc_bind_address == constants.IP4_ADDRESS_ANY:
1408 vnc_arg = ":%d" % (display)
1410 vnc_arg = "%s:%d" % (vnc_bind_address, display)
1412 logging.error("Network port is not a valid VNC display (%d < %d),"
1413 " not starting VNC",
1414 instance.network_port, constants.VNC_BASE_PORT)
1417 # Only allow tls and other option when not binding to a file, for now.
1418 # kvm/qemu gets confused otherwise about the filename to use.
1420 if hvp[constants.HV_VNC_TLS]:
1421 vnc_append = "%s,tls" % vnc_append
1422 if hvp[constants.HV_VNC_X509_VERIFY]:
1423 vnc_append = "%s,x509verify=%s" % (vnc_append,
1424 hvp[constants.HV_VNC_X509])
1425 elif hvp[constants.HV_VNC_X509]:
1426 vnc_append = "%s,x509=%s" % (vnc_append,
1427 hvp[constants.HV_VNC_X509])
1428 if hvp[constants.HV_VNC_PASSWORD_FILE]:
1429 vnc_append = "%s,password" % vnc_append
1431 vnc_arg = "%s%s" % (vnc_arg, vnc_append)
1434 vnc_arg = "unix:%s/%s.vnc" % (vnc_bind_address, instance.name)
1436 kvm_cmd.extend(["-vnc", vnc_arg])
1438 # FIXME: this is wrong here; the iface ip address differs
1439 # between systems, so it should be done in _ExecuteKVMRuntime
1440 if netutils.IsValidInterface(spice_bind):
1441 # The user specified a network interface, we have to figure out the IP
1443 addresses = netutils.GetInterfaceIpAddresses(spice_bind)
1444 spice_ip_version = hvp[constants.HV_KVM_SPICE_IP_VERSION]
1446 # if the user specified an IP version and the interface does not
1447 # have that kind of IP addresses, throw an exception
1448 if spice_ip_version != constants.IFACE_NO_IP_VERSION_SPECIFIED:
1449 if not addresses[spice_ip_version]:
1450 raise errors.HypervisorError("SPICE: Unable to get an IPv%s address"
1451 " for %s" % (spice_ip_version,
1454 # the user did not specify an IP version, we have to figure it out
1455 elif (addresses[constants.IP4_VERSION] and
1456 addresses[constants.IP6_VERSION]):
1457 # we have both ipv4 and ipv6, let's use the cluster default IP
1459 cluster_family = ssconf.SimpleStore().GetPrimaryIPFamily()
1460 spice_ip_version = \
1461 netutils.IPAddress.GetVersionFromAddressFamily(cluster_family)
1462 elif addresses[constants.IP4_VERSION]:
1463 spice_ip_version = constants.IP4_VERSION
1464 elif addresses[constants.IP6_VERSION]:
1465 spice_ip_version = constants.IP6_VERSION
1467 raise errors.HypervisorError("SPICE: Unable to get an IP address"
1468 " for %s" % (spice_bind))
1470 spice_address = addresses[spice_ip_version][0]
1473 # spice_bind is known to be a valid IP address, because
1474 # ValidateParameters checked it.
1475 spice_address = spice_bind
1477 spice_arg = "addr=%s" % spice_address
1478 if hvp[constants.HV_KVM_SPICE_USE_TLS]:
1479 spice_arg = ("%s,tls-port=%s,x509-cacert-file=%s" %
1480 (spice_arg, instance.network_port,
1481 pathutils.SPICE_CACERT_FILE))
1482 spice_arg = ("%s,x509-key-file=%s,x509-cert-file=%s" %
1483 (spice_arg, pathutils.SPICE_CERT_FILE,
1484 pathutils.SPICE_CERT_FILE))
1485 tls_ciphers = hvp[constants.HV_KVM_SPICE_TLS_CIPHERS]
1487 spice_arg = "%s,tls-ciphers=%s" % (spice_arg, tls_ciphers)
1489 spice_arg = "%s,port=%s" % (spice_arg, instance.network_port)
1491 if not hvp[constants.HV_KVM_SPICE_PASSWORD_FILE]:
1492 spice_arg = "%s,disable-ticketing" % spice_arg
1494 if spice_ip_version:
1495 spice_arg = "%s,ipv%s" % (spice_arg, spice_ip_version)
1497 # Image compression options
1498 img_lossless = hvp[constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR]
1499 img_jpeg = hvp[constants.HV_KVM_SPICE_JPEG_IMG_COMPR]
1500 img_zlib_glz = hvp[constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR]
1502 spice_arg = "%s,image-compression=%s" % (spice_arg, img_lossless)
1504 spice_arg = "%s,jpeg-wan-compression=%s" % (spice_arg, img_jpeg)
1506 spice_arg = "%s,zlib-glz-wan-compression=%s" % (spice_arg, img_zlib_glz)
1508 # Video stream detection
1509 video_streaming = hvp[constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION]
1511 spice_arg = "%s,streaming-video=%s" % (spice_arg, video_streaming)
1513 # Audio compression, by default in qemu-kvm it is on
1514 if not hvp[constants.HV_KVM_SPICE_AUDIO_COMPR]:
1515 spice_arg = "%s,playback-compression=off" % spice_arg
1516 if not hvp[constants.HV_KVM_SPICE_USE_VDAGENT]:
1517 spice_arg = "%s,agent-mouse=off" % spice_arg
1519 # Enable the spice agent communication channel between the host and the
1521 kvm_cmd.extend(["-device", "virtio-serial-pci"])
1524 "virtserialport,chardev=spicechannel0,name=com.redhat.spice.0",
1526 kvm_cmd.extend(["-chardev", "spicevmc,id=spicechannel0,name=vdagent"])
1528 logging.info("KVM: SPICE will listen on port %s", instance.network_port)
1529 kvm_cmd.extend(["-spice", spice_arg])
1532 # From qemu 1.4 -nographic is incompatible with -daemonize. The new way
1533 # also works in earlier versions though (tested with 1.1 and 1.3)
1534 if self._DISPLAY_RE.search(kvmhelp):
1535 kvm_cmd.extend(["-display", "none"])
1537 kvm_cmd.extend(["-nographic"])
1539 if hvp[constants.HV_USE_LOCALTIME]:
1540 kvm_cmd.extend(["-localtime"])
1542 if hvp[constants.HV_KVM_USE_CHROOT]:
1543 kvm_cmd.extend(["-chroot", self._InstanceChrootDir(instance.name)])
1545 # Add qemu-KVM -cpu param
1546 if hvp[constants.HV_CPU_TYPE]:
1547 kvm_cmd.extend(["-cpu", hvp[constants.HV_CPU_TYPE]])
1549 # As requested by music lovers
1550 if hvp[constants.HV_SOUNDHW]:
1551 kvm_cmd.extend(["-soundhw", hvp[constants.HV_SOUNDHW]])
1553 # Pass a -vga option if requested, or if spice is used, for backwards
1555 if hvp[constants.HV_VGA]:
1556 kvm_cmd.extend(["-vga", hvp[constants.HV_VGA]])
1558 kvm_cmd.extend(["-vga", "qxl"])
1560 # Various types of usb devices, comma separated
1561 if hvp[constants.HV_USB_DEVICES]:
1562 for dev in hvp[constants.HV_USB_DEVICES].split(","):
1563 kvm_cmd.extend(["-usbdevice", dev])
1565 if hvp[constants.HV_KVM_EXTRA]:
1566 kvm_cmd.extend(hvp[constants.HV_KVM_EXTRA].split(" "))
1568 pci_reservations = bitarray(self._DEFAULT_PCI_RESERVATIONS)
1570 for disk, link_name in block_devices:
1571 _UpdatePCISlots(disk, pci_reservations)
1572 kvm_disks.append((disk, link_name))
1575 for nic in instance.nics:
1576 _UpdatePCISlots(nic, pci_reservations)
1577 kvm_nics.append(nic)
1581 return (kvm_cmd, kvm_nics, hvparams, kvm_disks)
1583 def _WriteKVMRuntime(self, instance_name, data):
1584 """Write an instance's KVM runtime
1588 utils.WriteFile(self._InstanceKVMRuntime(instance_name),
1590 except EnvironmentError, err:
1591 raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err)
1593 def _ReadKVMRuntime(self, instance_name):
1594 """Read an instance's KVM runtime
1598 file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name))
1599 except EnvironmentError, err:
1600 raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err)
1603 def _SaveKVMRuntime(self, instance, kvm_runtime):
1604 """Save an instance's KVM runtime
1607 kvm_cmd, kvm_nics, hvparams, block_devices = kvm_runtime
1609 serialized_nics = [nic.ToDict() for nic in kvm_nics]
1610 serialized_blockdevs = [(blk.ToDict(), link)
1611 for blk, link in block_devices]
1612 serialized_form = serializer.Dump((kvm_cmd, serialized_nics, hvparams,
1613 serialized_blockdevs))
1615 self._WriteKVMRuntime(instance.name, serialized_form)
1617 def _LoadKVMRuntime(self, instance, serialized_runtime=None):
1618 """Load an instance's KVM runtime
1621 if not serialized_runtime:
1622 serialized_runtime = self._ReadKVMRuntime(instance.name)
1624 return _AnalyzeSerializedRuntime(serialized_runtime)
1626 def _RunKVMCmd(self, name, kvm_cmd, tap_fds=None):
1627 """Run the KVM cmd and check for errors
1630 @param name: instance name
1631 @type kvm_cmd: list of strings
1632 @param kvm_cmd: runcmd input for kvm
1633 @type tap_fds: list of int
1634 @param tap_fds: fds of tap devices opened by Ganeti
1638 result = utils.RunCmd(kvm_cmd, noclose_fds=tap_fds)
1641 utils_wrapper.CloseFdNoError(fd)
1644 raise errors.HypervisorError("Failed to start instance %s: %s (%s)" %
1645 (name, result.fail_reason, result.output))
1646 if not self._InstancePidAlive(name)[2]:
1647 raise errors.HypervisorError("Failed to start instance %s" % name)
1649 # 52/50 local variables
1650 # pylint: disable=R0914
1651 def _ExecuteKVMRuntime(self, instance, kvm_runtime, kvmhelp, incoming=None):
1652 """Execute a KVM cmd, after completing it with some last minute data.
1654 @type incoming: tuple of strings
1655 @param incoming: (target_host_ip, port)
1656 @type kvmhelp: string
1657 @param kvmhelp: output of kvm --help
1660 # Small _ExecuteKVMRuntime hv parameters programming howto:
1661 # - conf_hvp contains the parameters as configured on ganeti. they might
1662 # have changed since the instance started; only use them if the change
1663 # won't affect the inside of the instance (which hasn't been rebooted).
1664 # - up_hvp contains the parameters as they were when the instance was
1665 # started, plus any new parameter which has been added between ganeti
1666 # versions: it is paramount that those default to a value which won't
1667 # affect the inside of the instance as well.
1668 conf_hvp = instance.hvparams
1669 name = instance.name
1670 self._CheckDown(name)
1674 kvm_cmd, kvm_nics, up_hvp, block_devices = kvm_runtime
1675 # the first element of kvm_cmd is always the path to the kvm binary
1676 kvm_path = kvm_cmd[0]
1677 up_hvp = objects.FillDict(conf_hvp, up_hvp)
1679 # We know it's safe to run as a different user upon migration, so we'll use
1680 # the latest conf, from conf_hvp.
1681 security_model = conf_hvp[constants.HV_SECURITY_MODEL]
1682 if security_model == constants.HT_SM_USER:
1683 kvm_cmd.extend(["-runas", conf_hvp[constants.HV_SECURITY_DOMAIN]])
1685 keymap = conf_hvp[constants.HV_KEYMAP]
1687 keymap_path = self._InstanceKeymapFile(name)
1688 # If a keymap file is specified, KVM won't use its internal defaults. By
1689 # first including the "en-us" layout, an error on loading the actual
1690 # layout (e.g. because it can't be found) won't lead to a non-functional
1691 # keyboard. A keyboard with incorrect keys is still better than none.
1692 utils.WriteFile(keymap_path, data="include en-us\ninclude %s\n" % keymap)
1693 kvm_cmd.extend(["-k", keymap_path])
1695 # We have reasons to believe changing something like the nic driver/type
1696 # upon migration won't exactly fly with the instance kernel, so for nic
1697 # related parameters we'll use up_hvp
1701 kvm_cmd.extend(["-net", "none"])
1705 nic_type = up_hvp[constants.HV_NIC_TYPE]
1706 if nic_type == constants.HT_NIC_PARAVIRTUAL:
1707 nic_model = self._VIRTIO
1709 devlist = self._GetKVMOutput(kvm_path, self._KVMOPT_DEVICELIST)
1710 if self._VIRTIO_NET_RE.search(devlist):
1711 nic_model = self._VIRTIO_NET_PCI
1712 vnet_hdr = up_hvp[constants.HV_VNET_HDR]
1713 except errors.HypervisorError, _:
1714 # Older versions of kvm don't support DEVICE_LIST, but they don't
1715 # have new virtio syntax either.
1718 if up_hvp[constants.HV_VHOST_NET]:
1719 # check for vhost_net support
1720 if self._VHOST_RE.search(kvmhelp):
1721 tap_extra = ",vhost=on"
1723 raise errors.HypervisorError("vhost_net is configured"
1724 " but it is not available")
1726 nic_model = nic_type
1728 kvm_supports_netdev = self._NETDEV_RE.search(kvmhelp)
1730 for nic_seq, nic in enumerate(kvm_nics):
1731 tapname, tapfd = _OpenTap(vnet_hdr=vnet_hdr)
1732 tapfds.append(tapfd)
1733 taps.append(tapname)
1734 if kvm_supports_netdev:
1735 nic_val = "%s,mac=%s" % (nic_model, nic.mac)
1737 # kvm_nics already exist in old runtime files and thus there might
1738 # be some entries without pci slot (therefore try: except:)
1739 kvm_devid = _GenerateDeviceKVMId(constants.HOTPLUG_TARGET_NIC, nic)
1741 nic_val += (",id=%s,bus=pci.0,addr=%s" % (kvm_devid, hex(nic.pci)))
1742 except errors.HotplugError:
1743 netdev = "netdev%d" % nic_seq
1744 nic_val += (",netdev=%s" % netdev)
1745 tap_val = ("type=tap,id=%s,fd=%d%s" %
1746 (netdev, tapfd, tap_extra))
1747 kvm_cmd.extend(["-netdev", tap_val, "-device", nic_val])
1749 nic_val = "nic,vlan=%s,macaddr=%s,model=%s" % (nic_seq,
1751 tap_val = "tap,vlan=%s,fd=%d" % (nic_seq, tapfd)
1752 kvm_cmd.extend(["-net", tap_val, "-net", nic_val])
1755 target, port = incoming
1756 kvm_cmd.extend(["-incoming", "tcp:%s:%s" % (target, port)])
1758 # Changing the vnc password doesn't bother the guest that much. At most it
1759 # will surprise people who connect to it. Whether positively or negatively
1761 vnc_pwd_file = conf_hvp[constants.HV_VNC_PASSWORD_FILE]
1765 vnc_pwd = utils.ReadOneLineFile(vnc_pwd_file, strict=True)
1766 except EnvironmentError, err:
1767 raise errors.HypervisorError("Failed to open VNC password file %s: %s"
1768 % (vnc_pwd_file, err))
1770 if conf_hvp[constants.HV_KVM_USE_CHROOT]:
1771 utils.EnsureDirs([(self._InstanceChrootDir(name),
1772 constants.SECURE_DIR_MODE)])
1774 # Automatically enable QMP if version is >= 0.14
1775 if self._QMP_RE.search(kvmhelp):
1776 logging.debug("Enabling QMP")
1777 kvm_cmd.extend(["-qmp", "unix:%s,server,nowait" %
1778 self._InstanceQmpMonitor(instance.name)])
1780 # Configure the network now for starting instances and bridged interfaces,
1781 # during FinalizeMigration for incoming instances' routed interfaces
1782 for nic_seq, nic in enumerate(kvm_nics):
1784 nic.nicparams[constants.NIC_MODE] != constants.NIC_MODE_BRIDGED):
1786 self._ConfigureNIC(instance, nic_seq, nic, taps[nic_seq])
1788 bdev_opts = self._GenerateKVMBlockDevicesOptions(instance,
1791 kvm_cmd.extend(bdev_opts)
1792 # CPU affinity requires kvm to start paused, so we set this flag if the
1793 # instance is not already paused and if we are not going to accept a
1794 # migrating instance. In the latter case, pausing is not needed.
1795 start_kvm_paused = not (_KVM_START_PAUSED_FLAG in kvm_cmd) and not incoming
1796 if start_kvm_paused:
1797 kvm_cmd.extend([_KVM_START_PAUSED_FLAG])
1799 # Note: CPU pinning is using up_hvp since changes take effect
1800 # during instance startup anyway, and to avoid problems when soft
1801 # rebooting the instance.
1803 if up_hvp.get(constants.HV_CPU_MASK, None):
1806 if security_model == constants.HT_SM_POOL:
1807 ss = ssconf.SimpleStore()
1808 uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\n")
1809 all_uids = set(uidpool.ExpandUidPool(uid_pool))
1810 uid = uidpool.RequestUnusedUid(all_uids)
1812 username = pwd.getpwuid(uid.GetUid()).pw_name
1813 kvm_cmd.extend(["-runas", username])
1814 self._RunKVMCmd(name, kvm_cmd, tapfds)
1816 uidpool.ReleaseUid(uid)
1820 utils.WriteFile(self._InstanceUidFile(name), data=uid.AsStr())
1822 self._RunKVMCmd(name, kvm_cmd, tapfds)
1824 utils.EnsureDirs([(self._InstanceNICDir(instance.name),
1825 constants.RUN_DIRS_MODE)])
1826 for nic_seq, tap in enumerate(taps):
1827 utils.WriteFile(self._InstanceNICFile(instance.name, nic_seq),
1831 change_cmd = "change vnc password %s" % vnc_pwd
1832 self._CallMonitorCommand(instance.name, change_cmd)
1834 # Setting SPICE password. We are not vulnerable to malicious passwordless
1835 # connection attempts because SPICE by default does not allow connections
1836 # if neither a password nor the "disable_ticketing" options are specified.
1837 # As soon as we send the password via QMP, that password is a valid ticket
1839 spice_password_file = conf_hvp[constants.HV_KVM_SPICE_PASSWORD_FILE]
1840 if spice_password_file:
1843 spice_pwd = utils.ReadOneLineFile(spice_password_file, strict=True)
1844 except EnvironmentError, err:
1845 raise errors.HypervisorError("Failed to open SPICE password file %s: %s"
1846 % (spice_password_file, err))
1848 qmp = QmpConnection(self._InstanceQmpMonitor(instance.name))
1851 "protocol": "spice",
1852 "password": spice_pwd,
1854 qmp.Execute("set_password", arguments)
1856 for filename in temp_files:
1857 utils.RemoveFile(filename)
1859 # If requested, set CPU affinity and resume instance execution
1861 self._ExecuteCpuAffinity(instance.name, up_hvp[constants.HV_CPU_MASK])
1863 start_memory = self._InstanceStartupMemory(instance)
1864 if start_memory < instance.beparams[constants.BE_MAXMEM]:
1865 self.BalloonInstanceMemory(instance, start_memory)
1867 if start_kvm_paused:
1868 # To control CPU pinning, ballooning, and vnc/spice passwords
1869 # the VM was started in a frozen state. If freezing was not
1870 # explicitly requested resume the vm status.
1871 self._CallMonitorCommand(instance.name, self._CONT_CMD)
1873 def StartInstance(self, instance, block_devices, startup_paused):
1874 """Start an instance.
1877 self._CheckDown(instance.name)
1878 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
1879 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
1880 kvm_runtime = self._GenerateKVMRuntime(instance, block_devices,
1881 startup_paused, kvmhelp)
1882 self._SaveKVMRuntime(instance, kvm_runtime)
1883 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp)
1885 def _CallMonitorCommand(self, instance_name, command, timeout=None):
1886 """Invoke a command on the instance monitor.
1889 if timeout is not None:
1890 timeout_cmd = "timeout %s" % (timeout, )
1894 # TODO: Replace monitor calls with QMP once KVM >= 0.14 is the minimum
1895 # version. The monitor protocol is designed for human consumption, whereas
1896 # QMP is made for programmatic usage. In the worst case QMP can also
1897 # execute monitor commands. As it is, all calls to socat take at least
1898 # 500ms and likely more: socat can't detect the end of the reply and waits
1899 # for 500ms of no data received before exiting (500 ms is the default for
1900 # the "-t" parameter).
1901 socat = ("echo %s | %s %s STDIO UNIX-CONNECT:%s" %
1902 (utils.ShellQuote(command),
1904 constants.SOCAT_PATH,
1905 utils.ShellQuote(self._InstanceMonitor(instance_name))))
1907 result = utils.RunCmd(socat)
1909 msg = ("Failed to send command '%s' to instance '%s', reason '%s',"
1911 (command, instance_name, result.fail_reason, result.output))
1912 raise errors.HypervisorError(msg)
1916 def _GetFreePCISlot(self, instance, dev):
1917 """Get the first available pci slot of a runnung instance.
1920 slots = bitarray(32)
1921 slots.setall(False) # pylint: disable=E1101
1922 output = self._CallMonitorCommand(instance.name, self._INFO_PCI_CMD)
1923 for line in output.stdout.splitlines():
1924 match = self._INFO_PCI_RE.search(line)
1926 slot = int(match.group(1))
1929 [free] = slots.search(_AVAILABLE_PCI_SLOT, 1) # pylint: disable=E1101
1931 raise errors.HypervisorError("All PCI slots occupied")
1935 def HotplugSupported(self, instance, action, dev_type):
1936 """Check if hotplug is supported.
1938 Hotplug is *not* supported in case of:
1939 - qemu versions < 1.0
1940 - security models and chroot (disk hotplug)
1941 - fdsend module is missing (nic hot-add)
1943 @raise errors.HypervisorError: in previous cases
1946 output = self._CallMonitorCommand(instance.name, self._INFO_VERSION_CMD)
1947 # TODO: search for netdev_add, drive_add, device_add.....
1948 match = self._INFO_VERSION_RE.search(output.stdout)
1950 raise errors.HotplugError("Try hotplug only in running instances.")
1951 v_major, v_min, _, _ = match.groups()
1952 if (v_major, v_min) < (1, 0):
1953 raise errors.HotplugError("Hotplug not supported for qemu versions < 1.0")
1955 if dev_type == constants.HOTPLUG_TARGET_DISK:
1956 hvp = instance.hvparams
1957 security_model = hvp[constants.HV_SECURITY_MODEL]
1958 use_chroot = hvp[constants.HV_KVM_USE_CHROOT]
1960 raise errors.HotplugError("Disk hotplug is not supported"
1961 " in case of chroot.")
1962 if security_model != constants.HT_SM_NONE:
1963 raise errors.HotplugError("Disk Hotplug is not supported in case"
1964 " security models are used.")
1966 if (dev_type == constants.HOTPLUG_TARGET_NIC and
1967 action == constants.HOTPLUG_ACTION_ADD and not fdsend):
1968 raise errors.HotplugError("Cannot hot-add NIC."
1969 " fdsend python module is missing.")
1972 def _CallHotplugCommand(self, name, cmd):
1973 output = self._CallMonitorCommand(name, cmd)
1974 # TODO: parse output and check if succeeded
1975 for line in output.stdout.splitlines():
1976 logging.info("%s", line)
1979 def _ParseKVMVersion(cls, text):
1980 """Parse the KVM version from the --help output.
1983 @param text: output of kvm --help
1984 @return: (version, v_maj, v_min, v_rev)
1985 @raise errors.HypervisorError: when the KVM version cannot be retrieved
1988 match = cls._VERSION_RE.search(text.splitlines()[0])
1990 raise errors.HypervisorError("Unable to get KVM version")
1992 v_all = match.group(0)
1993 v_maj = int(match.group(1))
1994 v_min = int(match.group(2))
1996 v_rev = int(match.group(4))
1999 return (v_all, v_maj, v_min, v_rev)
2002 def _GetKVMOutput(cls, kvm_path, option):
2003 """Return the output of a kvm invocation
2005 @type kvm_path: string
2006 @param kvm_path: path to the kvm executable
2007 @type option: a key of _KVMOPTS_CMDS
2008 @param option: kvm option to fetch the output from
2009 @return: output a supported kvm invocation
2010 @raise errors.HypervisorError: when the KVM help output cannot be retrieved
2013 assert option in cls._KVMOPTS_CMDS, "Invalid output option"
2015 optlist, can_fail = cls._KVMOPTS_CMDS[option]
2017 result = utils.RunCmd([kvm_path] + optlist)
2018 if result.failed and not can_fail:
2019 raise errors.HypervisorError("Unable to get KVM %s output" %
2021 return result.output
2024 def _GetKVMVersion(cls, kvm_path):
2025 """Return the installed KVM version.
2027 @return: (version, v_maj, v_min, v_rev)
2028 @raise errors.HypervisorError: when the KVM version cannot be retrieved
2031 return cls._ParseKVMVersion(cls._GetKVMOutput(kvm_path, cls._KVMOPT_HELP))
2034 def _GetDefaultMachineVersion(cls, kvm_path):
2035 """Return the default hardware revision (e.g. pc-1.1)
2038 output = cls._GetKVMOutput(kvm_path, cls._KVMOPT_MLIST)
2039 match = cls._DEFAULT_MACHINE_VERSION_RE.search(output)
2041 return match.group(1)
2045 def StopInstance(self, instance, force=False, retry=False, name=None,
2047 """Stop an instance.
2050 assert(timeout is None or force is not None)
2052 if name is not None and not force:
2053 raise errors.HypervisorError("Cannot shutdown cleanly by name only")
2055 name = instance.name
2056 acpi = instance.hvparams[constants.HV_ACPI]
2059 _, pid, alive = self._InstancePidAlive(name)
2060 if pid > 0 and alive:
2061 if force or not acpi:
2062 utils.KillProcess(pid)
2064 self._CallMonitorCommand(name, "system_powerdown", timeout)
2066 def CleanupInstance(self, instance_name):
2067 """Cleanup after a stopped instance
2070 pidfile, pid, alive = self._InstancePidAlive(instance_name)
2071 if pid > 0 and alive:
2072 raise errors.HypervisorError("Cannot cleanup a live instance")
2073 self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
2075 def RebootInstance(self, instance):
2076 """Reboot an instance.
2079 # For some reason if we do a 'send-key ctrl-alt-delete' to the control
2080 # socket the instance will stop, but now power up again. So we'll resort
2081 # to shutdown and restart.
2082 _, _, alive = self._InstancePidAlive(instance.name)
2084 raise errors.HypervisorError("Failed to reboot instance %s:"
2085 " not running" % instance.name)
2086 # StopInstance will delete the saved KVM runtime so:
2087 # ...first load it...
2088 kvm_runtime = self._LoadKVMRuntime(instance)
2089 # ...now we can safely call StopInstance...
2090 if not self.StopInstance(instance):
2091 self.StopInstance(instance, force=True)
2092 # ...and finally we can save it again, and execute it...
2093 self._SaveKVMRuntime(instance, kvm_runtime)
2094 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
2095 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
2096 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp)
2098 def MigrationInfo(self, instance):
2099 """Get instance information to perform a migration.
2101 @type instance: L{objects.Instance}
2102 @param instance: instance to be migrated
2104 @return: content of the KVM runtime file
2107 return self._ReadKVMRuntime(instance.name)
2109 def AcceptInstance(self, instance, info, target):
2110 """Prepare to accept an instance.
2112 @type instance: L{objects.Instance}
2113 @param instance: instance to be accepted
2115 @param info: content of the KVM runtime file on the source node
2116 @type target: string
2117 @param target: target host (usually ip), on this node
2120 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
2121 incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT])
2122 kvmpath = instance.hvparams[constants.HV_KVM_PATH]
2123 kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP)
2124 self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp,
2125 incoming=incoming_address)
2127 def FinalizeMigrationDst(self, instance, info, success):
2128 """Finalize the instance migration on the target node.
2130 Stop the incoming mode KVM.
2132 @type instance: L{objects.Instance}
2133 @param instance: instance whose migration is being finalized
2137 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
2138 kvm_nics = kvm_runtime[1]
2140 for nic_seq, nic in enumerate(kvm_nics):
2141 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2142 # Bridged interfaces have already been configured
2145 tap = utils.ReadFile(self._InstanceNICFile(instance.name, nic_seq))
2146 except EnvironmentError, err:
2147 logging.warning("Failed to find host interface for %s NIC #%d: %s",
2148 instance.name, nic_seq, str(err))
2151 self._ConfigureNIC(instance, nic_seq, nic, tap)
2152 except errors.HypervisorError, err:
2153 logging.warning(str(err))
2155 self._WriteKVMRuntime(instance.name, info)
2157 self.StopInstance(instance, force=True)
2159 def MigrateInstance(self, instance, target, live):
2160 """Migrate an instance to a target node.
2162 The migration will not be attempted if the instance is not
2165 @type instance: L{objects.Instance}
2166 @param instance: the instance to be migrated
2167 @type target: string
2168 @param target: ip address of the target node
2170 @param live: perform a live migration
2173 instance_name = instance.name
2174 port = instance.hvparams[constants.HV_MIGRATION_PORT]
2175 _, _, alive = self._InstancePidAlive(instance_name)
2177 raise errors.HypervisorError("Instance not running, cannot migrate")
2180 self._CallMonitorCommand(instance_name, "stop")
2182 migrate_command = ("migrate_set_speed %dm" %
2183 instance.hvparams[constants.HV_MIGRATION_BANDWIDTH])
2184 self._CallMonitorCommand(instance_name, migrate_command)
2186 migrate_command = ("migrate_set_downtime %dms" %
2187 instance.hvparams[constants.HV_MIGRATION_DOWNTIME])
2188 self._CallMonitorCommand(instance_name, migrate_command)
2190 migrate_command = "migrate -d tcp:%s:%s" % (target, port)
2191 self._CallMonitorCommand(instance_name, migrate_command)
2193 def FinalizeMigrationSource(self, instance, success, live):
2194 """Finalize the instance migration on the source node.
2196 @type instance: L{objects.Instance}
2197 @param instance: the instance that was migrated
2199 @param success: whether the migration succeeded or not
2201 @param live: whether the user requested a live migration or not
2205 pidfile, pid, _ = self._InstancePidAlive(instance.name)
2206 utils.KillProcess(pid)
2207 self._RemoveInstanceRuntimeFiles(pidfile, instance.name)
2209 self._CallMonitorCommand(instance.name, self._CONT_CMD)
2211 def GetMigrationStatus(self, instance):
2212 """Get the migration status
2214 @type instance: L{objects.Instance}
2215 @param instance: the instance that is being migrated
2216 @rtype: L{objects.MigrationStatus}
2217 @return: the status of the current migration (one of
2218 L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional
2219 progress info that can be retrieved from the hypervisor
2222 info_command = "info migrate"
2223 for _ in range(self._MIGRATION_INFO_MAX_BAD_ANSWERS):
2224 result = self._CallMonitorCommand(instance.name, info_command)
2225 match = self._MIGRATION_STATUS_RE.search(result.stdout)
2227 if not result.stdout:
2228 logging.info("KVM: empty 'info migrate' result")
2230 logging.warning("KVM: unknown 'info migrate' result: %s",
2233 status = match.group(1)
2234 if status in constants.HV_KVM_MIGRATION_VALID_STATUSES:
2235 migration_status = objects.MigrationStatus(status=status)
2236 match = self._MIGRATION_PROGRESS_RE.search(result.stdout)
2238 migration_status.transferred_ram = match.group("transferred")
2239 migration_status.total_ram = match.group("total")
2241 return migration_status
2243 logging.warning("KVM: unknown migration status '%s'", status)
2245 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
2247 return objects.MigrationStatus(status=constants.HV_MIGRATION_FAILED)
2249 def BalloonInstanceMemory(self, instance, mem):
2250 """Balloon an instance memory to a certain value.
2252 @type instance: L{objects.Instance}
2253 @param instance: instance to be accepted
2255 @param mem: actual memory size to use for instance runtime
2258 self._CallMonitorCommand(instance.name, "balloon %d" % mem)
2260 def GetNodeInfo(self):
2261 """Return information about the node.
2263 @return: a dict with the following keys (values in MiB):
2264 - memory_total: the total memory size on the node
2265 - memory_free: the available memory on the node for instances
2266 - memory_dom0: the memory used by the node itself, if available
2267 - hv_version: the hypervisor version in the form (major, minor,
2271 result = self.GetLinuxNodeInfo()
2272 # FIXME: this is the global kvm version, but the actual version can be
2273 # customized as an hv parameter. we should use the nodegroup's default kvm
2274 # path parameter here.
2275 _, v_major, v_min, v_rev = self._GetKVMVersion(constants.KVM_PATH)
2276 result[constants.HV_NODEINFO_KEY_VERSION] = (v_major, v_min, v_rev)
2280 def GetInstanceConsole(cls, instance, hvparams, beparams):
2281 """Return a command for connecting to the console of an instance.
2284 if hvparams[constants.HV_SERIAL_CONSOLE]:
2285 cmd = [pathutils.KVM_CONSOLE_WRAPPER,
2286 constants.SOCAT_PATH, utils.ShellQuote(instance.name),
2287 utils.ShellQuote(cls._InstanceMonitor(instance.name)),
2288 "STDIO,%s" % cls._SocatUnixConsoleParams(),
2289 "UNIX-CONNECT:%s" % cls._InstanceSerial(instance.name)]
2290 return objects.InstanceConsole(instance=instance.name,
2291 kind=constants.CONS_SSH,
2292 host=instance.primary_node,
2293 user=constants.SSH_CONSOLE_USER,
2296 vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS]
2297 if vnc_bind_address and instance.network_port > constants.VNC_BASE_PORT:
2298 display = instance.network_port - constants.VNC_BASE_PORT
2299 return objects.InstanceConsole(instance=instance.name,
2300 kind=constants.CONS_VNC,
2301 host=vnc_bind_address,
2302 port=instance.network_port,
2305 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2307 return objects.InstanceConsole(instance=instance.name,
2308 kind=constants.CONS_SPICE,
2310 port=instance.network_port)
2312 return objects.InstanceConsole(instance=instance.name,
2313 kind=constants.CONS_MESSAGE,
2314 message=("No serial shell for instance %s" %
2318 """Verify the hypervisor.
2320 Check that the required binaries exist.
2322 @return: Problem description if something is wrong, C{None} otherwise
2326 # FIXME: this is the global kvm binary, but the actual path can be
2327 # customized as an hv parameter; we should use the nodegroup's
2328 # default kvm path parameter here.
2329 if not os.path.exists(constants.KVM_PATH):
2330 msgs.append("The KVM binary ('%s') does not exist" % constants.KVM_PATH)
2331 if not os.path.exists(constants.SOCAT_PATH):
2332 msgs.append("The socat binary ('%s') does not exist" %
2333 constants.SOCAT_PATH)
2335 return self._FormatVerifyResults(msgs)
2338 def CheckParameterSyntax(cls, hvparams):
2339 """Check the given parameters for validity.
2341 @type hvparams: dict
2342 @param hvparams: dictionary with parameter names/value
2343 @raise errors.HypervisorError: when a parameter is not valid
2346 super(KVMHypervisor, cls).CheckParameterSyntax(hvparams)
2348 kernel_path = hvparams[constants.HV_KERNEL_PATH]
2350 if not hvparams[constants.HV_ROOT_PATH]:
2351 raise errors.HypervisorError("Need a root partition for the instance,"
2352 " if a kernel is defined")
2354 if (hvparams[constants.HV_VNC_X509_VERIFY] and
2355 not hvparams[constants.HV_VNC_X509]):
2356 raise errors.HypervisorError("%s must be defined, if %s is" %
2357 (constants.HV_VNC_X509,
2358 constants.HV_VNC_X509_VERIFY))
2360 if hvparams[constants.HV_SERIAL_CONSOLE]:
2361 serial_speed = hvparams[constants.HV_SERIAL_SPEED]
2362 valid_speeds = constants.VALID_SERIAL_SPEEDS
2363 if not serial_speed or serial_speed not in valid_speeds:
2364 raise errors.HypervisorError("Invalid serial console speed, must be"
2366 utils.CommaJoin(valid_speeds))
2368 boot_order = hvparams[constants.HV_BOOT_ORDER]
2369 if (boot_order == constants.HT_BO_CDROM and
2370 not hvparams[constants.HV_CDROM_IMAGE_PATH]):
2371 raise errors.HypervisorError("Cannot boot from cdrom without an"
2374 security_model = hvparams[constants.HV_SECURITY_MODEL]
2375 if security_model == constants.HT_SM_USER:
2376 if not hvparams[constants.HV_SECURITY_DOMAIN]:
2377 raise errors.HypervisorError("A security domain (user to run kvm as)"
2378 " must be specified")
2379 elif (security_model == constants.HT_SM_NONE or
2380 security_model == constants.HT_SM_POOL):
2381 if hvparams[constants.HV_SECURITY_DOMAIN]:
2382 raise errors.HypervisorError("Cannot have a security domain when the"
2383 " security model is 'none' or 'pool'")
2385 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2386 spice_ip_version = hvparams[constants.HV_KVM_SPICE_IP_VERSION]
2388 if spice_ip_version != constants.IFACE_NO_IP_VERSION_SPECIFIED:
2389 # if an IP version is specified, the spice_bind parameter must be an
2391 if (netutils.IP4Address.IsValid(spice_bind) and
2392 spice_ip_version != constants.IP4_VERSION):
2393 raise errors.HypervisorError("SPICE: Got an IPv4 address (%s), but"
2394 " the specified IP version is %s" %
2395 (spice_bind, spice_ip_version))
2397 if (netutils.IP6Address.IsValid(spice_bind) and
2398 spice_ip_version != constants.IP6_VERSION):
2399 raise errors.HypervisorError("SPICE: Got an IPv6 address (%s), but"
2400 " the specified IP version is %s" %
2401 (spice_bind, spice_ip_version))
2403 # All the other SPICE parameters depend on spice_bind being set. Raise an
2404 # error if any of them is set without it.
2405 for param in _SPICE_ADDITIONAL_PARAMS:
2407 raise errors.HypervisorError("SPICE: %s requires %s to be set" %
2408 (param, constants.HV_KVM_SPICE_BIND))
2411 def ValidateParameters(cls, hvparams):
2412 """Check the given parameters for validity.
2414 @type hvparams: dict
2415 @param hvparams: dictionary with parameter names/value
2416 @raise errors.HypervisorError: when a parameter is not valid
2419 super(KVMHypervisor, cls).ValidateParameters(hvparams)
2421 kvm_path = hvparams[constants.HV_KVM_PATH]
2423 security_model = hvparams[constants.HV_SECURITY_MODEL]
2424 if security_model == constants.HT_SM_USER:
2425 username = hvparams[constants.HV_SECURITY_DOMAIN]
2427 pwd.getpwnam(username)
2429 raise errors.HypervisorError("Unknown security domain user %s"
2432 spice_bind = hvparams[constants.HV_KVM_SPICE_BIND]
2434 # only one of VNC and SPICE can be used currently.
2435 if hvparams[constants.HV_VNC_BIND_ADDRESS]:
2436 raise errors.HypervisorError("Both SPICE and VNC are configured, but"
2437 " only one of them can be used at a"
2440 # check that KVM supports SPICE
2441 kvmhelp = cls._GetKVMOutput(kvm_path, cls._KVMOPT_HELP)
2442 if not cls._SPICE_RE.search(kvmhelp):
2443 raise errors.HypervisorError("SPICE is configured, but it is not"
2444 " supported according to 'kvm --help'")
2446 # if spice_bind is not an IP address, it must be a valid interface
2447 bound_to_addr = (netutils.IP4Address.IsValid(spice_bind) or
2448 netutils.IP6Address.IsValid(spice_bind))
2449 if not bound_to_addr and not netutils.IsValidInterface(spice_bind):
2450 raise errors.HypervisorError("SPICE: The %s parameter must be either"
2451 " a valid IP address or interface name" %
2452 constants.HV_KVM_SPICE_BIND)
2454 machine_version = hvparams[constants.HV_KVM_MACHINE_VERSION]
2456 output = cls._GetKVMOutput(kvm_path, cls._KVMOPT_MLIST)
2457 if not cls._CHECK_MACHINE_VERSION_RE(machine_version).search(output):
2458 raise errors.HypervisorError("Unsupported machine version: %s" %
2462 def PowercycleNode(cls):
2463 """KVM powercycle, just a wrapper over Linux powercycle.
2466 cls.LinuxPowercycle()