4 # Copyright (C) 2008, 2009, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
38 from ganeti import utils
39 from ganeti import constants
40 from ganeti import errors
41 from ganeti import serializer
42 from ganeti import objects
43 from ganeti import uidpool
44 from ganeti import ssconf
45 from ganeti.hypervisor import hv_base
46 from ganeti import netutils
47 from ganeti.utils import wrapper as utils_wrapper
50 _KVM_NETWORK_SCRIPT = constants.SYSCONFDIR + "/ganeti/kvm-vif-bridge"
52 # TUN/TAP driver constants, taken from <linux/if_tun.h>
53 # They are architecture-independent and already hardcoded in qemu-kvm source,
54 # so we can safely include them here.
55 TUNSETIFF = 0x400454ca
56 TUNGETIFF = 0x800454d2
57 TUNGETFEATURES = 0x800454cf
63 def _ProbeTapVnetHdr(fd):
64 """Check whether to enable the IFF_VNET_HDR flag.
66 To do this, _all_ of the following conditions must be met:
67 1. TUNGETFEATURES ioctl() *must* be implemented
68 2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag
69 3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in
70 drivers/net/tun.c there is no way to test this until after the tap device
71 has been created using TUNSETIFF, and there is no way to change the
72 IFF_VNET_HDR flag after creating the interface, catch-22! However both
73 TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27,
74 thus we can expect TUNGETIFF to be present if TUNGETFEATURES is.
77 @param fd: the file descriptor of /dev/net/tun
80 req = struct.pack("I", 0)
82 res = fcntl.ioctl(fd, TUNGETFEATURES, req)
83 except EnvironmentError:
84 logging.warning("TUNGETFEATURES ioctl() not implemented")
87 tunflags = struct.unpack("I", res)[0]
88 if tunflags & IFF_VNET_HDR:
91 logging.warning("Host does not support IFF_VNET_HDR, not enabling")
95 def _OpenTap(vnet_hdr=True):
96 """Open a new tap device and return its file descriptor.
98 This is intended to be used by a qemu-type hypervisor together with the -net
99 tap,fd=<fd> command line parameter.
101 @type vnet_hdr: boolean
102 @param vnet_hdr: Enable the VNET Header
103 @return: (ifname, tapfd)
108 tapfd = os.open("/dev/net/tun", os.O_RDWR)
109 except EnvironmentError:
110 raise errors.HypervisorError("Failed to open /dev/net/tun")
112 flags = IFF_TAP | IFF_NO_PI
114 if vnet_hdr and _ProbeTapVnetHdr(tapfd):
115 flags |= IFF_VNET_HDR
117 # The struct ifreq ioctl request (see netdevice(7))
118 ifr = struct.pack("16sh", "", flags)
121 res = fcntl.ioctl(tapfd, TUNSETIFF, ifr)
122 except EnvironmentError:
123 raise errors.HypervisorError("Failed to allocate a new TAP device")
125 # Get the interface name from the ioctl
126 ifname = struct.unpack("16sh", res)[0].strip("\x00")
127 return (ifname, tapfd)
130 class KVMHypervisor(hv_base.BaseHypervisor):
131 """KVM hypervisor interface"""
134 _ROOT_DIR = constants.RUN_GANETI_DIR + "/kvm-hypervisor"
135 _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids
136 _UIDS_DIR = _ROOT_DIR + "/uid" # contains instances reserved uids
137 _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets
138 _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data
139 _NICS_DIR = _ROOT_DIR + "/nic" # contains instances nic <-> tap associations
140 # KVM instances with chroot enabled are started in empty chroot directories.
141 _CHROOT_DIR = _ROOT_DIR + "/chroot" # for empty chroot directories
142 # After an instance is stopped, its chroot directory is removed.
143 # If the chroot directory is not empty, it can't be removed.
144 # A non-empty chroot directory indicates a possible security incident.
145 # To support forensics, the non-empty chroot directory is quarantined in
146 # a separate directory, called 'chroot-quarantine'.
147 _CHROOT_QUARANTINE_DIR = _ROOT_DIR + "/chroot-quarantine"
148 _DIRS = [_ROOT_DIR, _PIDS_DIR, _UIDS_DIR, _CTRL_DIR, _CONF_DIR, _NICS_DIR,
149 _CHROOT_DIR, _CHROOT_QUARANTINE_DIR]
152 constants.HV_KERNEL_PATH: hv_base.OPT_FILE_CHECK,
153 constants.HV_INITRD_PATH: hv_base.OPT_FILE_CHECK,
154 constants.HV_ROOT_PATH: hv_base.NO_CHECK,
155 constants.HV_KERNEL_ARGS: hv_base.NO_CHECK,
156 constants.HV_ACPI: hv_base.NO_CHECK,
157 constants.HV_SERIAL_CONSOLE: hv_base.NO_CHECK,
158 constants.HV_VNC_BIND_ADDRESS:
159 (False, lambda x: (netutils.IP4Address.IsValid(x) or
160 utils.IsNormAbsPath(x)),
161 "the VNC bind address must be either a valid IP address or an absolute"
162 " pathname", None, None),
163 constants.HV_VNC_TLS: hv_base.NO_CHECK,
164 constants.HV_VNC_X509: hv_base.OPT_DIR_CHECK,
165 constants.HV_VNC_X509_VERIFY: hv_base.NO_CHECK,
166 constants.HV_VNC_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
167 constants.HV_KVM_FLOPPY_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
168 constants.HV_CDROM_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
169 constants.HV_KVM_CDROM2_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
170 constants.HV_BOOT_ORDER:
171 hv_base.ParamInSet(True, constants.HT_KVM_VALID_BO_TYPES),
172 constants.HV_NIC_TYPE:
173 hv_base.ParamInSet(True, constants.HT_KVM_VALID_NIC_TYPES),
174 constants.HV_DISK_TYPE:
175 hv_base.ParamInSet(True, constants.HT_KVM_VALID_DISK_TYPES),
176 constants.HV_KVM_CDROM_DISK_TYPE:
177 hv_base.ParamInSet(False, constants.HT_KVM_VALID_DISK_TYPES),
178 constants.HV_USB_MOUSE:
179 hv_base.ParamInSet(False, constants.HT_KVM_VALID_MOUSE_TYPES),
180 constants.HV_MIGRATION_PORT: hv_base.NET_PORT_CHECK,
181 constants.HV_MIGRATION_BANDWIDTH: hv_base.NO_CHECK,
182 constants.HV_MIGRATION_DOWNTIME: hv_base.NO_CHECK,
183 constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK,
184 constants.HV_USE_LOCALTIME: hv_base.NO_CHECK,
185 constants.HV_DISK_CACHE:
186 hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES),
187 constants.HV_SECURITY_MODEL:
188 hv_base.ParamInSet(True, constants.HT_KVM_VALID_SM_TYPES),
189 constants.HV_SECURITY_DOMAIN: hv_base.NO_CHECK,
190 constants.HV_KVM_FLAG:
191 hv_base.ParamInSet(False, constants.HT_KVM_FLAG_VALUES),
192 constants.HV_VHOST_NET: hv_base.NO_CHECK,
193 constants.HV_KVM_USE_CHROOT: hv_base.NO_CHECK,
194 constants.HV_MEM_PATH: hv_base.OPT_DIR_CHECK,
197 _MIGRATION_STATUS_RE = re.compile('Migration\s+status:\s+(\w+)',
199 _MIGRATION_INFO_MAX_BAD_ANSWERS = 5
200 _MIGRATION_INFO_RETRY_DELAY = 2
202 _VERSION_RE = re.compile(r"\b(\d+)\.(\d+)\.(\d+)\b")
209 hv_base.BaseHypervisor.__init__(self)
210 # Let's make sure the directories we need exist, even if the RUN_DIR lives
211 # in a tmpfs filesystem or has been otherwise wiped out.
212 dirs = [(dname, constants.RUN_DIRS_MODE) for dname in self._DIRS]
213 utils.EnsureDirs(dirs)
216 def _InstancePidFile(cls, instance_name):
217 """Returns the instance pidfile.
220 return utils.PathJoin(cls._PIDS_DIR, instance_name)
223 def _InstanceUidFile(cls, instance_name):
224 """Returns the instance uidfile.
227 return utils.PathJoin(cls._UIDS_DIR, instance_name)
230 def _InstancePidInfo(cls, pid):
231 """Check pid file for instance information.
233 Check that a pid file is associated with an instance, and retrieve
234 information from its command line.
236 @type pid: string or int
237 @param pid: process id of the instance to check
239 @return: (instance_name, memory, vcpus)
240 @raise errors.HypervisorError: when an instance cannot be found
243 alive = utils.IsProcessAlive(pid)
245 raise errors.HypervisorError("Cannot get info for pid %s" % pid)
247 cmdline_file = utils.PathJoin("/proc", str(pid), "cmdline")
249 cmdline = utils.ReadFile(cmdline_file)
250 except EnvironmentError, err:
251 raise errors.HypervisorError("Can't open cmdline file for pid %s: %s" %
258 arg_list = cmdline.split('\x00')
260 arg = arg_list.pop(0)
262 instance = arg_list.pop(0)
264 memory = int(arg_list.pop(0))
266 vcpus = int(arg_list.pop(0))
269 raise errors.HypervisorError("Pid %s doesn't contain a ganeti kvm"
272 return (instance, memory, vcpus)
274 def _InstancePidAlive(self, instance_name):
275 """Returns the instance pidfile, pid, and liveness.
277 @type instance_name: string
278 @param instance_name: instance name
280 @return: (pid file name, pid, liveness)
283 pidfile = self._InstancePidFile(instance_name)
284 pid = utils.ReadPidFile(pidfile)
288 cmd_instance = self._InstancePidInfo(pid)[0]
289 alive = (cmd_instance == instance_name)
290 except errors.HypervisorError:
293 return (pidfile, pid, alive)
295 def _CheckDown(self, instance_name):
296 """Raises an error unless the given instance is down.
299 alive = self._InstancePidAlive(instance_name)[2]
301 raise errors.HypervisorError("Failed to start instance %s: %s" %
302 (instance_name, "already running"))
305 def _InstanceMonitor(cls, instance_name):
306 """Returns the instance monitor socket name
309 return utils.PathJoin(cls._CTRL_DIR, "%s.monitor" % instance_name)
312 def _InstanceSerial(cls, instance_name):
313 """Returns the instance serial socket name
316 return utils.PathJoin(cls._CTRL_DIR, "%s.serial" % instance_name)
319 def _SocatUnixConsoleParams():
320 """Returns the correct parameters for socat
322 If we have a new-enough socat we can use raw mode with an escape character.
325 if constants.SOCAT_USE_ESCAPE:
326 return "raw,echo=0,escape=%s" % constants.SOCAT_ESCAPE_CODE
328 return "echo=0,icanon=0"
331 def _InstanceKVMRuntime(cls, instance_name):
332 """Returns the instance KVM runtime filename
335 return utils.PathJoin(cls._CONF_DIR, "%s.runtime" % instance_name)
338 def _InstanceChrootDir(cls, instance_name):
339 """Returns the name of the KVM chroot dir of the instance
342 return utils.PathJoin(cls._CHROOT_DIR, instance_name)
345 def _InstanceNICDir(cls, instance_name):
346 """Returns the name of the directory holding the tap device files for a
350 return utils.PathJoin(cls._NICS_DIR, instance_name)
353 def _InstanceNICFile(cls, instance_name, seq):
354 """Returns the name of the file containing the tap device for a given NIC
357 return utils.PathJoin(cls._InstanceNICDir(instance_name), str(seq))
360 def _TryReadUidFile(cls, uid_file):
361 """Try to read a uid file
364 if os.path.exists(uid_file):
366 uid = int(utils.ReadOneLineFile(uid_file))
368 except EnvironmentError:
369 logging.warning("Can't read uid file", exc_info=True)
370 except (TypeError, ValueError):
371 logging.warning("Can't parse uid file contents", exc_info=True)
375 def _RemoveInstanceRuntimeFiles(cls, pidfile, instance_name):
376 """Removes an instance's rutime sockets/files/dirs.
379 utils.RemoveFile(pidfile)
380 utils.RemoveFile(cls._InstanceMonitor(instance_name))
381 utils.RemoveFile(cls._InstanceSerial(instance_name))
382 utils.RemoveFile(cls._InstanceKVMRuntime(instance_name))
383 uid_file = cls._InstanceUidFile(instance_name)
384 uid = cls._TryReadUidFile(uid_file)
385 utils.RemoveFile(uid_file)
387 uidpool.ReleaseUid(uid)
389 shutil.rmtree(cls._InstanceNICDir(instance_name))
391 if err.errno != errno.ENOENT:
394 chroot_dir = cls._InstanceChrootDir(instance_name)
395 utils.RemoveDir(chroot_dir)
397 if err.errno == errno.ENOTEMPTY:
398 # The chroot directory is expected to be empty, but it isn't.
399 new_chroot_dir = tempfile.mkdtemp(dir=cls._CHROOT_QUARANTINE_DIR,
402 utils.TimestampForFilename()))
403 logging.warning("The chroot directory of instance %s can not be"
404 " removed as it is not empty. Moving it to the"
405 " quarantine instead. Please investigate the"
406 " contents (%s) and clean up manually",
407 instance_name, new_chroot_dir)
408 utils.RenameFile(chroot_dir, new_chroot_dir)
413 def _ConfigureNIC(instance, seq, nic, tap):
414 """Run the network configuration script for a specified NIC
416 @param instance: instance we're acting on
417 @type instance: instance object
418 @param seq: nic sequence number
420 @param nic: nic we're acting on
421 @type nic: nic object
422 @param tap: the host's tap interface this NIC corresponds to
428 tags = " ".join(instance.tags)
433 "PATH": "%s:/sbin:/usr/sbin" % os.environ["PATH"],
434 "INSTANCE": instance.name,
436 "MODE": nic.nicparams[constants.NIC_MODE],
438 "INTERFACE_INDEX": str(seq),
445 if nic.nicparams[constants.NIC_LINK]:
446 env["LINK"] = nic.nicparams[constants.NIC_LINK]
448 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
449 env["BRIDGE"] = nic.nicparams[constants.NIC_LINK]
451 result = utils.RunCmd([constants.KVM_IFUP, tap], env=env)
453 raise errors.HypervisorError("Failed to configure interface %s: %s."
454 " Network configuration script output: %s" %
455 (tap, result.fail_reason, result.output))
457 def ListInstances(self):
458 """Get the list of running instances.
460 We can do this by listing our live instances directory and
461 checking whether the associated kvm process is still alive.
465 for name in os.listdir(self._PIDS_DIR):
466 if self._InstancePidAlive(name)[2]:
470 def GetInstanceInfo(self, instance_name):
471 """Get instance properties.
473 @type instance_name: string
474 @param instance_name: the instance name
475 @rtype: tuple of strings
476 @return: (name, id, memory, vcpus, stat, times)
479 _, pid, alive = self._InstancePidAlive(instance_name)
483 _, memory, vcpus = self._InstancePidInfo(pid)
487 return (instance_name, pid, memory, vcpus, stat, times)
489 def GetAllInstancesInfo(self):
490 """Get properties of all instances.
492 @return: list of tuples (name, id, memory, vcpus, stat, times)
496 for name in os.listdir(self._PIDS_DIR):
498 info = self.GetInstanceInfo(name)
499 except errors.HypervisorError:
505 def _GenerateKVMRuntime(self, instance, block_devices, startup_paused):
506 """Generate KVM information to start an instance.
509 kvm_version = self._GetKVMVersion()
511 _, v_major, v_min, _ = kvm_version
513 raise errors.HypervisorError("Unable to get KVM version")
515 pidfile = self._InstancePidFile(instance.name)
516 kvm = constants.KVM_PATH
518 # used just by the vnc server, if enabled
519 kvm_cmd.extend(['-name', instance.name])
520 kvm_cmd.extend(['-m', instance.beparams[constants.BE_MEMORY]])
521 kvm_cmd.extend(['-smp', instance.beparams[constants.BE_VCPUS]])
522 kvm_cmd.extend(['-pidfile', pidfile])
523 kvm_cmd.extend(['-daemonize'])
524 if not instance.hvparams[constants.HV_ACPI]:
525 kvm_cmd.extend(['-no-acpi'])
527 kvm_cmd.extend(['-S'])
529 hvp = instance.hvparams
530 boot_disk = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
531 boot_cdrom = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_CDROM
532 boot_floppy = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_FLOPPY
533 boot_network = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_NETWORK
535 if hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED:
536 kvm_cmd.extend(["-enable-kvm"])
537 elif hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_DISABLED:
538 kvm_cmd.extend(["-disable-kvm"])
541 kvm_cmd.extend(['-boot', 'n'])
543 disk_type = hvp[constants.HV_DISK_TYPE]
544 if disk_type == constants.HT_DISK_PARAVIRTUAL:
545 if_val = ',if=virtio'
547 if_val = ',if=%s' % disk_type
549 disk_cache = hvp[constants.HV_DISK_CACHE]
550 if instance.disk_template in constants.DTS_EXT_MIRROR:
551 if disk_cache != "none":
552 # TODO: make this a hard error, instead of a silent overwrite
553 logging.warning("KVM: overriding disk_cache setting '%s' with 'none'"
554 " to prevent shared storage corruption on migration",
556 cache_val = ",cache=none"
557 elif disk_cache != constants.HT_CACHE_DEFAULT:
558 cache_val = ",cache=%s" % disk_cache
561 for cfdev, dev_path in block_devices:
562 if cfdev.mode != constants.DISK_RDWR:
563 raise errors.HypervisorError("Instance has read-only disks which"
564 " are not supported by KVM")
565 # TODO: handle FD_LOOP and FD_BLKTAP (?)
568 kvm_cmd.extend(['-boot', 'c'])
570 if (v_major, v_min) < (0, 14) and disk_type != constants.HT_DISK_IDE:
571 boot_val = ",boot=on"
573 drive_val = 'file=%s,format=raw%s%s%s' % (dev_path, if_val, boot_val,
575 kvm_cmd.extend(['-drive', drive_val])
577 #Now we can specify a different device type for CDROM devices.
578 cdrom_disk_type = hvp[constants.HV_KVM_CDROM_DISK_TYPE]
579 if not cdrom_disk_type:
580 cdrom_disk_type = disk_type
582 iso_image = hvp[constants.HV_CDROM_IMAGE_PATH]
584 options = ',format=raw,media=cdrom'
586 kvm_cmd.extend(['-boot', 'd'])
587 if cdrom_disk_type != constants.HT_DISK_IDE:
588 options = '%s,boot=on,if=%s' % (options, constants.HT_DISK_IDE)
590 options = '%s,boot=on' % options
592 if cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
593 if_val = ',if=virtio'
595 if_val = ',if=%s' % cdrom_disk_type
596 options = '%s%s' % (options, if_val)
597 drive_val = 'file=%s%s' % (iso_image, options)
598 kvm_cmd.extend(['-drive', drive_val])
600 iso_image2 = hvp[constants.HV_KVM_CDROM2_IMAGE_PATH]
602 options = ',format=raw,media=cdrom'
603 if cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
604 if_val = ',if=virtio'
606 if_val = ',if=%s' % cdrom_disk_type
607 options = '%s%s' % (options, if_val)
608 drive_val = 'file=%s%s' % (iso_image2, options)
609 kvm_cmd.extend(['-drive', drive_val])
611 floppy_image = hvp[constants.HV_KVM_FLOPPY_IMAGE_PATH]
613 options = ',format=raw,media=disk'
615 kvm_cmd.extend(['-boot', 'a'])
616 options = '%s,boot=on' % options
617 if_val = ',if=floppy'
618 options = '%s%s' % (options, if_val)
619 drive_val = 'file=%s%s' % (floppy_image, options)
620 kvm_cmd.extend(['-drive', drive_val])
622 kernel_path = hvp[constants.HV_KERNEL_PATH]
624 kvm_cmd.extend(['-kernel', kernel_path])
625 initrd_path = hvp[constants.HV_INITRD_PATH]
627 kvm_cmd.extend(['-initrd', initrd_path])
628 root_append = ['root=%s' % hvp[constants.HV_ROOT_PATH],
629 hvp[constants.HV_KERNEL_ARGS]]
630 if hvp[constants.HV_SERIAL_CONSOLE]:
631 root_append.append('console=ttyS0,38400')
632 kvm_cmd.extend(['-append', ' '.join(root_append)])
634 mem_path = hvp[constants.HV_MEM_PATH]
636 kvm_cmd.extend(["-mem-path", mem_path, "-mem-prealloc"])
638 mouse_type = hvp[constants.HV_USB_MOUSE]
639 vnc_bind_address = hvp[constants.HV_VNC_BIND_ADDRESS]
642 kvm_cmd.extend(['-usb'])
643 kvm_cmd.extend(['-usbdevice', mouse_type])
644 elif vnc_bind_address:
645 kvm_cmd.extend(['-usbdevice', constants.HT_MOUSE_TABLET])
648 if netutils.IP4Address.IsValid(vnc_bind_address):
649 if instance.network_port > constants.VNC_BASE_PORT:
650 display = instance.network_port - constants.VNC_BASE_PORT
651 if vnc_bind_address == constants.IP4_ADDRESS_ANY:
652 vnc_arg = ':%d' % (display)
654 vnc_arg = '%s:%d' % (vnc_bind_address, display)
656 logging.error("Network port is not a valid VNC display (%d < %d)."
657 " Not starting VNC", instance.network_port,
658 constants.VNC_BASE_PORT)
661 # Only allow tls and other option when not binding to a file, for now.
662 # kvm/qemu gets confused otherwise about the filename to use.
664 if hvp[constants.HV_VNC_TLS]:
665 vnc_append = '%s,tls' % vnc_append
666 if hvp[constants.HV_VNC_X509_VERIFY]:
667 vnc_append = '%s,x509verify=%s' % (vnc_append,
668 hvp[constants.HV_VNC_X509])
669 elif hvp[constants.HV_VNC_X509]:
670 vnc_append = '%s,x509=%s' % (vnc_append,
671 hvp[constants.HV_VNC_X509])
672 if hvp[constants.HV_VNC_PASSWORD_FILE]:
673 vnc_append = '%s,password' % vnc_append
675 vnc_arg = '%s%s' % (vnc_arg, vnc_append)
678 vnc_arg = 'unix:%s/%s.vnc' % (vnc_bind_address, instance.name)
680 kvm_cmd.extend(['-vnc', vnc_arg])
682 kvm_cmd.extend(['-nographic'])
684 monitor_dev = ("unix:%s,server,nowait" %
685 self._InstanceMonitor(instance.name))
686 kvm_cmd.extend(['-monitor', monitor_dev])
687 if hvp[constants.HV_SERIAL_CONSOLE]:
688 serial_dev = ('unix:%s,server,nowait' %
689 self._InstanceSerial(instance.name))
690 kvm_cmd.extend(['-serial', serial_dev])
692 kvm_cmd.extend(['-serial', 'none'])
694 if hvp[constants.HV_USE_LOCALTIME]:
695 kvm_cmd.extend(['-localtime'])
697 if hvp[constants.HV_KVM_USE_CHROOT]:
698 kvm_cmd.extend(['-chroot', self._InstanceChrootDir(instance.name)])
700 # Save the current instance nics, but defer their expansion as parameters,
701 # as we'll need to generate executable temp files for them.
702 kvm_nics = instance.nics
705 return (kvm_cmd, kvm_nics, hvparams)
707 def _WriteKVMRuntime(self, instance_name, data):
708 """Write an instance's KVM runtime
712 utils.WriteFile(self._InstanceKVMRuntime(instance_name),
714 except EnvironmentError, err:
715 raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err)
717 def _ReadKVMRuntime(self, instance_name):
718 """Read an instance's KVM runtime
722 file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name))
723 except EnvironmentError, err:
724 raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err)
727 def _SaveKVMRuntime(self, instance, kvm_runtime):
728 """Save an instance's KVM runtime
731 kvm_cmd, kvm_nics, hvparams = kvm_runtime
732 serialized_nics = [nic.ToDict() for nic in kvm_nics]
733 serialized_form = serializer.Dump((kvm_cmd, serialized_nics, hvparams))
734 self._WriteKVMRuntime(instance.name, serialized_form)
736 def _LoadKVMRuntime(self, instance, serialized_runtime=None):
737 """Load an instance's KVM runtime
740 if not serialized_runtime:
741 serialized_runtime = self._ReadKVMRuntime(instance.name)
742 loaded_runtime = serializer.Load(serialized_runtime)
743 kvm_cmd, serialized_nics, hvparams = loaded_runtime
744 kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics]
745 return (kvm_cmd, kvm_nics, hvparams)
747 def _RunKVMCmd(self, name, kvm_cmd, tap_fds=None):
748 """Run the KVM cmd and check for errors
751 @param name: instance name
752 @type kvm_cmd: list of strings
753 @param kvm_cmd: runcmd input for kvm
754 @type tap_fds: list of int
755 @param tap_fds: fds of tap devices opened by Ganeti
759 result = utils.RunCmd(kvm_cmd, noclose_fds=tap_fds)
762 utils_wrapper.CloseFdNoError(fd)
765 raise errors.HypervisorError("Failed to start instance %s: %s (%s)" %
766 (name, result.fail_reason, result.output))
767 if not self._InstancePidAlive(name)[2]:
768 raise errors.HypervisorError("Failed to start instance %s" % name)
770 def _ExecuteKVMRuntime(self, instance, kvm_runtime, incoming=None):
771 """Execute a KVM cmd, after completing it with some last minute data
773 @type incoming: tuple of strings
774 @param incoming: (target_host_ip, port)
777 # Small _ExecuteKVMRuntime hv parameters programming howto:
778 # - conf_hvp contains the parameters as configured on ganeti. they might
779 # have changed since the instance started; only use them if the change
780 # won't affect the inside of the instance (which hasn't been rebooted).
781 # - up_hvp contains the parameters as they were when the instance was
782 # started, plus any new parameter which has been added between ganeti
783 # versions: it is paramount that those default to a value which won't
784 # affect the inside of the instance as well.
785 conf_hvp = instance.hvparams
787 self._CheckDown(name)
791 kvm_cmd, kvm_nics, up_hvp = kvm_runtime
792 up_hvp = objects.FillDict(conf_hvp, up_hvp)
794 kvm_version = self._GetKVMVersion()
796 _, v_major, v_min, _ = kvm_version
798 raise errors.HypervisorError("Unable to get KVM version")
800 # We know it's safe to run as a different user upon migration, so we'll use
801 # the latest conf, from conf_hvp.
802 security_model = conf_hvp[constants.HV_SECURITY_MODEL]
803 if security_model == constants.HT_SM_USER:
804 kvm_cmd.extend(["-runas", conf_hvp[constants.HV_SECURITY_DOMAIN]])
806 # We have reasons to believe changing something like the nic driver/type
807 # upon migration won't exactly fly with the instance kernel, so for nic
808 # related parameters we'll use up_hvp
812 kvm_cmd.extend(["-net", "none"])
816 nic_type = up_hvp[constants.HV_NIC_TYPE]
817 if nic_type == constants.HT_NIC_PARAVIRTUAL:
818 # From version 0.12.0, kvm uses a new sintax for network configuration.
819 if (v_major, v_min) >= (0, 12):
820 nic_model = "virtio-net-pci"
825 if up_hvp[constants.HV_VHOST_NET]:
826 # vhost_net is only available from version 0.13.0 or newer
827 if (v_major, v_min) >= (0, 13):
828 tap_extra = ",vhost=on"
830 raise errors.HypervisorError("vhost_net is configured"
831 " but it is not available")
835 for nic_seq, nic in enumerate(kvm_nics):
836 tapname, tapfd = _OpenTap(vnet_hdr)
839 if (v_major, v_min) >= (0, 12):
840 nic_val = "%s,mac=%s,netdev=netdev%s" % (nic_model, nic.mac, nic_seq)
841 tap_val = "type=tap,id=netdev%s,fd=%d%s" % (nic_seq, tapfd, tap_extra)
842 kvm_cmd.extend(["-netdev", tap_val, "-device", nic_val])
844 nic_val = "nic,vlan=%s,macaddr=%s,model=%s" % (nic_seq,
846 tap_val = "tap,vlan=%s,fd=%d" % (nic_seq, tapfd)
847 kvm_cmd.extend(["-net", tap_val, "-net", nic_val])
850 target, port = incoming
851 kvm_cmd.extend(['-incoming', 'tcp:%s:%s' % (target, port)])
853 # Changing the vnc password doesn't bother the guest that much. At most it
854 # will surprise people who connect to it. Whether positively or negatively
856 vnc_pwd_file = conf_hvp[constants.HV_VNC_PASSWORD_FILE]
860 vnc_pwd = utils.ReadOneLineFile(vnc_pwd_file, strict=True)
861 except EnvironmentError, err:
862 raise errors.HypervisorError("Failed to open VNC password file %s: %s"
863 % (vnc_pwd_file, err))
865 if conf_hvp[constants.HV_KVM_USE_CHROOT]:
866 utils.EnsureDirs([(self._InstanceChrootDir(name),
867 constants.SECURE_DIR_MODE)])
869 # Configure the network now for starting instances and bridged interfaces,
870 # during FinalizeMigration for incoming instances' routed interfaces
871 for nic_seq, nic in enumerate(kvm_nics):
873 nic.nicparams[constants.NIC_MODE] != constants.NIC_MODE_BRIDGED):
875 self._ConfigureNIC(instance, nic_seq, nic, taps[nic_seq])
877 if security_model == constants.HT_SM_POOL:
878 ss = ssconf.SimpleStore()
879 uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\n")
880 all_uids = set(uidpool.ExpandUidPool(uid_pool))
881 uid = uidpool.RequestUnusedUid(all_uids)
883 username = pwd.getpwuid(uid.GetUid()).pw_name
884 kvm_cmd.extend(["-runas", username])
885 self._RunKVMCmd(name, kvm_cmd, tapfds)
887 uidpool.ReleaseUid(uid)
891 utils.WriteFile(self._InstanceUidFile(name), data=uid.AsStr())
893 self._RunKVMCmd(name, kvm_cmd, tapfds)
895 utils.EnsureDirs([(self._InstanceNICDir(instance.name),
896 constants.RUN_DIRS_MODE)])
897 for nic_seq, tap in enumerate(taps):
898 utils.WriteFile(self._InstanceNICFile(instance.name, nic_seq),
902 change_cmd = 'change vnc password %s' % vnc_pwd
903 self._CallMonitorCommand(instance.name, change_cmd)
905 for filename in temp_files:
906 utils.RemoveFile(filename)
908 def StartInstance(self, instance, block_devices, startup_paused):
909 """Start an instance.
912 self._CheckDown(instance.name)
913 kvm_runtime = self._GenerateKVMRuntime(instance, block_devices,
915 self._SaveKVMRuntime(instance, kvm_runtime)
916 self._ExecuteKVMRuntime(instance, kvm_runtime)
918 def _CallMonitorCommand(self, instance_name, command):
919 """Invoke a command on the instance monitor.
922 socat = ("echo %s | %s STDIO UNIX-CONNECT:%s" %
923 (utils.ShellQuote(command),
924 constants.SOCAT_PATH,
925 utils.ShellQuote(self._InstanceMonitor(instance_name))))
926 result = utils.RunCmd(socat)
928 msg = ("Failed to send command '%s' to instance %s."
929 " output: %s, error: %s, fail_reason: %s" %
930 (command, instance_name,
931 result.stdout, result.stderr, result.fail_reason))
932 raise errors.HypervisorError(msg)
937 def _GetKVMVersion(cls):
938 """Return the installed KVM version
940 @return: (version, v_maj, v_min, v_rev), or None
943 result = utils.RunCmd([constants.KVM_PATH, "--help"])
946 match = cls._VERSION_RE.search(result.output.splitlines()[0])
950 return (match.group(0), int(match.group(1)), int(match.group(2)),
953 def StopInstance(self, instance, force=False, retry=False, name=None):
957 if name is not None and not force:
958 raise errors.HypervisorError("Cannot shutdown cleanly by name only")
961 acpi = instance.hvparams[constants.HV_ACPI]
964 _, pid, alive = self._InstancePidAlive(name)
965 if pid > 0 and alive:
966 if force or not acpi:
967 utils.KillProcess(pid)
969 self._CallMonitorCommand(name, 'system_powerdown')
971 def CleanupInstance(self, instance_name):
972 """Cleanup after a stopped instance
975 pidfile, pid, alive = self._InstancePidAlive(instance_name)
976 if pid > 0 and alive:
977 raise errors.HypervisorError("Cannot cleanup a live instance")
978 self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
980 def RebootInstance(self, instance):
981 """Reboot an instance.
984 # For some reason if we do a 'send-key ctrl-alt-delete' to the control
985 # socket the instance will stop, but now power up again. So we'll resort
986 # to shutdown and restart.
987 _, _, alive = self._InstancePidAlive(instance.name)
989 raise errors.HypervisorError("Failed to reboot instance %s:"
990 " not running" % instance.name)
991 # StopInstance will delete the saved KVM runtime so:
992 # ...first load it...
993 kvm_runtime = self._LoadKVMRuntime(instance)
994 # ...now we can safely call StopInstance...
995 if not self.StopInstance(instance):
996 self.StopInstance(instance, force=True)
997 # ...and finally we can save it again, and execute it...
998 self._SaveKVMRuntime(instance, kvm_runtime)
999 self._ExecuteKVMRuntime(instance, kvm_runtime)
1001 def MigrationInfo(self, instance):
1002 """Get instance information to perform a migration.
1004 @type instance: L{objects.Instance}
1005 @param instance: instance to be migrated
1007 @return: content of the KVM runtime file
1010 return self._ReadKVMRuntime(instance.name)
1012 def AcceptInstance(self, instance, info, target):
1013 """Prepare to accept an instance.
1015 @type instance: L{objects.Instance}
1016 @param instance: instance to be accepted
1018 @param info: content of the KVM runtime file on the source node
1019 @type target: string
1020 @param target: target host (usually ip), on this node
1023 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
1024 incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT])
1025 self._ExecuteKVMRuntime(instance, kvm_runtime, incoming=incoming_address)
1027 def FinalizeMigration(self, instance, info, success):
1028 """Finalize an instance migration.
1030 Stop the incoming mode KVM.
1032 @type instance: L{objects.Instance}
1033 @param instance: instance whose migration is being finalized
1037 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
1038 kvm_nics = kvm_runtime[1]
1040 for nic_seq, nic in enumerate(kvm_nics):
1041 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
1042 # Bridged interfaces have already been configured
1045 tap = utils.ReadFile(self._InstanceNICFile(instance.name, nic_seq))
1046 except EnvironmentError, err:
1047 logging.warning("Failed to find host interface for %s NIC #%d: %s",
1048 instance.name, nic_seq, str(err))
1051 self._ConfigureNIC(instance, nic_seq, nic, tap)
1052 except errors.HypervisorError, err:
1053 logging.warning(str(err))
1055 self._WriteKVMRuntime(instance.name, info)
1057 self.StopInstance(instance, force=True)
1059 def MigrateInstance(self, instance, target, live):
1060 """Migrate an instance to a target node.
1062 The migration will not be attempted if the instance is not
1065 @type instance: L{objects.Instance}
1066 @param instance: the instance to be migrated
1067 @type target: string
1068 @param target: ip address of the target node
1070 @param live: perform a live migration
1073 instance_name = instance.name
1074 port = instance.hvparams[constants.HV_MIGRATION_PORT]
1075 pidfile, pid, alive = self._InstancePidAlive(instance_name)
1077 raise errors.HypervisorError("Instance not running, cannot migrate")
1080 self._CallMonitorCommand(instance_name, 'stop')
1082 migrate_command = ('migrate_set_speed %dm' %
1083 instance.hvparams[constants.HV_MIGRATION_BANDWIDTH])
1084 self._CallMonitorCommand(instance_name, migrate_command)
1086 migrate_command = ('migrate_set_downtime %dms' %
1087 instance.hvparams[constants.HV_MIGRATION_DOWNTIME])
1088 self._CallMonitorCommand(instance_name, migrate_command)
1090 migrate_command = 'migrate -d tcp:%s:%s' % (target, port)
1091 self._CallMonitorCommand(instance_name, migrate_command)
1093 info_command = 'info migrate'
1097 result = self._CallMonitorCommand(instance_name, info_command)
1098 match = self._MIGRATION_STATUS_RE.search(result.stdout)
1101 if not result.stdout:
1102 logging.info("KVM: empty 'info migrate' result")
1104 logging.warning("KVM: unknown 'info migrate' result: %s",
1106 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1108 status = match.group(1)
1109 if status == 'completed':
1111 elif status == 'active':
1112 # reset the broken answers count
1114 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1115 elif status == 'failed' or status == 'cancelled':
1117 self._CallMonitorCommand(instance_name, 'cont')
1118 raise errors.HypervisorError("Migration %s at the kvm level" %
1121 logging.warning("KVM: unknown migration status '%s'", status)
1123 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1124 if broken_answers >= self._MIGRATION_INFO_MAX_BAD_ANSWERS:
1125 raise errors.HypervisorError("Too many 'info migrate' broken answers")
1127 utils.KillProcess(pid)
1128 self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
1130 def GetNodeInfo(self):
1131 """Return information about the node.
1133 This is just a wrapper over the base GetLinuxNodeInfo method.
1135 @return: a dict with the following keys (values in MiB):
1136 - memory_total: the total memory size on the node
1137 - memory_free: the available memory on the node for instances
1138 - memory_dom0: the memory used by the node itself, if available
1141 return self.GetLinuxNodeInfo()
1144 def GetInstanceConsole(cls, instance, hvparams, beparams):
1145 """Return a command for connecting to the console of an instance.
1148 if hvparams[constants.HV_SERIAL_CONSOLE]:
1149 cmd = [constants.KVM_CONSOLE_WRAPPER,
1150 constants.SOCAT_PATH, utils.ShellQuote(instance.name),
1151 utils.ShellQuote(cls._InstanceMonitor(instance.name)),
1152 "STDIO,%s" % cls._SocatUnixConsoleParams(),
1153 "UNIX-CONNECT:%s" % cls._InstanceSerial(instance.name)]
1154 return objects.InstanceConsole(instance=instance.name,
1155 kind=constants.CONS_SSH,
1156 host=instance.primary_node,
1157 user=constants.GANETI_RUNAS,
1160 vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS]
1161 if vnc_bind_address and instance.network_port > constants.VNC_BASE_PORT:
1162 display = instance.network_port - constants.VNC_BASE_PORT
1163 return objects.InstanceConsole(instance=instance.name,
1164 kind=constants.CONS_VNC,
1165 host=vnc_bind_address,
1166 port=instance.network_port,
1169 return objects.InstanceConsole(instance=instance.name,
1170 kind=constants.CONS_MESSAGE,
1171 message=("No serial shell for instance %s" %
1175 """Verify the hypervisor.
1177 Check that the binary exists.
1180 if not os.path.exists(constants.KVM_PATH):
1181 return "The kvm binary ('%s') does not exist." % constants.KVM_PATH
1182 if not os.path.exists(constants.SOCAT_PATH):
1183 return "The socat binary ('%s') does not exist." % constants.SOCAT_PATH
1186 def CheckParameterSyntax(cls, hvparams):
1187 """Check the given parameters for validity.
1189 @type hvparams: dict
1190 @param hvparams: dictionary with parameter names/value
1191 @raise errors.HypervisorError: when a parameter is not valid
1194 super(KVMHypervisor, cls).CheckParameterSyntax(hvparams)
1196 kernel_path = hvparams[constants.HV_KERNEL_PATH]
1198 if not hvparams[constants.HV_ROOT_PATH]:
1199 raise errors.HypervisorError("Need a root partition for the instance,"
1200 " if a kernel is defined")
1202 if (hvparams[constants.HV_VNC_X509_VERIFY] and
1203 not hvparams[constants.HV_VNC_X509]):
1204 raise errors.HypervisorError("%s must be defined, if %s is" %
1205 (constants.HV_VNC_X509,
1206 constants.HV_VNC_X509_VERIFY))
1208 boot_order = hvparams[constants.HV_BOOT_ORDER]
1209 if (boot_order == constants.HT_BO_CDROM and
1210 not hvparams[constants.HV_CDROM_IMAGE_PATH]):
1211 raise errors.HypervisorError("Cannot boot from cdrom without an"
1214 security_model = hvparams[constants.HV_SECURITY_MODEL]
1215 if security_model == constants.HT_SM_USER:
1216 if not hvparams[constants.HV_SECURITY_DOMAIN]:
1217 raise errors.HypervisorError("A security domain (user to run kvm as)"
1218 " must be specified")
1219 elif (security_model == constants.HT_SM_NONE or
1220 security_model == constants.HT_SM_POOL):
1221 if hvparams[constants.HV_SECURITY_DOMAIN]:
1222 raise errors.HypervisorError("Cannot have a security domain when the"
1223 " security model is 'none' or 'pool'")
1226 def ValidateParameters(cls, hvparams):
1227 """Check the given parameters for validity.
1229 @type hvparams: dict
1230 @param hvparams: dictionary with parameter names/value
1231 @raise errors.HypervisorError: when a parameter is not valid
1234 super(KVMHypervisor, cls).ValidateParameters(hvparams)
1236 security_model = hvparams[constants.HV_SECURITY_MODEL]
1237 if security_model == constants.HT_SM_USER:
1238 username = hvparams[constants.HV_SECURITY_DOMAIN]
1240 pwd.getpwnam(username)
1242 raise errors.HypervisorError("Unknown security domain user %s"
1246 def PowercycleNode(cls):
1247 """KVM powercycle, just a wrapper over Linux powercycle.
1250 cls.LinuxPowercycle()