4 # Copyright (C) 2008, 2009, 2010 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
38 from ganeti import utils
39 from ganeti import constants
40 from ganeti import errors
41 from ganeti import serializer
42 from ganeti import objects
43 from ganeti import uidpool
44 from ganeti import ssconf
45 from ganeti.hypervisor import hv_base
46 from ganeti import netutils
47 from ganeti.utils import wrapper as utils_wrapper
50 _KVM_NETWORK_SCRIPT = constants.SYSCONFDIR + "/ganeti/kvm-vif-bridge"
52 # TUN/TAP driver constants, taken from <linux/if_tun.h>
53 # They are architecture-independent and already hardcoded in qemu-kvm source,
54 # so we can safely include them here.
55 TUNSETIFF = 0x400454ca
56 TUNGETIFF = 0x800454d2
57 TUNGETFEATURES = 0x800454cf
63 def _ProbeTapVnetHdr(fd):
64 """Check whether to enable the IFF_VNET_HDR flag.
66 To do this, _all_ of the following conditions must be met:
67 1. TUNGETFEATURES ioctl() *must* be implemented
68 2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag
69 3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in
70 drivers/net/tun.c there is no way to test this until after the tap device
71 has been created using TUNSETIFF, and there is no way to change the
72 IFF_VNET_HDR flag after creating the interface, catch-22! However both
73 TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27,
74 thus we can expect TUNGETIFF to be present if TUNGETFEATURES is.
77 @param fd: the file descriptor of /dev/net/tun
80 req = struct.pack("I", 0)
82 res = fcntl.ioctl(fd, TUNGETFEATURES, req)
83 except EnvironmentError:
84 logging.warning("TUNGETFEATURES ioctl() not implemented")
87 tunflags = struct.unpack("I", res)[0]
88 if tunflags & IFF_VNET_HDR:
91 logging.warning("Host does not support IFF_VNET_HDR, not enabling")
95 def _OpenTap(vnet_hdr=True):
96 """Open a new tap device and return its file descriptor.
98 This is intended to be used by a qemu-type hypervisor together with the -net
99 tap,fd=<fd> command line parameter.
101 @type vnet_hdr: boolean
102 @param vnet_hdr: Enable the VNET Header
103 @return: (ifname, tapfd)
108 tapfd = os.open("/dev/net/tun", os.O_RDWR)
109 except EnvironmentError:
110 raise errors.HypervisorError("Failed to open /dev/net/tun")
112 flags = IFF_TAP | IFF_NO_PI
114 if vnet_hdr and _ProbeTapVnetHdr(tapfd):
115 flags |= IFF_VNET_HDR
117 # The struct ifreq ioctl request (see netdevice(7))
118 ifr = struct.pack("16sh", "", flags)
121 res = fcntl.ioctl(tapfd, TUNSETIFF, ifr)
122 except EnvironmentError:
123 raise errors.HypervisorError("Failed to allocate a new TAP device")
125 # Get the interface name from the ioctl
126 ifname = struct.unpack("16sh", res)[0].strip("\x00")
127 return (ifname, tapfd)
130 class KVMHypervisor(hv_base.BaseHypervisor):
131 """KVM hypervisor interface"""
134 _ROOT_DIR = constants.RUN_GANETI_DIR + "/kvm-hypervisor"
135 _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids
136 _UIDS_DIR = _ROOT_DIR + "/uid" # contains instances reserved uids
137 _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets
138 _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data
139 _NICS_DIR = _ROOT_DIR + "/nic" # contains instances nic <-> tap associations
140 # KVM instances with chroot enabled are started in empty chroot directories.
141 _CHROOT_DIR = _ROOT_DIR + "/chroot" # for empty chroot directories
142 # After an instance is stopped, its chroot directory is removed.
143 # If the chroot directory is not empty, it can't be removed.
144 # A non-empty chroot directory indicates a possible security incident.
145 # To support forensics, the non-empty chroot directory is quarantined in
146 # a separate directory, called 'chroot-quarantine'.
147 _CHROOT_QUARANTINE_DIR = _ROOT_DIR + "/chroot-quarantine"
148 _DIRS = [_ROOT_DIR, _PIDS_DIR, _UIDS_DIR, _CTRL_DIR, _CONF_DIR, _NICS_DIR,
149 _CHROOT_DIR, _CHROOT_QUARANTINE_DIR]
152 constants.HV_KERNEL_PATH: hv_base.OPT_FILE_CHECK,
153 constants.HV_INITRD_PATH: hv_base.OPT_FILE_CHECK,
154 constants.HV_ROOT_PATH: hv_base.NO_CHECK,
155 constants.HV_KERNEL_ARGS: hv_base.NO_CHECK,
156 constants.HV_ACPI: hv_base.NO_CHECK,
157 constants.HV_SERIAL_CONSOLE: hv_base.NO_CHECK,
158 constants.HV_VNC_BIND_ADDRESS:
159 (False, lambda x: (netutils.IP4Address.IsValid(x) or
160 utils.IsNormAbsPath(x)),
161 "the VNC bind address must be either a valid IP address or an absolute"
162 " pathname", None, None),
163 constants.HV_VNC_TLS: hv_base.NO_CHECK,
164 constants.HV_VNC_X509: hv_base.OPT_DIR_CHECK,
165 constants.HV_VNC_X509_VERIFY: hv_base.NO_CHECK,
166 constants.HV_VNC_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
167 constants.HV_KVM_FLOPPY_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
168 constants.HV_CDROM_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
169 constants.HV_KVM_CDROM2_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
170 constants.HV_BOOT_ORDER:
171 hv_base.ParamInSet(True, constants.HT_KVM_VALID_BO_TYPES),
172 constants.HV_NIC_TYPE:
173 hv_base.ParamInSet(True, constants.HT_KVM_VALID_NIC_TYPES),
174 constants.HV_DISK_TYPE:
175 hv_base.ParamInSet(True, constants.HT_KVM_VALID_DISK_TYPES),
176 constants.HV_KVM_CDROM_DISK_TYPE:
177 hv_base.ParamInSet(False, constants.HT_KVM_VALID_DISK_TYPES),
178 constants.HV_USB_MOUSE:
179 hv_base.ParamInSet(False, constants.HT_KVM_VALID_MOUSE_TYPES),
180 constants.HV_MIGRATION_PORT: hv_base.NET_PORT_CHECK,
181 constants.HV_MIGRATION_BANDWIDTH: hv_base.NO_CHECK,
182 constants.HV_MIGRATION_DOWNTIME: hv_base.NO_CHECK,
183 constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK,
184 constants.HV_USE_LOCALTIME: hv_base.NO_CHECK,
185 constants.HV_DISK_CACHE:
186 hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES),
187 constants.HV_SECURITY_MODEL:
188 hv_base.ParamInSet(True, constants.HT_KVM_VALID_SM_TYPES),
189 constants.HV_SECURITY_DOMAIN: hv_base.NO_CHECK,
190 constants.HV_KVM_FLAG:
191 hv_base.ParamInSet(False, constants.HT_KVM_FLAG_VALUES),
192 constants.HV_VHOST_NET: hv_base.NO_CHECK,
193 constants.HV_KVM_USE_CHROOT: hv_base.NO_CHECK,
194 constants.HV_MEM_PATH: hv_base.OPT_DIR_CHECK,
197 _MIGRATION_STATUS_RE = re.compile('Migration\s+status:\s+(\w+)',
199 _MIGRATION_INFO_MAX_BAD_ANSWERS = 5
200 _MIGRATION_INFO_RETRY_DELAY = 2
202 _VERSION_RE = re.compile(r"\b(\d+)\.(\d+)\.(\d+)\b")
209 hv_base.BaseHypervisor.__init__(self)
210 # Let's make sure the directories we need exist, even if the RUN_DIR lives
211 # in a tmpfs filesystem or has been otherwise wiped out.
212 dirs = [(dname, constants.RUN_DIRS_MODE) for dname in self._DIRS]
213 utils.EnsureDirs(dirs)
216 def _InstancePidFile(cls, instance_name):
217 """Returns the instance pidfile.
220 return utils.PathJoin(cls._PIDS_DIR, instance_name)
223 def _InstanceUidFile(cls, instance_name):
224 """Returns the instance uidfile.
227 return utils.PathJoin(cls._UIDS_DIR, instance_name)
230 def _InstancePidInfo(cls, pid):
231 """Check pid file for instance information.
233 Check that a pid file is associated with an instance, and retrieve
234 information from its command line.
236 @type pid: string or int
237 @param pid: process id of the instance to check
239 @return: (instance_name, memory, vcpus)
240 @raise errors.HypervisorError: when an instance cannot be found
243 alive = utils.IsProcessAlive(pid)
245 raise errors.HypervisorError("Cannot get info for pid %s" % pid)
247 cmdline_file = utils.PathJoin("/proc", str(pid), "cmdline")
249 cmdline = utils.ReadFile(cmdline_file)
250 except EnvironmentError, err:
251 raise errors.HypervisorError("Can't open cmdline file for pid %s: %s" %
258 arg_list = cmdline.split('\x00')
260 arg = arg_list.pop(0)
262 instance = arg_list.pop(0)
264 memory = int(arg_list.pop(0))
266 vcpus = int(arg_list.pop(0))
269 raise errors.HypervisorError("Pid %s doesn't contain a ganeti kvm"
272 return (instance, memory, vcpus)
274 def _InstancePidAlive(self, instance_name):
275 """Returns the instance pidfile, pid, and liveness.
277 @type instance_name: string
278 @param instance_name: instance name
280 @return: (pid file name, pid, liveness)
283 pidfile = self._InstancePidFile(instance_name)
284 pid = utils.ReadPidFile(pidfile)
288 cmd_instance = self._InstancePidInfo(pid)[0]
289 alive = (cmd_instance == instance_name)
290 except errors.HypervisorError:
293 return (pidfile, pid, alive)
295 def _CheckDown(self, instance_name):
296 """Raises an error unless the given instance is down.
299 alive = self._InstancePidAlive(instance_name)[2]
301 raise errors.HypervisorError("Failed to start instance %s: %s" %
302 (instance_name, "already running"))
305 def _InstanceMonitor(cls, instance_name):
306 """Returns the instance monitor socket name
309 return utils.PathJoin(cls._CTRL_DIR, "%s.monitor" % instance_name)
312 def _InstanceSerial(cls, instance_name):
313 """Returns the instance serial socket name
316 return utils.PathJoin(cls._CTRL_DIR, "%s.serial" % instance_name)
319 def _SocatUnixConsoleParams():
320 """Returns the correct parameters for socat
322 If we have a new-enough socat we can use raw mode with an escape character.
325 if constants.SOCAT_USE_ESCAPE:
326 return "raw,echo=0,escape=%s" % constants.SOCAT_ESCAPE_CODE
328 return "echo=0,icanon=0"
331 def _InstanceKVMRuntime(cls, instance_name):
332 """Returns the instance KVM runtime filename
335 return utils.PathJoin(cls._CONF_DIR, "%s.runtime" % instance_name)
338 def _InstanceChrootDir(cls, instance_name):
339 """Returns the name of the KVM chroot dir of the instance
342 return utils.PathJoin(cls._CHROOT_DIR, instance_name)
345 def _InstanceNICDir(cls, instance_name):
346 """Returns the name of the directory holding the tap device files for a
350 return utils.PathJoin(cls._NICS_DIR, instance_name)
353 def _InstanceNICFile(cls, instance_name, seq):
354 """Returns the name of the file containing the tap device for a given NIC
357 return utils.PathJoin(cls._InstanceNICDir(instance_name), str(seq))
360 def _TryReadUidFile(cls, uid_file):
361 """Try to read a uid file
364 if os.path.exists(uid_file):
366 uid = int(utils.ReadOneLineFile(uid_file))
368 except EnvironmentError:
369 logging.warning("Can't read uid file", exc_info=True)
370 except (TypeError, ValueError):
371 logging.warning("Can't parse uid file contents", exc_info=True)
375 def _RemoveInstanceRuntimeFiles(cls, pidfile, instance_name):
376 """Removes an instance's rutime sockets/files/dirs.
379 utils.RemoveFile(pidfile)
380 utils.RemoveFile(cls._InstanceMonitor(instance_name))
381 utils.RemoveFile(cls._InstanceSerial(instance_name))
382 utils.RemoveFile(cls._InstanceKVMRuntime(instance_name))
383 uid_file = cls._InstanceUidFile(instance_name)
384 uid = cls._TryReadUidFile(uid_file)
385 utils.RemoveFile(uid_file)
387 uidpool.ReleaseUid(uid)
389 shutil.rmtree(cls._InstanceNICDir(instance_name))
391 if err.errno != errno.ENOENT:
394 chroot_dir = cls._InstanceChrootDir(instance_name)
395 utils.RemoveDir(chroot_dir)
397 if err.errno == errno.ENOTEMPTY:
398 # The chroot directory is expected to be empty, but it isn't.
399 new_chroot_dir = tempfile.mkdtemp(dir=cls._CHROOT_QUARANTINE_DIR,
402 utils.TimestampForFilename()))
403 logging.warning("The chroot directory of instance %s can not be"
404 " removed as it is not empty. Moving it to the"
405 " quarantine instead. Please investigate the"
406 " contents (%s) and clean up manually",
407 instance_name, new_chroot_dir)
408 utils.RenameFile(chroot_dir, new_chroot_dir)
413 def _ConfigureNIC(instance, seq, nic, tap):
414 """Run the network configuration script for a specified NIC
416 @param instance: instance we're acting on
417 @type instance: instance object
418 @param seq: nic sequence number
420 @param nic: nic we're acting on
421 @type nic: nic object
422 @param tap: the host's tap interface this NIC corresponds to
428 tags = " ".join(instance.tags)
433 "PATH": "%s:/sbin:/usr/sbin" % os.environ["PATH"],
434 "INSTANCE": instance.name,
436 "MODE": nic.nicparams[constants.NIC_MODE],
438 "INTERFACE_INDEX": str(seq),
445 if nic.nicparams[constants.NIC_LINK]:
446 env["LINK"] = nic.nicparams[constants.NIC_LINK]
448 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
449 env["BRIDGE"] = nic.nicparams[constants.NIC_LINK]
451 result = utils.RunCmd([constants.KVM_IFUP, tap], env=env)
453 raise errors.HypervisorError("Failed to configure interface %s: %s."
454 " Network configuration script output: %s" %
455 (tap, result.fail_reason, result.output))
457 def ListInstances(self):
458 """Get the list of running instances.
460 We can do this by listing our live instances directory and
461 checking whether the associated kvm process is still alive.
465 for name in os.listdir(self._PIDS_DIR):
466 if self._InstancePidAlive(name)[2]:
470 def GetInstanceInfo(self, instance_name):
471 """Get instance properties.
473 @type instance_name: string
474 @param instance_name: the instance name
475 @rtype: tuple of strings
476 @return: (name, id, memory, vcpus, stat, times)
479 _, pid, alive = self._InstancePidAlive(instance_name)
483 _, memory, vcpus = self._InstancePidInfo(pid)
487 return (instance_name, pid, memory, vcpus, stat, times)
489 def GetAllInstancesInfo(self):
490 """Get properties of all instances.
492 @return: list of tuples (name, id, memory, vcpus, stat, times)
496 for name in os.listdir(self._PIDS_DIR):
498 info = self.GetInstanceInfo(name)
499 except errors.HypervisorError:
505 def _GenerateKVMRuntime(self, instance, block_devices):
506 """Generate KVM information to start an instance.
509 kvm_version = self._GetKVMVersion()
511 _, v_major, v_min, _ = kvm_version
513 raise errors.HypervisorError("Unable to get KVM version")
515 pidfile = self._InstancePidFile(instance.name)
516 kvm = constants.KVM_PATH
518 # used just by the vnc server, if enabled
519 kvm_cmd.extend(['-name', instance.name])
520 kvm_cmd.extend(['-m', instance.beparams[constants.BE_MEMORY]])
521 kvm_cmd.extend(['-smp', instance.beparams[constants.BE_VCPUS]])
522 kvm_cmd.extend(['-pidfile', pidfile])
523 kvm_cmd.extend(['-daemonize'])
524 if not instance.hvparams[constants.HV_ACPI]:
525 kvm_cmd.extend(['-no-acpi'])
527 hvp = instance.hvparams
528 boot_disk = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
529 boot_cdrom = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_CDROM
530 boot_floppy = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_FLOPPY
531 boot_network = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_NETWORK
533 if hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED:
534 kvm_cmd.extend(["-enable-kvm"])
535 elif hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_DISABLED:
536 kvm_cmd.extend(["-disable-kvm"])
539 kvm_cmd.extend(['-boot', 'n'])
541 disk_type = hvp[constants.HV_DISK_TYPE]
542 if disk_type == constants.HT_DISK_PARAVIRTUAL:
543 if_val = ',if=virtio'
545 if_val = ',if=%s' % disk_type
547 disk_cache = hvp[constants.HV_DISK_CACHE]
548 if disk_cache != constants.HT_CACHE_DEFAULT:
549 cache_val = ",cache=%s" % disk_cache
552 for cfdev, dev_path in block_devices:
553 if cfdev.mode != constants.DISK_RDWR:
554 raise errors.HypervisorError("Instance has read-only disks which"
555 " are not supported by KVM")
556 # TODO: handle FD_LOOP and FD_BLKTAP (?)
559 kvm_cmd.extend(['-boot', 'c'])
561 if (v_major, v_min) < (0, 14) and disk_type != constants.HT_DISK_IDE:
562 boot_val = ",boot=on"
564 drive_val = 'file=%s,format=raw%s%s%s' % (dev_path, if_val, boot_val,
566 kvm_cmd.extend(['-drive', drive_val])
568 #Now we can specify a different device type for CDROM devices.
569 cdrom_disk_type = hvp[constants.HV_KVM_CDROM_DISK_TYPE]
570 if not cdrom_disk_type:
571 cdrom_disk_type = disk_type
573 iso_image = hvp[constants.HV_CDROM_IMAGE_PATH]
575 options = ',format=raw,media=cdrom'
577 kvm_cmd.extend(['-boot', 'd'])
578 if cdrom_disk_type != constants.HT_DISK_IDE:
579 options = '%s,boot=on,if=%s' % (options, constants.HT_DISK_IDE)
581 options = '%s,boot=on' % options
583 if cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
584 if_val = ',if=virtio'
586 if_val = ',if=%s' % cdrom_disk_type
587 options = '%s%s' % (options, if_val)
588 drive_val = 'file=%s%s' % (iso_image, options)
589 kvm_cmd.extend(['-drive', drive_val])
591 iso_image2 = hvp[constants.HV_KVM_CDROM2_IMAGE_PATH]
593 options = ',format=raw,media=cdrom'
594 if cdrom_disk_type == constants.HT_DISK_PARAVIRTUAL:
595 if_val = ',if=virtio'
597 if_val = ',if=%s' % cdrom_disk_type
598 options = '%s%s' % (options, if_val)
599 drive_val = 'file=%s%s' % (iso_image2, options)
600 kvm_cmd.extend(['-drive', drive_val])
602 floppy_image = hvp[constants.HV_KVM_FLOPPY_IMAGE_PATH]
604 options = ',format=raw,media=disk'
606 kvm_cmd.extend(['-boot', 'a'])
607 options = '%s,boot=on' % options
608 if_val = ',if=floppy'
609 options = '%s%s' % (options, if_val)
610 drive_val = 'file=%s%s' % (floppy_image, options)
611 kvm_cmd.extend(['-drive', drive_val])
613 kernel_path = hvp[constants.HV_KERNEL_PATH]
615 kvm_cmd.extend(['-kernel', kernel_path])
616 initrd_path = hvp[constants.HV_INITRD_PATH]
618 kvm_cmd.extend(['-initrd', initrd_path])
619 root_append = ['root=%s' % hvp[constants.HV_ROOT_PATH],
620 hvp[constants.HV_KERNEL_ARGS]]
621 if hvp[constants.HV_SERIAL_CONSOLE]:
622 root_append.append('console=ttyS0,38400')
623 kvm_cmd.extend(['-append', ' '.join(root_append)])
625 mem_path = hvp[constants.HV_MEM_PATH]
627 kvm_cmd.extend(["-mem-path", mem_path, "-mem-prealloc"])
629 mouse_type = hvp[constants.HV_USB_MOUSE]
630 vnc_bind_address = hvp[constants.HV_VNC_BIND_ADDRESS]
633 kvm_cmd.extend(['-usb'])
634 kvm_cmd.extend(['-usbdevice', mouse_type])
635 elif vnc_bind_address:
636 kvm_cmd.extend(['-usbdevice', constants.HT_MOUSE_TABLET])
639 if netutils.IP4Address.IsValid(vnc_bind_address):
640 if instance.network_port > constants.VNC_BASE_PORT:
641 display = instance.network_port - constants.VNC_BASE_PORT
642 if vnc_bind_address == constants.IP4_ADDRESS_ANY:
643 vnc_arg = ':%d' % (display)
645 vnc_arg = '%s:%d' % (vnc_bind_address, display)
647 logging.error("Network port is not a valid VNC display (%d < %d)."
648 " Not starting VNC", instance.network_port,
649 constants.VNC_BASE_PORT)
652 # Only allow tls and other option when not binding to a file, for now.
653 # kvm/qemu gets confused otherwise about the filename to use.
655 if hvp[constants.HV_VNC_TLS]:
656 vnc_append = '%s,tls' % vnc_append
657 if hvp[constants.HV_VNC_X509_VERIFY]:
658 vnc_append = '%s,x509verify=%s' % (vnc_append,
659 hvp[constants.HV_VNC_X509])
660 elif hvp[constants.HV_VNC_X509]:
661 vnc_append = '%s,x509=%s' % (vnc_append,
662 hvp[constants.HV_VNC_X509])
663 if hvp[constants.HV_VNC_PASSWORD_FILE]:
664 vnc_append = '%s,password' % vnc_append
666 vnc_arg = '%s%s' % (vnc_arg, vnc_append)
669 vnc_arg = 'unix:%s/%s.vnc' % (vnc_bind_address, instance.name)
671 kvm_cmd.extend(['-vnc', vnc_arg])
673 kvm_cmd.extend(['-nographic'])
675 monitor_dev = ("unix:%s,server,nowait" %
676 self._InstanceMonitor(instance.name))
677 kvm_cmd.extend(['-monitor', monitor_dev])
678 if hvp[constants.HV_SERIAL_CONSOLE]:
679 serial_dev = ('unix:%s,server,nowait' %
680 self._InstanceSerial(instance.name))
681 kvm_cmd.extend(['-serial', serial_dev])
683 kvm_cmd.extend(['-serial', 'none'])
685 if hvp[constants.HV_USE_LOCALTIME]:
686 kvm_cmd.extend(['-localtime'])
688 if hvp[constants.HV_KVM_USE_CHROOT]:
689 kvm_cmd.extend(['-chroot', self._InstanceChrootDir(instance.name)])
691 # Save the current instance nics, but defer their expansion as parameters,
692 # as we'll need to generate executable temp files for them.
693 kvm_nics = instance.nics
696 return (kvm_cmd, kvm_nics, hvparams)
698 def _WriteKVMRuntime(self, instance_name, data):
699 """Write an instance's KVM runtime
703 utils.WriteFile(self._InstanceKVMRuntime(instance_name),
705 except EnvironmentError, err:
706 raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err)
708 def _ReadKVMRuntime(self, instance_name):
709 """Read an instance's KVM runtime
713 file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name))
714 except EnvironmentError, err:
715 raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err)
718 def _SaveKVMRuntime(self, instance, kvm_runtime):
719 """Save an instance's KVM runtime
722 kvm_cmd, kvm_nics, hvparams = kvm_runtime
723 serialized_nics = [nic.ToDict() for nic in kvm_nics]
724 serialized_form = serializer.Dump((kvm_cmd, serialized_nics, hvparams))
725 self._WriteKVMRuntime(instance.name, serialized_form)
727 def _LoadKVMRuntime(self, instance, serialized_runtime=None):
728 """Load an instance's KVM runtime
731 if not serialized_runtime:
732 serialized_runtime = self._ReadKVMRuntime(instance.name)
733 loaded_runtime = serializer.Load(serialized_runtime)
734 kvm_cmd, serialized_nics, hvparams = loaded_runtime
735 kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics]
736 return (kvm_cmd, kvm_nics, hvparams)
738 def _RunKVMCmd(self, name, kvm_cmd, tap_fds=None):
739 """Run the KVM cmd and check for errors
742 @param name: instance name
743 @type kvm_cmd: list of strings
744 @param kvm_cmd: runcmd input for kvm
745 @type tap_fds: list of int
746 @param tap_fds: fds of tap devices opened by Ganeti
750 result = utils.RunCmd(kvm_cmd, noclose_fds=tap_fds)
753 utils_wrapper.CloseFdNoError(fd)
756 raise errors.HypervisorError("Failed to start instance %s: %s (%s)" %
757 (name, result.fail_reason, result.output))
758 if not self._InstancePidAlive(name)[2]:
759 raise errors.HypervisorError("Failed to start instance %s" % name)
761 def _ExecuteKVMRuntime(self, instance, kvm_runtime, incoming=None):
762 """Execute a KVM cmd, after completing it with some last minute data
764 @type incoming: tuple of strings
765 @param incoming: (target_host_ip, port)
768 # Small _ExecuteKVMRuntime hv parameters programming howto:
769 # - conf_hvp contains the parameters as configured on ganeti. they might
770 # have changed since the instance started; only use them if the change
771 # won't affect the inside of the instance (which hasn't been rebooted).
772 # - up_hvp contains the parameters as they were when the instance was
773 # started, plus any new parameter which has been added between ganeti
774 # versions: it is paramount that those default to a value which won't
775 # affect the inside of the instance as well.
776 conf_hvp = instance.hvparams
778 self._CheckDown(name)
782 kvm_cmd, kvm_nics, up_hvp = kvm_runtime
783 up_hvp = objects.FillDict(conf_hvp, up_hvp)
785 kvm_version = self._GetKVMVersion()
787 _, v_major, v_min, _ = kvm_version
789 raise errors.HypervisorError("Unable to get KVM version")
791 # We know it's safe to run as a different user upon migration, so we'll use
792 # the latest conf, from conf_hvp.
793 security_model = conf_hvp[constants.HV_SECURITY_MODEL]
794 if security_model == constants.HT_SM_USER:
795 kvm_cmd.extend(["-runas", conf_hvp[constants.HV_SECURITY_DOMAIN]])
797 # We have reasons to believe changing something like the nic driver/type
798 # upon migration won't exactly fly with the instance kernel, so for nic
799 # related parameters we'll use up_hvp
803 kvm_cmd.extend(["-net", "none"])
807 nic_type = up_hvp[constants.HV_NIC_TYPE]
808 if nic_type == constants.HT_NIC_PARAVIRTUAL:
809 # From version 0.12.0, kvm uses a new sintax for network configuration.
810 if (v_major, v_min) >= (0, 12):
811 nic_model = "virtio-net-pci"
816 if up_hvp[constants.HV_VHOST_NET]:
817 # vhost_net is only available from version 0.13.0 or newer
818 if (v_major, v_min) >= (0, 13):
819 tap_extra = ",vhost=on"
821 raise errors.HypervisorError("vhost_net is configured"
822 " but it is not available")
826 for nic_seq, nic in enumerate(kvm_nics):
827 tapname, tapfd = _OpenTap(vnet_hdr)
830 if (v_major, v_min) >= (0, 12):
831 nic_val = "%s,mac=%s,netdev=netdev%s" % (nic_model, nic.mac, nic_seq)
832 tap_val = "type=tap,id=netdev%s,fd=%d%s" % (nic_seq, tapfd, tap_extra)
833 kvm_cmd.extend(["-netdev", tap_val, "-device", nic_val])
835 nic_val = "nic,vlan=%s,macaddr=%s,model=%s" % (nic_seq,
837 tap_val = "tap,vlan=%s,fd=%d" % (nic_seq, tapfd)
838 kvm_cmd.extend(["-net", tap_val, "-net", nic_val])
841 target, port = incoming
842 kvm_cmd.extend(['-incoming', 'tcp:%s:%s' % (target, port)])
844 # Changing the vnc password doesn't bother the guest that much. At most it
845 # will surprise people who connect to it. Whether positively or negatively
847 vnc_pwd_file = conf_hvp[constants.HV_VNC_PASSWORD_FILE]
851 vnc_pwd = utils.ReadOneLineFile(vnc_pwd_file, strict=True)
852 except EnvironmentError, err:
853 raise errors.HypervisorError("Failed to open VNC password file %s: %s"
854 % (vnc_pwd_file, err))
856 if conf_hvp[constants.HV_KVM_USE_CHROOT]:
857 utils.EnsureDirs([(self._InstanceChrootDir(name),
858 constants.SECURE_DIR_MODE)])
861 # Configure the network now for starting instances, during
862 # FinalizeMigration for incoming instances
863 for nic_seq, nic in enumerate(kvm_nics):
864 self._ConfigureNIC(instance, nic_seq, nic, taps[nic_seq])
866 if security_model == constants.HT_SM_POOL:
867 ss = ssconf.SimpleStore()
868 uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\n")
869 all_uids = set(uidpool.ExpandUidPool(uid_pool))
870 uid = uidpool.RequestUnusedUid(all_uids)
872 username = pwd.getpwuid(uid.GetUid()).pw_name
873 kvm_cmd.extend(["-runas", username])
874 self._RunKVMCmd(name, kvm_cmd, tapfds)
876 uidpool.ReleaseUid(uid)
880 utils.WriteFile(self._InstanceUidFile(name), data=uid.AsStr())
882 self._RunKVMCmd(name, kvm_cmd, tapfds)
884 utils.EnsureDirs([(self._InstanceNICDir(instance.name),
885 constants.RUN_DIRS_MODE)])
886 for nic_seq, tap in enumerate(taps):
887 utils.WriteFile(self._InstanceNICFile(instance.name, nic_seq),
891 change_cmd = 'change vnc password %s' % vnc_pwd
892 self._CallMonitorCommand(instance.name, change_cmd)
894 for filename in temp_files:
895 utils.RemoveFile(filename)
897 def StartInstance(self, instance, block_devices):
898 """Start an instance.
901 self._CheckDown(instance.name)
902 kvm_runtime = self._GenerateKVMRuntime(instance, block_devices)
903 self._SaveKVMRuntime(instance, kvm_runtime)
904 self._ExecuteKVMRuntime(instance, kvm_runtime)
906 def _CallMonitorCommand(self, instance_name, command):
907 """Invoke a command on the instance monitor.
910 socat = ("echo %s | %s STDIO UNIX-CONNECT:%s" %
911 (utils.ShellQuote(command),
912 constants.SOCAT_PATH,
913 utils.ShellQuote(self._InstanceMonitor(instance_name))))
914 result = utils.RunCmd(socat)
916 msg = ("Failed to send command '%s' to instance %s."
917 " output: %s, error: %s, fail_reason: %s" %
918 (command, instance_name,
919 result.stdout, result.stderr, result.fail_reason))
920 raise errors.HypervisorError(msg)
925 def _GetKVMVersion(cls):
926 """Return the installed KVM version
928 @return: (version, v_maj, v_min, v_rev), or None
931 result = utils.RunCmd([constants.KVM_PATH, "--help"])
934 match = cls._VERSION_RE.search(result.output.splitlines()[0])
938 return (match.group(0), int(match.group(1)), int(match.group(2)),
941 def StopInstance(self, instance, force=False, retry=False, name=None):
945 if name is not None and not force:
946 raise errors.HypervisorError("Cannot shutdown cleanly by name only")
949 acpi = instance.hvparams[constants.HV_ACPI]
952 _, pid, alive = self._InstancePidAlive(name)
953 if pid > 0 and alive:
954 if force or not acpi:
955 utils.KillProcess(pid)
957 self._CallMonitorCommand(name, 'system_powerdown')
959 def CleanupInstance(self, instance_name):
960 """Cleanup after a stopped instance
963 pidfile, pid, alive = self._InstancePidAlive(instance_name)
964 if pid > 0 and alive:
965 raise errors.HypervisorError("Cannot cleanup a live instance")
966 self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
968 def RebootInstance(self, instance):
969 """Reboot an instance.
972 # For some reason if we do a 'send-key ctrl-alt-delete' to the control
973 # socket the instance will stop, but now power up again. So we'll resort
974 # to shutdown and restart.
975 _, _, alive = self._InstancePidAlive(instance.name)
977 raise errors.HypervisorError("Failed to reboot instance %s:"
978 " not running" % instance.name)
979 # StopInstance will delete the saved KVM runtime so:
980 # ...first load it...
981 kvm_runtime = self._LoadKVMRuntime(instance)
982 # ...now we can safely call StopInstance...
983 if not self.StopInstance(instance):
984 self.StopInstance(instance, force=True)
985 # ...and finally we can save it again, and execute it...
986 self._SaveKVMRuntime(instance, kvm_runtime)
987 self._ExecuteKVMRuntime(instance, kvm_runtime)
989 def MigrationInfo(self, instance):
990 """Get instance information to perform a migration.
992 @type instance: L{objects.Instance}
993 @param instance: instance to be migrated
995 @return: content of the KVM runtime file
998 return self._ReadKVMRuntime(instance.name)
1000 def AcceptInstance(self, instance, info, target):
1001 """Prepare to accept an instance.
1003 @type instance: L{objects.Instance}
1004 @param instance: instance to be accepted
1006 @param info: content of the KVM runtime file on the source node
1007 @type target: string
1008 @param target: target host (usually ip), on this node
1011 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
1012 incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT])
1013 self._ExecuteKVMRuntime(instance, kvm_runtime, incoming=incoming_address)
1015 def FinalizeMigration(self, instance, info, success):
1016 """Finalize an instance migration.
1018 Stop the incoming mode KVM.
1020 @type instance: L{objects.Instance}
1021 @param instance: instance whose migration is being finalized
1025 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
1026 kvm_nics = kvm_runtime[1]
1028 for nic_seq, nic in enumerate(kvm_nics):
1030 tap = utils.ReadFile(self._InstanceNICFile(instance.name, nic_seq))
1031 except EnvironmentError, err:
1032 logging.warning("Failed to find host interface for %s NIC #%d: %s",
1033 instance.name, nic_seq, str(err))
1036 self._ConfigureNIC(instance, nic_seq, nic, tap)
1037 except errors.HypervisorError, err:
1038 logging.warning(str(err))
1040 self._WriteKVMRuntime(instance.name, info)
1042 self.StopInstance(instance, force=True)
1044 def MigrateInstance(self, instance, target, live):
1045 """Migrate an instance to a target node.
1047 The migration will not be attempted if the instance is not
1050 @type instance: L{objects.Instance}
1051 @param instance: the instance to be migrated
1052 @type target: string
1053 @param target: ip address of the target node
1055 @param live: perform a live migration
1058 instance_name = instance.name
1059 port = instance.hvparams[constants.HV_MIGRATION_PORT]
1060 pidfile, pid, alive = self._InstancePidAlive(instance_name)
1062 raise errors.HypervisorError("Instance not running, cannot migrate")
1065 self._CallMonitorCommand(instance_name, 'stop')
1067 migrate_command = ('migrate_set_speed %dm' %
1068 instance.hvparams[constants.HV_MIGRATION_BANDWIDTH])
1069 self._CallMonitorCommand(instance_name, migrate_command)
1071 migrate_command = ('migrate_set_downtime %dms' %
1072 instance.hvparams[constants.HV_MIGRATION_DOWNTIME])
1073 self._CallMonitorCommand(instance_name, migrate_command)
1075 migrate_command = 'migrate -d tcp:%s:%s' % (target, port)
1076 self._CallMonitorCommand(instance_name, migrate_command)
1078 info_command = 'info migrate'
1082 result = self._CallMonitorCommand(instance_name, info_command)
1083 match = self._MIGRATION_STATUS_RE.search(result.stdout)
1086 if not result.stdout:
1087 logging.info("KVM: empty 'info migrate' result")
1089 logging.warning("KVM: unknown 'info migrate' result: %s",
1091 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1093 status = match.group(1)
1094 if status == 'completed':
1096 elif status == 'active':
1097 # reset the broken answers count
1099 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1100 elif status == 'failed' or status == 'cancelled':
1102 self._CallMonitorCommand(instance_name, 'cont')
1103 raise errors.HypervisorError("Migration %s at the kvm level" %
1106 logging.warning("KVM: unknown migration status '%s'", status)
1108 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1109 if broken_answers >= self._MIGRATION_INFO_MAX_BAD_ANSWERS:
1110 raise errors.HypervisorError("Too many 'info migrate' broken answers")
1112 utils.KillProcess(pid)
1113 self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
1115 def GetNodeInfo(self):
1116 """Return information about the node.
1118 This is just a wrapper over the base GetLinuxNodeInfo method.
1120 @return: a dict with the following keys (values in MiB):
1121 - memory_total: the total memory size on the node
1122 - memory_free: the available memory on the node for instances
1123 - memory_dom0: the memory used by the node itself, if available
1126 return self.GetLinuxNodeInfo()
1129 def GetInstanceConsole(cls, instance, hvparams, beparams):
1130 """Return a command for connecting to the console of an instance.
1133 if hvparams[constants.HV_SERIAL_CONSOLE]:
1134 cmd = [constants.SOCAT_PATH,
1135 "STDIO,%s" % cls._SocatUnixConsoleParams(),
1136 "UNIX-CONNECT:%s" % cls._InstanceSerial(instance.name)]
1137 return objects.InstanceConsole(instance=instance.name,
1138 kind=constants.CONS_SSH,
1139 host=instance.primary_node,
1140 user=constants.GANETI_RUNAS,
1143 vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS]
1144 if vnc_bind_address and instance.network_port > constants.VNC_BASE_PORT:
1145 display = instance.network_port - constants.VNC_BASE_PORT
1146 return objects.InstanceConsole(instance=instance.name,
1147 kind=constants.CONS_VNC,
1148 host=vnc_bind_address,
1149 port=instance.network_port,
1152 return objects.InstanceConsole(instance=instance.name,
1153 kind=constants.CONS_MESSAGE,
1154 message=("No serial shell for instance %s" %
1158 """Verify the hypervisor.
1160 Check that the binary exists.
1163 if not os.path.exists(constants.KVM_PATH):
1164 return "The kvm binary ('%s') does not exist." % constants.KVM_PATH
1165 if not os.path.exists(constants.SOCAT_PATH):
1166 return "The socat binary ('%s') does not exist." % constants.SOCAT_PATH
1169 def CheckParameterSyntax(cls, hvparams):
1170 """Check the given parameters for validity.
1172 @type hvparams: dict
1173 @param hvparams: dictionary with parameter names/value
1174 @raise errors.HypervisorError: when a parameter is not valid
1177 super(KVMHypervisor, cls).CheckParameterSyntax(hvparams)
1179 kernel_path = hvparams[constants.HV_KERNEL_PATH]
1181 if not hvparams[constants.HV_ROOT_PATH]:
1182 raise errors.HypervisorError("Need a root partition for the instance,"
1183 " if a kernel is defined")
1185 if (hvparams[constants.HV_VNC_X509_VERIFY] and
1186 not hvparams[constants.HV_VNC_X509]):
1187 raise errors.HypervisorError("%s must be defined, if %s is" %
1188 (constants.HV_VNC_X509,
1189 constants.HV_VNC_X509_VERIFY))
1191 boot_order = hvparams[constants.HV_BOOT_ORDER]
1192 if (boot_order == constants.HT_BO_CDROM and
1193 not hvparams[constants.HV_CDROM_IMAGE_PATH]):
1194 raise errors.HypervisorError("Cannot boot from cdrom without an"
1197 security_model = hvparams[constants.HV_SECURITY_MODEL]
1198 if security_model == constants.HT_SM_USER:
1199 if not hvparams[constants.HV_SECURITY_DOMAIN]:
1200 raise errors.HypervisorError("A security domain (user to run kvm as)"
1201 " must be specified")
1202 elif (security_model == constants.HT_SM_NONE or
1203 security_model == constants.HT_SM_POOL):
1204 if hvparams[constants.HV_SECURITY_DOMAIN]:
1205 raise errors.HypervisorError("Cannot have a security domain when the"
1206 " security model is 'none' or 'pool'")
1209 def ValidateParameters(cls, hvparams):
1210 """Check the given parameters for validity.
1212 @type hvparams: dict
1213 @param hvparams: dictionary with parameter names/value
1214 @raise errors.HypervisorError: when a parameter is not valid
1217 super(KVMHypervisor, cls).ValidateParameters(hvparams)
1219 security_model = hvparams[constants.HV_SECURITY_MODEL]
1220 if security_model == constants.HT_SM_USER:
1221 username = hvparams[constants.HV_SECURITY_DOMAIN]
1223 pwd.getpwnam(username)
1225 raise errors.HypervisorError("Unknown security domain user %s"
1229 def PowercycleNode(cls):
1230 """KVM powercycle, just a wrapper over Linux powercycle.
1233 cls.LinuxPowercycle()