4 # Copyright (C) 2008, 2009, 2010 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
36 from cStringIO import StringIO
38 from ganeti import utils
39 from ganeti import constants
40 from ganeti import errors
41 from ganeti import serializer
42 from ganeti import objects
43 from ganeti import uidpool
44 from ganeti import ssconf
45 from ganeti.hypervisor import hv_base
46 from ganeti import netutils
49 _KVM_NETWORK_SCRIPT = constants.SYSCONFDIR + "/ganeti/kvm-vif-bridge"
51 # TUN/TAP driver constants, taken from <linux/if_tun.h>
52 # They are architecture-independent and already hardcoded in qemu-kvm source,
53 # so we can safely include them here.
54 TUNSETIFF = 0x400454ca
55 TUNGETIFF = 0x800454d2
56 TUNGETFEATURES = 0x800454cf
62 def _ProbeTapVnetHdr(fd):
63 """Check whether to enable the IFF_VNET_HDR flag.
65 To do this, _all_ of the following conditions must be met:
66 1. TUNGETFEATURES ioctl() *must* be implemented
67 2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag
68 3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in
69 drivers/net/tun.c there is no way to test this until after the tap device
70 has been created using TUNSETIFF, and there is no way to change the
71 IFF_VNET_HDR flag after creating the interface, catch-22! However both
72 TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27,
73 thus we can expect TUNGETIFF to be present if TUNGETFEATURES is.
76 @param fd: the file descriptor of /dev/net/tun
79 req = struct.pack("I", 0)
81 res = fcntl.ioctl(fd, TUNGETFEATURES, req)
82 except EnvironmentError:
83 logging.warning("TUNGETFEATURES ioctl() not implemented")
86 tunflags = struct.unpack("I", res)[0]
87 if tunflags & IFF_VNET_HDR:
90 logging.warning("Host does not support IFF_VNET_HDR, not enabling")
94 def _OpenTap(vnet_hdr=True):
95 """Open a new tap device and return its file descriptor.
97 This is intended to be used by a qemu-type hypervisor together with the -net
98 tap,fd=<fd> command line parameter.
100 @type vnet_hdr: boolean
101 @param vnet_hdr: Enable the VNET Header
102 @return: (ifname, tapfd)
107 tapfd = os.open("/dev/net/tun", os.O_RDWR)
108 except EnvironmentError:
109 raise errors.HypervisorError("Failed to open /dev/net/tun")
111 flags = IFF_TAP | IFF_NO_PI
113 if vnet_hdr and _ProbeTapVnetHdr(tapfd):
114 flags |= IFF_VNET_HDR
116 # The struct ifreq ioctl request (see netdevice(7))
117 ifr = struct.pack("16sh", "", flags)
120 res = fcntl.ioctl(tapfd, TUNSETIFF, ifr)
121 except EnvironmentError:
122 raise errors.HypervisorError("Failed to allocate a new TAP device")
124 # Get the interface name from the ioctl
125 ifname = struct.unpack("16sh", res)[0].strip("\x00")
126 return (ifname, tapfd)
129 def _WriteNetScript(instance, nic, index):
130 """Write a script to connect a net interface to the proper bridge.
132 This can be used by any qemu-type hypervisor.
134 @type instance: L{objects.Instance}
135 @param instance: Instance object
136 @type nic: L{objects.NIC}
137 @param nic: NIC object
139 @param index: NIC index
145 tags = " ".join(instance.tags)
150 sw = utils.ShellWriter(buf)
151 sw.Write("#!/bin/sh")
152 sw.Write("# this is autogenerated by Ganeti, please do not edit")
153 sw.Write("export PATH=$PATH:/sbin:/usr/sbin")
154 sw.Write("export INSTANCE=%s", utils.ShellQuote(instance.name))
155 sw.Write("export MAC=%s", utils.ShellQuote(nic.mac))
156 sw.Write("export MODE=%s",
157 utils.ShellQuote(nic.nicparams[constants.NIC_MODE]))
158 sw.Write("export INTERFACE=\"$1\"")
159 sw.Write("export TAGS=%s", utils.ShellQuote(tags))
162 sw.Write("export IP=%s", utils.ShellQuote(nic.ip))
164 if nic.nicparams[constants.NIC_LINK]:
165 sw.Write("export LINK=%s",
166 utils.ShellQuote(nic.nicparams[constants.NIC_LINK]))
168 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
169 sw.Write("export BRIDGE=%s",
170 utils.ShellQuote(nic.nicparams[constants.NIC_LINK]))
172 # TODO: make this configurable at ./configure time
173 sw.Write("if [ -x %s ]; then", utils.ShellQuote(_KVM_NETWORK_SCRIPT))
176 sw.Write("# Execute the user-specific vif file")
177 sw.Write(_KVM_NETWORK_SCRIPT)
183 sw.Write("ifconfig $INTERFACE 0.0.0.0 up")
185 if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
186 sw.Write("# Connect the interface to the bridge")
187 sw.Write("brctl addif $BRIDGE $INTERFACE")
189 elif nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_ROUTED:
191 raise errors.HypervisorError("nic/%d is routed, but has no IP"
194 sw.Write("# Route traffic targeted at the IP to the interface")
195 if nic.nicparams[constants.NIC_LINK]:
196 sw.Write("while ip rule del dev $INTERFACE; do :; done")
197 sw.Write("ip rule add dev $INTERFACE table $LINK")
198 sw.Write("ip route replace $IP table $LINK proto static"
201 sw.Write("ip route replace $IP proto static dev $INTERFACE")
203 interface_v4_conf = "/proc/sys/net/ipv4/conf/$INTERFACE"
204 sw.Write(" if [ -d %s ]; then", interface_v4_conf)
207 sw.Write("echo 1 > %s/proxy_arp", interface_v4_conf)
208 sw.Write("echo 1 > %s/forwarding", interface_v4_conf)
213 interface_v6_conf = "/proc/sys/net/ipv6/conf/$INTERFACE"
214 sw.Write("if [ -d %s ]; then", interface_v6_conf)
217 sw.Write("echo 1 > %s/proxy_ndp", interface_v6_conf)
218 sw.Write("echo 1 > %s/forwarding", interface_v6_conf)
226 return buf.getvalue()
229 class KVMHypervisor(hv_base.BaseHypervisor):
230 """KVM hypervisor interface"""
233 _ROOT_DIR = constants.RUN_GANETI_DIR + "/kvm-hypervisor"
234 _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids
235 _UIDS_DIR = _ROOT_DIR + "/uid" # contains instances reserved uids
236 _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets
237 _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data
238 # KVM instances with chroot enabled are started in empty chroot directories.
239 _CHROOT_DIR = _ROOT_DIR + "/chroot" # for empty chroot directories
240 # After an instance is stopped, its chroot directory is removed.
241 # If the chroot directory is not empty, it can't be removed.
242 # A non-empty chroot directory indicates a possible security incident.
243 # To support forensics, the non-empty chroot directory is quarantined in
244 # a separate directory, called 'chroot-quarantine'.
245 _CHROOT_QUARANTINE_DIR = _ROOT_DIR + "/chroot-quarantine"
246 _DIRS = [_ROOT_DIR, _PIDS_DIR, _UIDS_DIR, _CTRL_DIR, _CONF_DIR,
247 _CHROOT_DIR, _CHROOT_QUARANTINE_DIR]
250 constants.HV_KERNEL_PATH: hv_base.OPT_FILE_CHECK,
251 constants.HV_INITRD_PATH: hv_base.OPT_FILE_CHECK,
252 constants.HV_ROOT_PATH: hv_base.NO_CHECK,
253 constants.HV_KERNEL_ARGS: hv_base.NO_CHECK,
254 constants.HV_ACPI: hv_base.NO_CHECK,
255 constants.HV_SERIAL_CONSOLE: hv_base.NO_CHECK,
256 constants.HV_VNC_BIND_ADDRESS:
257 (False, lambda x: (netutils.IP4Address.IsValid(x) or
258 utils.IsNormAbsPath(x)),
259 "the VNC bind address must be either a valid IP address or an absolute"
260 " pathname", None, None),
261 constants.HV_VNC_TLS: hv_base.NO_CHECK,
262 constants.HV_VNC_X509: hv_base.OPT_DIR_CHECK,
263 constants.HV_VNC_X509_VERIFY: hv_base.NO_CHECK,
264 constants.HV_VNC_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
265 constants.HV_CDROM_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
266 constants.HV_BOOT_ORDER:
267 hv_base.ParamInSet(True, constants.HT_KVM_VALID_BO_TYPES),
268 constants.HV_NIC_TYPE:
269 hv_base.ParamInSet(True, constants.HT_KVM_VALID_NIC_TYPES),
270 constants.HV_DISK_TYPE:
271 hv_base.ParamInSet(True, constants.HT_KVM_VALID_DISK_TYPES),
272 constants.HV_USB_MOUSE:
273 hv_base.ParamInSet(False, constants.HT_KVM_VALID_MOUSE_TYPES),
274 constants.HV_MIGRATION_PORT: hv_base.NET_PORT_CHECK,
275 constants.HV_MIGRATION_BANDWIDTH: hv_base.NO_CHECK,
276 constants.HV_MIGRATION_DOWNTIME: hv_base.NO_CHECK,
277 constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK,
278 constants.HV_USE_LOCALTIME: hv_base.NO_CHECK,
279 constants.HV_DISK_CACHE:
280 hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES),
281 constants.HV_SECURITY_MODEL:
282 hv_base.ParamInSet(True, constants.HT_KVM_VALID_SM_TYPES),
283 constants.HV_SECURITY_DOMAIN: hv_base.NO_CHECK,
284 constants.HV_KVM_FLAG:
285 hv_base.ParamInSet(False, constants.HT_KVM_FLAG_VALUES),
286 constants.HV_VHOST_NET: hv_base.NO_CHECK,
287 constants.HV_KVM_USE_CHROOT: hv_base.NO_CHECK,
288 constants.HV_MEM_PATH: hv_base.OPT_DIR_CHECK,
291 _MIGRATION_STATUS_RE = re.compile('Migration\s+status:\s+(\w+)',
293 _MIGRATION_INFO_MAX_BAD_ANSWERS = 5
294 _MIGRATION_INFO_RETRY_DELAY = 2
296 _VERSION_RE = re.compile(r"\b(\d+)\.(\d+)\.(\d+)\b")
303 hv_base.BaseHypervisor.__init__(self)
304 # Let's make sure the directories we need exist, even if the RUN_DIR lives
305 # in a tmpfs filesystem or has been otherwise wiped out.
306 dirs = [(dname, constants.RUN_DIRS_MODE) for dname in self._DIRS]
307 utils.EnsureDirs(dirs)
310 def _InstancePidFile(cls, instance_name):
311 """Returns the instance pidfile.
314 return utils.PathJoin(cls._PIDS_DIR, instance_name)
317 def _InstanceUidFile(cls, instance_name):
318 """Returns the instance uidfile.
321 return utils.PathJoin(cls._UIDS_DIR, instance_name)
324 def _InstancePidInfo(cls, pid):
325 """Check pid file for instance information.
327 Check that a pid file is associated with an instance, and retrieve
328 information from its command line.
330 @type pid: string or int
331 @param pid: process id of the instance to check
333 @return: (instance_name, memory, vcpus)
334 @raise errors.HypervisorError: when an instance cannot be found
337 alive = utils.IsProcessAlive(pid)
339 raise errors.HypervisorError("Cannot get info for pid %s" % pid)
341 cmdline_file = utils.PathJoin("/proc", str(pid), "cmdline")
343 cmdline = utils.ReadFile(cmdline_file)
344 except EnvironmentError, err:
345 raise errors.HypervisorError("Can't open cmdline file for pid %s: %s" %
352 arg_list = cmdline.split('\x00')
354 arg = arg_list.pop(0)
356 instance = arg_list.pop(0)
358 memory = int(arg_list.pop(0))
360 vcpus = int(arg_list.pop(0))
363 raise errors.HypervisorError("Pid %s doesn't contain a ganeti kvm"
366 return (instance, memory, vcpus)
368 def _InstancePidAlive(self, instance_name):
369 """Returns the instance pidfile, pid, and liveness.
371 @type instance_name: string
372 @param instance_name: instance name
374 @return: (pid file name, pid, liveness)
377 pidfile = self._InstancePidFile(instance_name)
378 pid = utils.ReadPidFile(pidfile)
382 cmd_instance = self._InstancePidInfo(pid)[0]
383 alive = (cmd_instance == instance_name)
384 except errors.HypervisorError:
387 return (pidfile, pid, alive)
389 def _CheckDown(self, instance_name):
390 """Raises an error unless the given instance is down.
393 alive = self._InstancePidAlive(instance_name)[2]
395 raise errors.HypervisorError("Failed to start instance %s: %s" %
396 (instance_name, "already running"))
399 def _InstanceMonitor(cls, instance_name):
400 """Returns the instance monitor socket name
403 return utils.PathJoin(cls._CTRL_DIR, "%s.monitor" % instance_name)
406 def _InstanceSerial(cls, instance_name):
407 """Returns the instance serial socket name
410 return utils.PathJoin(cls._CTRL_DIR, "%s.serial" % instance_name)
413 def _SocatUnixConsoleParams():
414 """Returns the correct parameters for socat
416 If we have a new-enough socat we can use raw mode with an escape character.
419 if constants.SOCAT_USE_ESCAPE:
420 return "raw,echo=0,escape=%s" % constants.SOCAT_ESCAPE_CODE
422 return "echo=0,icanon=0"
425 def _InstanceKVMRuntime(cls, instance_name):
426 """Returns the instance KVM runtime filename
429 return utils.PathJoin(cls._CONF_DIR, "%s.runtime" % instance_name)
432 def _InstanceChrootDir(cls, instance_name):
433 """Returns the name of the KVM chroot dir of the instance
436 return utils.PathJoin(cls._CHROOT_DIR, instance_name)
439 def _TryReadUidFile(cls, uid_file):
440 """Try to read a uid file
443 if os.path.exists(uid_file):
445 uid = int(utils.ReadOneLineFile(uid_file))
447 except EnvironmentError:
448 logging.warning("Can't read uid file", exc_info=True)
449 except (TypeError, ValueError):
450 logging.warning("Can't parse uid file contents", exc_info=True)
454 def _RemoveInstanceRuntimeFiles(cls, pidfile, instance_name):
455 """Removes an instance's rutime sockets/files/dirs.
458 utils.RemoveFile(pidfile)
459 utils.RemoveFile(cls._InstanceMonitor(instance_name))
460 utils.RemoveFile(cls._InstanceSerial(instance_name))
461 utils.RemoveFile(cls._InstanceKVMRuntime(instance_name))
462 uid_file = cls._InstanceUidFile(instance_name)
463 uid = cls._TryReadUidFile(uid_file)
464 utils.RemoveFile(uid_file)
466 uidpool.ReleaseUid(uid)
468 chroot_dir = cls._InstanceChrootDir(instance_name)
469 utils.RemoveDir(chroot_dir)
471 if err.errno == errno.ENOTEMPTY:
472 # The chroot directory is expected to be empty, but it isn't.
473 new_chroot_dir = tempfile.mkdtemp(dir=cls._CHROOT_QUARANTINE_DIR,
476 utils.TimestampForFilename()))
477 logging.warning("The chroot directory of instance %s can not be"
478 " removed as it is not empty. Moving it to the"
479 " quarantine instead. Please investigate the"
480 " contents (%s) and clean up manually",
481 instance_name, new_chroot_dir)
482 utils.RenameFile(chroot_dir, new_chroot_dir)
487 def _WriteNetScriptFile(instance, seq, nic):
488 """Write a script to connect a net interface to the proper bridge.
490 This can be used by any qemu-type hypervisor.
492 @param instance: instance we're acting on
493 @type instance: instance object
494 @param seq: nic sequence number
496 @param nic: nic we're acting on
497 @type nic: nic object
498 @return: netscript file name
502 script = _WriteNetScript(instance, nic, seq)
504 # As much as we'd like to put this in our _ROOT_DIR, that will happen to be
505 # mounted noexec sometimes, so we'll have to find another place.
506 (tmpfd, tmpfile_name) = tempfile.mkstemp()
507 tmpfile = os.fdopen(tmpfd, 'w')
509 tmpfile.write(script)
512 os.chmod(tmpfile_name, 0755)
515 def ListInstances(self):
516 """Get the list of running instances.
518 We can do this by listing our live instances directory and
519 checking whether the associated kvm process is still alive.
523 for name in os.listdir(self._PIDS_DIR):
524 if self._InstancePidAlive(name)[2]:
528 def GetInstanceInfo(self, instance_name):
529 """Get instance properties.
531 @type instance_name: string
532 @param instance_name: the instance name
533 @rtype: tuple of strings
534 @return: (name, id, memory, vcpus, stat, times)
537 _, pid, alive = self._InstancePidAlive(instance_name)
541 _, memory, vcpus = self._InstancePidInfo(pid)
545 return (instance_name, pid, memory, vcpus, stat, times)
547 def GetAllInstancesInfo(self):
548 """Get properties of all instances.
550 @return: list of tuples (name, id, memory, vcpus, stat, times)
554 for name in os.listdir(self._PIDS_DIR):
556 info = self.GetInstanceInfo(name)
557 except errors.HypervisorError:
563 def _GenerateKVMRuntime(self, instance, block_devices):
564 """Generate KVM information to start an instance.
567 pidfile = self._InstancePidFile(instance.name)
568 kvm = constants.KVM_PATH
570 # used just by the vnc server, if enabled
571 kvm_cmd.extend(['-name', instance.name])
572 kvm_cmd.extend(['-m', instance.beparams[constants.BE_MEMORY]])
573 kvm_cmd.extend(['-smp', instance.beparams[constants.BE_VCPUS]])
574 kvm_cmd.extend(['-pidfile', pidfile])
575 kvm_cmd.extend(['-daemonize'])
576 if not instance.hvparams[constants.HV_ACPI]:
577 kvm_cmd.extend(['-no-acpi'])
579 hvp = instance.hvparams
580 boot_disk = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
581 boot_cdrom = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_CDROM
582 boot_network = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_NETWORK
584 if hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED:
585 kvm_cmd.extend(["-enable-kvm"])
586 elif hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_DISABLED:
587 kvm_cmd.extend(["-disable-kvm"])
590 kvm_cmd.extend(['-boot', 'n'])
592 disk_type = hvp[constants.HV_DISK_TYPE]
593 if disk_type == constants.HT_DISK_PARAVIRTUAL:
594 if_val = ',if=virtio'
596 if_val = ',if=%s' % disk_type
598 disk_cache = hvp[constants.HV_DISK_CACHE]
599 if disk_cache != constants.HT_CACHE_DEFAULT:
600 cache_val = ",cache=%s" % disk_cache
603 for cfdev, dev_path in block_devices:
604 if cfdev.mode != constants.DISK_RDWR:
605 raise errors.HypervisorError("Instance has read-only disks which"
606 " are not supported by KVM")
607 # TODO: handle FD_LOOP and FD_BLKTAP (?)
609 kvm_cmd.extend(['-boot', 'c'])
610 if disk_type != constants.HT_DISK_IDE:
611 boot_val = ',boot=on'
614 # We only boot from the first disk
619 drive_val = 'file=%s,format=raw%s%s%s' % (dev_path, if_val, boot_val,
621 kvm_cmd.extend(['-drive', drive_val])
623 iso_image = hvp[constants.HV_CDROM_IMAGE_PATH]
625 options = ',format=raw,media=cdrom'
627 kvm_cmd.extend(['-boot', 'd'])
628 if disk_type != constants.HT_DISK_IDE:
629 options = '%s,boot=on' % options
631 if disk_type == constants.HT_DISK_PARAVIRTUAL:
632 if_val = ',if=virtio'
634 if_val = ',if=%s' % disk_type
635 options = '%s%s' % (options, if_val)
636 drive_val = 'file=%s%s' % (iso_image, options)
637 kvm_cmd.extend(['-drive', drive_val])
639 kernel_path = hvp[constants.HV_KERNEL_PATH]
641 kvm_cmd.extend(['-kernel', kernel_path])
642 initrd_path = hvp[constants.HV_INITRD_PATH]
644 kvm_cmd.extend(['-initrd', initrd_path])
645 root_append = ['root=%s' % hvp[constants.HV_ROOT_PATH],
646 hvp[constants.HV_KERNEL_ARGS]]
647 if hvp[constants.HV_SERIAL_CONSOLE]:
648 root_append.append('console=ttyS0,38400')
649 kvm_cmd.extend(['-append', ' '.join(root_append)])
651 mem_path = hvp[constants.HV_MEM_PATH]
653 kvm_cmd.extend(["-mem-path", mem_path, "-mem-prealloc"])
655 mouse_type = hvp[constants.HV_USB_MOUSE]
656 vnc_bind_address = hvp[constants.HV_VNC_BIND_ADDRESS]
659 kvm_cmd.extend(['-usb'])
660 kvm_cmd.extend(['-usbdevice', mouse_type])
661 elif vnc_bind_address:
662 kvm_cmd.extend(['-usbdevice', constants.HT_MOUSE_TABLET])
665 if netutils.IP4Address.IsValid(vnc_bind_address):
666 if instance.network_port > constants.VNC_BASE_PORT:
667 display = instance.network_port - constants.VNC_BASE_PORT
668 if vnc_bind_address == constants.IP4_ADDRESS_ANY:
669 vnc_arg = ':%d' % (display)
671 vnc_arg = '%s:%d' % (vnc_bind_address, display)
673 logging.error("Network port is not a valid VNC display (%d < %d)."
674 " Not starting VNC", instance.network_port,
675 constants.VNC_BASE_PORT)
678 # Only allow tls and other option when not binding to a file, for now.
679 # kvm/qemu gets confused otherwise about the filename to use.
681 if hvp[constants.HV_VNC_TLS]:
682 vnc_append = '%s,tls' % vnc_append
683 if hvp[constants.HV_VNC_X509_VERIFY]:
684 vnc_append = '%s,x509verify=%s' % (vnc_append,
685 hvp[constants.HV_VNC_X509])
686 elif hvp[constants.HV_VNC_X509]:
687 vnc_append = '%s,x509=%s' % (vnc_append,
688 hvp[constants.HV_VNC_X509])
689 if hvp[constants.HV_VNC_PASSWORD_FILE]:
690 vnc_append = '%s,password' % vnc_append
692 vnc_arg = '%s%s' % (vnc_arg, vnc_append)
695 vnc_arg = 'unix:%s/%s.vnc' % (vnc_bind_address, instance.name)
697 kvm_cmd.extend(['-vnc', vnc_arg])
699 kvm_cmd.extend(['-nographic'])
701 monitor_dev = ("unix:%s,server,nowait" %
702 self._InstanceMonitor(instance.name))
703 kvm_cmd.extend(['-monitor', monitor_dev])
704 if hvp[constants.HV_SERIAL_CONSOLE]:
705 serial_dev = ('unix:%s,server,nowait' %
706 self._InstanceSerial(instance.name))
707 kvm_cmd.extend(['-serial', serial_dev])
709 kvm_cmd.extend(['-serial', 'none'])
711 if hvp[constants.HV_USE_LOCALTIME]:
712 kvm_cmd.extend(['-localtime'])
714 if hvp[constants.HV_KVM_USE_CHROOT]:
715 kvm_cmd.extend(['-chroot', self._InstanceChrootDir(instance.name)])
717 # Save the current instance nics, but defer their expansion as parameters,
718 # as we'll need to generate executable temp files for them.
719 kvm_nics = instance.nics
722 return (kvm_cmd, kvm_nics, hvparams)
724 def _WriteKVMRuntime(self, instance_name, data):
725 """Write an instance's KVM runtime
729 utils.WriteFile(self._InstanceKVMRuntime(instance_name),
731 except EnvironmentError, err:
732 raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err)
734 def _ReadKVMRuntime(self, instance_name):
735 """Read an instance's KVM runtime
739 file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name))
740 except EnvironmentError, err:
741 raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err)
744 def _SaveKVMRuntime(self, instance, kvm_runtime):
745 """Save an instance's KVM runtime
748 kvm_cmd, kvm_nics, hvparams = kvm_runtime
749 serialized_nics = [nic.ToDict() for nic in kvm_nics]
750 serialized_form = serializer.Dump((kvm_cmd, serialized_nics, hvparams))
751 self._WriteKVMRuntime(instance.name, serialized_form)
753 def _LoadKVMRuntime(self, instance, serialized_runtime=None):
754 """Load an instance's KVM runtime
757 if not serialized_runtime:
758 serialized_runtime = self._ReadKVMRuntime(instance.name)
759 loaded_runtime = serializer.Load(serialized_runtime)
760 kvm_cmd, serialized_nics, hvparams = loaded_runtime
761 kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics]
762 return (kvm_cmd, kvm_nics, hvparams)
764 def _RunKVMCmd(self, name, kvm_cmd):
765 """Run the KVM cmd and check for errors
768 @param name: instance name
769 @type kvm_cmd: list of strings
770 @param kvm_cmd: runcmd input for kvm
773 result = utils.RunCmd(kvm_cmd)
775 raise errors.HypervisorError("Failed to start instance %s: %s (%s)" %
776 (name, result.fail_reason, result.output))
777 if not self._InstancePidAlive(name)[2]:
778 raise errors.HypervisorError("Failed to start instance %s" % name)
780 def _ExecuteKVMRuntime(self, instance, kvm_runtime, incoming=None):
781 """Execute a KVM cmd, after completing it with some last minute data
783 @type incoming: tuple of strings
784 @param incoming: (target_host_ip, port)
787 # Small _ExecuteKVMRuntime hv parameters programming howto:
788 # - conf_hvp contains the parameters as configured on ganeti. they might
789 # have changed since the instance started; only use them if the change
790 # won't affect the inside of the instance (which hasn't been rebooted).
791 # - up_hvp contains the parameters as they were when the instance was
792 # started, plus any new parameter which has been added between ganeti
793 # versions: it is paramount that those default to a value which won't
794 # affect the inside of the instance as well.
795 conf_hvp = instance.hvparams
797 self._CheckDown(name)
801 kvm_cmd, kvm_nics, up_hvp = kvm_runtime
802 up_hvp = objects.FillDict(conf_hvp, up_hvp)
804 kvm_version = self._GetKVMVersion()
806 _, v_major, v_min, _ = kvm_version
808 raise errors.HypervisorError("Unable to get KVM version")
810 # We know it's safe to run as a different user upon migration, so we'll use
811 # the latest conf, from conf_hvp.
812 security_model = conf_hvp[constants.HV_SECURITY_MODEL]
813 if security_model == constants.HT_SM_USER:
814 kvm_cmd.extend(["-runas", conf_hvp[constants.HV_SECURITY_DOMAIN]])
816 # We have reasons to believe changing something like the nic driver/type
817 # upon migration won't exactly fly with the instance kernel, so for nic
818 # related parameters we'll use up_hvp
820 kvm_cmd.extend(["-net", "none"])
823 nic_type = up_hvp[constants.HV_NIC_TYPE]
824 if nic_type == constants.HT_NIC_PARAVIRTUAL:
825 # From version 0.12.0, kvm uses a new sintax for network configuration.
826 if (v_major, v_min) >= (0, 12):
827 nic_model = "virtio-net-pci"
831 if up_hvp[constants.HV_VHOST_NET]:
832 # vhost_net is only available from version 0.13.0 or newer
833 if (v_major, v_min) >= (0, 13):
834 tap_extra = ",vhost=on"
836 raise errors.HypervisorError("vhost_net is configured"
837 " but it is not available")
841 for nic_seq, nic in enumerate(kvm_nics):
842 script = self._WriteNetScriptFile(instance, nic_seq, nic)
843 temp_files.append(script)
844 if (v_major, v_min) >= (0, 12):
845 nic_val = "%s,mac=%s,netdev=netdev%s" % (nic_model, nic.mac, nic_seq)
846 tap_val = "type=tap,id=netdev%s,script=%s%s" % (nic_seq,
848 kvm_cmd.extend(["-netdev", tap_val, "-device", nic_val])
850 nic_val = "nic,vlan=%s,macaddr=%s,model=%s" % (nic_seq,
852 tap_val = "tap,vlan=%s,script=%s" % (nic_seq, script)
853 kvm_cmd.extend(["-net", tap_val, "-net", nic_val])
856 target, port = incoming
857 kvm_cmd.extend(['-incoming', 'tcp:%s:%s' % (target, port)])
859 # Changing the vnc password doesn't bother the guest that much. At most it
860 # will surprise people who connect to it. Whether positively or negatively
862 vnc_pwd_file = conf_hvp[constants.HV_VNC_PASSWORD_FILE]
866 vnc_pwd = utils.ReadOneLineFile(vnc_pwd_file, strict=True)
867 except EnvironmentError, err:
868 raise errors.HypervisorError("Failed to open VNC password file %s: %s"
869 % (vnc_pwd_file, err))
871 if conf_hvp[constants.HV_KVM_USE_CHROOT]:
872 utils.EnsureDirs([(self._InstanceChrootDir(name),
873 constants.SECURE_DIR_MODE)])
875 if security_model == constants.HT_SM_POOL:
876 ss = ssconf.SimpleStore()
877 uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\n")
878 all_uids = set(uidpool.ExpandUidPool(uid_pool))
879 uid = uidpool.RequestUnusedUid(all_uids)
881 username = pwd.getpwuid(uid.GetUid()).pw_name
882 kvm_cmd.extend(["-runas", username])
883 self._RunKVMCmd(name, kvm_cmd)
885 uidpool.ReleaseUid(uid)
889 utils.WriteFile(self._InstanceUidFile(name), data=str(uid))
891 self._RunKVMCmd(name, kvm_cmd)
894 change_cmd = 'change vnc password %s' % vnc_pwd
895 self._CallMonitorCommand(instance.name, change_cmd)
897 for filename in temp_files:
898 utils.RemoveFile(filename)
900 def StartInstance(self, instance, block_devices):
901 """Start an instance.
904 self._CheckDown(instance.name)
905 kvm_runtime = self._GenerateKVMRuntime(instance, block_devices)
906 self._SaveKVMRuntime(instance, kvm_runtime)
907 self._ExecuteKVMRuntime(instance, kvm_runtime)
909 def _CallMonitorCommand(self, instance_name, command):
910 """Invoke a command on the instance monitor.
913 socat = ("echo %s | %s STDIO UNIX-CONNECT:%s" %
914 (utils.ShellQuote(command),
915 constants.SOCAT_PATH,
916 utils.ShellQuote(self._InstanceMonitor(instance_name))))
917 result = utils.RunCmd(socat)
919 msg = ("Failed to send command '%s' to instance %s."
920 " output: %s, error: %s, fail_reason: %s" %
921 (command, instance_name,
922 result.stdout, result.stderr, result.fail_reason))
923 raise errors.HypervisorError(msg)
928 def _GetKVMVersion(cls):
929 """Return the installed KVM version
931 @return: (version, v_maj, v_min, v_rev), or None
934 result = utils.RunCmd([constants.KVM_PATH, "--help"])
937 match = cls._VERSION_RE.search(result.output.splitlines()[0])
941 return (match.group(0), int(match.group(1)), int(match.group(2)),
944 def StopInstance(self, instance, force=False, retry=False, name=None):
948 if name is not None and not force:
949 raise errors.HypervisorError("Cannot shutdown cleanly by name only")
952 acpi = instance.hvparams[constants.HV_ACPI]
955 _, pid, alive = self._InstancePidAlive(name)
956 if pid > 0 and alive:
957 if force or not acpi:
958 utils.KillProcess(pid)
960 self._CallMonitorCommand(name, 'system_powerdown')
962 def CleanupInstance(self, instance_name):
963 """Cleanup after a stopped instance
966 pidfile, pid, alive = self._InstancePidAlive(instance_name)
967 if pid > 0 and alive:
968 raise errors.HypervisorError("Cannot cleanup a live instance")
969 self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
971 def RebootInstance(self, instance):
972 """Reboot an instance.
975 # For some reason if we do a 'send-key ctrl-alt-delete' to the control
976 # socket the instance will stop, but now power up again. So we'll resort
977 # to shutdown and restart.
978 _, _, alive = self._InstancePidAlive(instance.name)
980 raise errors.HypervisorError("Failed to reboot instance %s:"
981 " not running" % instance.name)
982 # StopInstance will delete the saved KVM runtime so:
983 # ...first load it...
984 kvm_runtime = self._LoadKVMRuntime(instance)
985 # ...now we can safely call StopInstance...
986 if not self.StopInstance(instance):
987 self.StopInstance(instance, force=True)
988 # ...and finally we can save it again, and execute it...
989 self._SaveKVMRuntime(instance, kvm_runtime)
990 self._ExecuteKVMRuntime(instance, kvm_runtime)
992 def MigrationInfo(self, instance):
993 """Get instance information to perform a migration.
995 @type instance: L{objects.Instance}
996 @param instance: instance to be migrated
998 @return: content of the KVM runtime file
1001 return self._ReadKVMRuntime(instance.name)
1003 def AcceptInstance(self, instance, info, target):
1004 """Prepare to accept an instance.
1006 @type instance: L{objects.Instance}
1007 @param instance: instance to be accepted
1009 @param info: content of the KVM runtime file on the source node
1010 @type target: string
1011 @param target: target host (usually ip), on this node
1014 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
1015 incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT])
1016 self._ExecuteKVMRuntime(instance, kvm_runtime, incoming=incoming_address)
1018 def FinalizeMigration(self, instance, info, success):
1019 """Finalize an instance migration.
1021 Stop the incoming mode KVM.
1023 @type instance: L{objects.Instance}
1024 @param instance: instance whose migration is being finalized
1028 self._WriteKVMRuntime(instance.name, info)
1030 self.StopInstance(instance, force=True)
1032 def MigrateInstance(self, instance, target, live):
1033 """Migrate an instance to a target node.
1035 The migration will not be attempted if the instance is not
1038 @type instance: L{objects.Instance}
1039 @param instance: the instance to be migrated
1040 @type target: string
1041 @param target: ip address of the target node
1043 @param live: perform a live migration
1046 instance_name = instance.name
1047 port = instance.hvparams[constants.HV_MIGRATION_PORT]
1048 pidfile, pid, alive = self._InstancePidAlive(instance_name)
1050 raise errors.HypervisorError("Instance not running, cannot migrate")
1053 self._CallMonitorCommand(instance_name, 'stop')
1055 migrate_command = ('migrate_set_speed %dm' %
1056 instance.hvparams[constants.HV_MIGRATION_BANDWIDTH])
1057 self._CallMonitorCommand(instance_name, migrate_command)
1059 migrate_command = ('migrate_set_downtime %dms' %
1060 instance.hvparams[constants.HV_MIGRATION_DOWNTIME])
1061 self._CallMonitorCommand(instance_name, migrate_command)
1063 migrate_command = 'migrate -d tcp:%s:%s' % (target, port)
1064 self._CallMonitorCommand(instance_name, migrate_command)
1066 info_command = 'info migrate'
1070 result = self._CallMonitorCommand(instance_name, info_command)
1071 match = self._MIGRATION_STATUS_RE.search(result.stdout)
1074 if not result.stdout:
1075 logging.info("KVM: empty 'info migrate' result")
1077 logging.warning("KVM: unknown 'info migrate' result: %s",
1079 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1081 status = match.group(1)
1082 if status == 'completed':
1084 elif status == 'active':
1085 # reset the broken answers count
1087 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1088 elif status == 'failed' or status == 'cancelled':
1090 self._CallMonitorCommand(instance_name, 'cont')
1091 raise errors.HypervisorError("Migration %s at the kvm level" %
1094 logging.warning("KVM: unknown migration status '%s'", status)
1096 time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1097 if broken_answers >= self._MIGRATION_INFO_MAX_BAD_ANSWERS:
1098 raise errors.HypervisorError("Too many 'info migrate' broken answers")
1100 utils.KillProcess(pid)
1101 self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
1103 def GetNodeInfo(self):
1104 """Return information about the node.
1106 This is just a wrapper over the base GetLinuxNodeInfo method.
1108 @return: a dict with the following keys (values in MiB):
1109 - memory_total: the total memory size on the node
1110 - memory_free: the available memory on the node for instances
1111 - memory_dom0: the memory used by the node itself, if available
1114 return self.GetLinuxNodeInfo()
1117 def GetInstanceConsole(cls, instance, hvparams, beparams):
1118 """Return a command for connecting to the console of an instance.
1121 if hvparams[constants.HV_SERIAL_CONSOLE]:
1122 cmd = [constants.SOCAT_PATH,
1123 "STDIO,%s" % cls._SocatUnixConsoleParams(),
1124 "UNIX-CONNECT:%s" % cls._InstanceSerial(instance.name)]
1125 return objects.InstanceConsole(instance=instance.name,
1126 kind=constants.CONS_SSH,
1127 host=instance.primary_node,
1128 user=constants.GANETI_RUNAS,
1131 vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS]
1132 if vnc_bind_address and instance.network_port > constants.VNC_BASE_PORT:
1133 display = instance.network_port - constants.VNC_BASE_PORT
1134 return objects.InstanceConsole(instance=instance.name,
1135 kind=constants.CONS_VNC,
1136 host=vnc_bind_address,
1137 port=instance.network_port,
1140 return objects.InstanceConsole(instance=instance.name,
1141 kind=constants.CONS_MESSAGE,
1142 message=("No serial shell for instance %s" %
1146 """Verify the hypervisor.
1148 Check that the binary exists.
1151 if not os.path.exists(constants.KVM_PATH):
1152 return "The kvm binary ('%s') does not exist." % constants.KVM_PATH
1153 if not os.path.exists(constants.SOCAT_PATH):
1154 return "The socat binary ('%s') does not exist." % constants.SOCAT_PATH
1157 def CheckParameterSyntax(cls, hvparams):
1158 """Check the given parameters for validity.
1160 @type hvparams: dict
1161 @param hvparams: dictionary with parameter names/value
1162 @raise errors.HypervisorError: when a parameter is not valid
1165 super(KVMHypervisor, cls).CheckParameterSyntax(hvparams)
1167 kernel_path = hvparams[constants.HV_KERNEL_PATH]
1169 if not hvparams[constants.HV_ROOT_PATH]:
1170 raise errors.HypervisorError("Need a root partition for the instance,"
1171 " if a kernel is defined")
1173 if (hvparams[constants.HV_VNC_X509_VERIFY] and
1174 not hvparams[constants.HV_VNC_X509]):
1175 raise errors.HypervisorError("%s must be defined, if %s is" %
1176 (constants.HV_VNC_X509,
1177 constants.HV_VNC_X509_VERIFY))
1179 boot_order = hvparams[constants.HV_BOOT_ORDER]
1180 if (boot_order == constants.HT_BO_CDROM and
1181 not hvparams[constants.HV_CDROM_IMAGE_PATH]):
1182 raise errors.HypervisorError("Cannot boot from cdrom without an"
1185 security_model = hvparams[constants.HV_SECURITY_MODEL]
1186 if security_model == constants.HT_SM_USER:
1187 if not hvparams[constants.HV_SECURITY_DOMAIN]:
1188 raise errors.HypervisorError("A security domain (user to run kvm as)"
1189 " must be specified")
1190 elif (security_model == constants.HT_SM_NONE or
1191 security_model == constants.HT_SM_POOL):
1192 if hvparams[constants.HV_SECURITY_DOMAIN]:
1193 raise errors.HypervisorError("Cannot have a security domain when the"
1194 " security model is 'none' or 'pool'")
1197 def ValidateParameters(cls, hvparams):
1198 """Check the given parameters for validity.
1200 @type hvparams: dict
1201 @param hvparams: dictionary with parameter names/value
1202 @raise errors.HypervisorError: when a parameter is not valid
1205 super(KVMHypervisor, cls).ValidateParameters(hvparams)
1207 security_model = hvparams[constants.HV_SECURITY_MODEL]
1208 if security_model == constants.HT_SM_USER:
1209 username = hvparams[constants.HV_SECURITY_DOMAIN]
1211 pwd.getpwnam(username)
1213 raise errors.HypervisorError("Unknown security domain user %s"
1217 def PowercycleNode(cls):
1218 """KVM powercycle, just a wrapper over Linux powercycle.
1221 cls.LinuxPowercycle()