X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/1feb39ec8cd916d5d0d7adde4eec651cde156da3..8a53b55f3a83f3bdf3b231f90e766fc15ec51895:/lib/hypervisor/hv_kvm.py diff --git a/lib/hypervisor/hv_kvm.py b/lib/hypervisor/hv_kvm.py index e4fd08d..3e9e025 100644 --- a/lib/hypervisor/hv_kvm.py +++ b/lib/hypervisor/hv_kvm.py @@ -1,7 +1,7 @@ # # -# Copyright (C) 2008 Google Inc. +# Copyright (C) 2008, 2009, 2010 Google Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,6 +23,7 @@ """ +import errno import os import os.path import re @@ -30,24 +31,122 @@ import tempfile import time import logging import pwd -from cStringIO import StringIO +import struct +import fcntl +import shutil from ganeti import utils from ganeti import constants from ganeti import errors from ganeti import serializer from ganeti import objects +from ganeti import uidpool +from ganeti import ssconf from ganeti.hypervisor import hv_base +from ganeti import netutils +from ganeti.utils import wrapper as utils_wrapper + + +_KVM_NETWORK_SCRIPT = constants.SYSCONFDIR + "/ganeti/kvm-vif-bridge" + +# TUN/TAP driver constants, taken from +# They are architecture-independent and already hardcoded in qemu-kvm source, +# so we can safely include them here. +TUNSETIFF = 0x400454ca +TUNGETIFF = 0x800454d2 +TUNGETFEATURES = 0x800454cf +IFF_TAP = 0x0002 +IFF_NO_PI = 0x1000 +IFF_VNET_HDR = 0x4000 + + +def _ProbeTapVnetHdr(fd): + """Check whether to enable the IFF_VNET_HDR flag. + + To do this, _all_ of the following conditions must be met: + 1. TUNGETFEATURES ioctl() *must* be implemented + 2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag + 3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in + drivers/net/tun.c there is no way to test this until after the tap device + has been created using TUNSETIFF, and there is no way to change the + IFF_VNET_HDR flag after creating the interface, catch-22! However both + TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27, + thus we can expect TUNGETIFF to be present if TUNGETFEATURES is. + + @type fd: int + @param fd: the file descriptor of /dev/net/tun + + """ + req = struct.pack("I", 0) + try: + res = fcntl.ioctl(fd, TUNGETFEATURES, req) + except EnvironmentError: + logging.warning("TUNGETFEATURES ioctl() not implemented") + return False + + tunflags = struct.unpack("I", res)[0] + if tunflags & IFF_VNET_HDR: + return True + else: + logging.warning("Host does not support IFF_VNET_HDR, not enabling") + return False + + +def _OpenTap(vnet_hdr=True): + """Open a new tap device and return its file descriptor. + + This is intended to be used by a qemu-type hypervisor together with the -net + tap,fd= command line parameter. + + @type vnet_hdr: boolean + @param vnet_hdr: Enable the VNET Header + @return: (ifname, tapfd) + @rtype: tuple + + """ + try: + tapfd = os.open("/dev/net/tun", os.O_RDWR) + except EnvironmentError: + raise errors.HypervisorError("Failed to open /dev/net/tun") + + flags = IFF_TAP | IFF_NO_PI + + if vnet_hdr and _ProbeTapVnetHdr(tapfd): + flags |= IFF_VNET_HDR + + # The struct ifreq ioctl request (see netdevice(7)) + ifr = struct.pack("16sh", "", flags) + + try: + res = fcntl.ioctl(tapfd, TUNSETIFF, ifr) + except EnvironmentError: + raise errors.HypervisorError("Failed to allocate a new TAP device") + + # Get the interface name from the ioctl + ifname = struct.unpack("16sh", res)[0].strip("\x00") + return (ifname, tapfd) class KVMHypervisor(hv_base.BaseHypervisor): """KVM hypervisor interface""" + CAN_MIGRATE = True _ROOT_DIR = constants.RUN_GANETI_DIR + "/kvm-hypervisor" _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids + _UIDS_DIR = _ROOT_DIR + "/uid" # contains instances reserved uids _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data - _DIRS = [_ROOT_DIR, _PIDS_DIR, _CTRL_DIR, _CONF_DIR] + _NICS_DIR = _ROOT_DIR + "/nic" # contains instances nic <-> tap associations + # KVM instances with chroot enabled are started in empty chroot directories. + _CHROOT_DIR = _ROOT_DIR + "/chroot" # for empty chroot directories + # After an instance is stopped, its chroot directory is removed. + # If the chroot directory is not empty, it can't be removed. + # A non-empty chroot directory indicates a possible security incident. + # To support forensics, the non-empty chroot directory is quarantined in + # a separate directory, called 'chroot-quarantine'. + _CHROOT_QUARANTINE_DIR = _ROOT_DIR + "/chroot-quarantine" + _DIRS = [_ROOT_DIR, _PIDS_DIR, _UIDS_DIR, _CTRL_DIR, _CONF_DIR, _NICS_DIR, + _CHROOT_DIR, _CHROOT_QUARANTINE_DIR] PARAMETERS = { constants.HV_KERNEL_PATH: hv_base.OPT_FILE_CHECK, @@ -57,7 +156,8 @@ class KVMHypervisor(hv_base.BaseHypervisor): constants.HV_ACPI: hv_base.NO_CHECK, constants.HV_SERIAL_CONSOLE: hv_base.NO_CHECK, constants.HV_VNC_BIND_ADDRESS: - (False, lambda x: (utils.IsValidIP(x) or utils.IsNormAbsPath(x)), + (False, lambda x: (netutils.IP4Address.IsValid(x) or + utils.IsNormAbsPath(x)), "the VNC bind address must be either a valid IP address or an absolute" " pathname", None, None), constants.HV_VNC_TLS: hv_base.NO_CHECK, @@ -74,12 +174,20 @@ class KVMHypervisor(hv_base.BaseHypervisor): constants.HV_USB_MOUSE: hv_base.ParamInSet(False, constants.HT_KVM_VALID_MOUSE_TYPES), constants.HV_MIGRATION_PORT: hv_base.NET_PORT_CHECK, + constants.HV_MIGRATION_BANDWIDTH: hv_base.NO_CHECK, + constants.HV_MIGRATION_DOWNTIME: hv_base.NO_CHECK, + constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK, constants.HV_USE_LOCALTIME: hv_base.NO_CHECK, constants.HV_DISK_CACHE: hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES), constants.HV_SECURITY_MODEL: hv_base.ParamInSet(True, constants.HT_KVM_VALID_SM_TYPES), constants.HV_SECURITY_DOMAIN: hv_base.NO_CHECK, + constants.HV_KVM_FLAG: + hv_base.ParamInSet(False, constants.HT_KVM_FLAG_VALUES), + constants.HV_VHOST_NET: hv_base.NO_CHECK, + constants.HV_KVM_USE_CHROOT: hv_base.NO_CHECK, + constants.HV_MEM_PATH: hv_base.OPT_DIR_CHECK, } _MIGRATION_STATUS_RE = re.compile('Migration\s+status:\s+(\w+)', @@ -87,7 +195,7 @@ class KVMHypervisor(hv_base.BaseHypervisor): _MIGRATION_INFO_MAX_BAD_ANSWERS = 5 _MIGRATION_INFO_RETRY_DELAY = 2 - _KVM_NETWORK_SCRIPT = constants.SYSCONFDIR + "/ganeti/kvm-vif-bridge" + _VERSION_RE = re.compile(r"\b(\d+)\.(\d+)\.(\d+)\b") ANCILLARY_FILES = [ _KVM_NETWORK_SCRIPT, @@ -107,13 +215,76 @@ class KVMHypervisor(hv_base.BaseHypervisor): """ return utils.PathJoin(cls._PIDS_DIR, instance_name) + @classmethod + def _InstanceUidFile(cls, instance_name): + """Returns the instance uidfile. + + """ + return utils.PathJoin(cls._UIDS_DIR, instance_name) + + @classmethod + def _InstancePidInfo(cls, pid): + """Check pid file for instance information. + + Check that a pid file is associated with an instance, and retrieve + information from its command line. + + @type pid: string or int + @param pid: process id of the instance to check + @rtype: tuple + @return: (instance_name, memory, vcpus) + @raise errors.HypervisorError: when an instance cannot be found + + """ + alive = utils.IsProcessAlive(pid) + if not alive: + raise errors.HypervisorError("Cannot get info for pid %s" % pid) + + cmdline_file = utils.PathJoin("/proc", str(pid), "cmdline") + try: + cmdline = utils.ReadFile(cmdline_file) + except EnvironmentError, err: + raise errors.HypervisorError("Can't open cmdline file for pid %s: %s" % + (pid, err)) + + instance = None + memory = 0 + vcpus = 0 + + arg_list = cmdline.split('\x00') + while arg_list: + arg = arg_list.pop(0) + if arg == "-name": + instance = arg_list.pop(0) + elif arg == "-m": + memory = int(arg_list.pop(0)) + elif arg == "-smp": + vcpus = int(arg_list.pop(0)) + + if instance is None: + raise errors.HypervisorError("Pid %s doesn't contain a ganeti kvm" + " instance" % pid) + + return (instance, memory, vcpus) + def _InstancePidAlive(self, instance_name): - """Returns the instance pid and pidfile + """Returns the instance pidfile, pid, and liveness. + + @type instance_name: string + @param instance_name: instance name + @rtype: tuple + @return: (pid file name, pid, liveness) """ pidfile = self._InstancePidFile(instance_name) pid = utils.ReadPidFile(pidfile) - alive = utils.IsProcessAlive(pid) + + alive = False + try: + cmd_instance = self._InstancePidInfo(pid)[0] + alive = (cmd_instance == instance_name) + except errors.HypervisorError: + pass return (pidfile, pid, alive) @@ -160,19 +331,83 @@ class KVMHypervisor(hv_base.BaseHypervisor): return utils.PathJoin(cls._CONF_DIR, "%s.runtime" % instance_name) @classmethod + def _InstanceChrootDir(cls, instance_name): + """Returns the name of the KVM chroot dir of the instance + + """ + return utils.PathJoin(cls._CHROOT_DIR, instance_name) + + @classmethod + def _InstanceNICDir(cls, instance_name): + """Returns the name of the directory holding the tap device files for a + given instance. + + """ + return utils.PathJoin(cls._NICS_DIR, instance_name) + + @classmethod + def _InstanceNICFile(cls, instance_name, seq): + """Returns the name of the file containing the tap device for a given NIC + + """ + return utils.PathJoin(cls._InstanceNICDir(instance_name), str(seq)) + + @classmethod + def _TryReadUidFile(cls, uid_file): + """Try to read a uid file + + """ + if os.path.exists(uid_file): + try: + uid = int(utils.ReadOneLineFile(uid_file)) + return uid + except EnvironmentError: + logging.warning("Can't read uid file", exc_info=True) + except (TypeError, ValueError): + logging.warning("Can't parse uid file contents", exc_info=True) + return None + + @classmethod def _RemoveInstanceRuntimeFiles(cls, pidfile, instance_name): - """Removes an instance's rutime sockets/files. + """Removes an instance's rutime sockets/files/dirs. """ utils.RemoveFile(pidfile) utils.RemoveFile(cls._InstanceMonitor(instance_name)) utils.RemoveFile(cls._InstanceSerial(instance_name)) utils.RemoveFile(cls._InstanceKVMRuntime(instance_name)) + uid_file = cls._InstanceUidFile(instance_name) + uid = cls._TryReadUidFile(uid_file) + utils.RemoveFile(uid_file) + if uid is not None: + uidpool.ReleaseUid(uid) + try: + shutil.rmtree(cls._InstanceNICDir(instance_name)) + except OSError, err: + if err.errno != errno.ENOENT: + raise + try: + chroot_dir = cls._InstanceChrootDir(instance_name) + utils.RemoveDir(chroot_dir) + except OSError, err: + if err.errno == errno.ENOTEMPTY: + # The chroot directory is expected to be empty, but it isn't. + new_chroot_dir = tempfile.mkdtemp(dir=cls._CHROOT_QUARANTINE_DIR, + prefix="%s-%s-" % + (instance_name, + utils.TimestampForFilename())) + logging.warning("The chroot directory of instance %s can not be" + " removed as it is not empty. Moving it to the" + " quarantine instead. Please investigate the" + " contents (%s) and clean up manually", + instance_name, new_chroot_dir) + utils.RenameFile(chroot_dir, new_chroot_dir) + else: + raise - def _WriteNetScript(self, instance, seq, nic): - """Write a script to connect a net interface to the proper bridge. - - This can be used by any qemu-type hypervisor. + @staticmethod + def _ConfigureNIC(instance, seq, nic, tap): + """Run the network configuration script for a specified NIC @param instance: instance we're acting on @type instance: instance object @@ -180,66 +415,40 @@ class KVMHypervisor(hv_base.BaseHypervisor): @type seq: int @param nic: nic we're acting on @type nic: nic object - @return: netscript file name - @rtype: string + @param tap: the host's tap interface this NIC corresponds to + @type tap: str """ - script = StringIO() - script.write("#!/bin/sh\n") - script.write("# this is autogenerated by Ganeti, please do not edit\n#\n") - script.write("PATH=$PATH:/sbin:/usr/sbin\n") - script.write("export INSTANCE=%s\n" % instance.name) - script.write("export MAC=%s\n" % nic.mac) + + if instance.tags: + tags = " ".join(instance.tags) + else: + tags = "" + + env = { + "PATH": "%s:/sbin:/usr/sbin" % os.environ["PATH"], + "INSTANCE": instance.name, + "MAC": nic.mac, + "MODE": nic.nicparams[constants.NIC_MODE], + "INTERFACE": tap, + "INTERFACE_INDEX": str(seq), + "TAGS": tags, + } + if nic.ip: - script.write("export IP=%s\n" % nic.ip) - script.write("export MODE=%s\n" % nic.nicparams[constants.NIC_MODE]) + env["IP"] = nic.ip + if nic.nicparams[constants.NIC_LINK]: - script.write("export LINK=%s\n" % nic.nicparams[constants.NIC_LINK]) - if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED: - script.write("export BRIDGE=%s\n" % nic.nicparams[constants.NIC_LINK]) - script.write("export INTERFACE=$1\n") - # TODO: make this configurable at ./configure time - script.write("if [ -x '%s' ]; then\n" % self._KVM_NETWORK_SCRIPT) - script.write(" # Execute the user-specific vif file\n") - script.write(" %s\n" % self._KVM_NETWORK_SCRIPT) - script.write("else\n") - script.write(" ifconfig $INTERFACE 0.0.0.0 up\n") + env["LINK"] = nic.nicparams[constants.NIC_LINK] + if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED: - script.write(" # Connect the interface to the bridge\n") - script.write(" brctl addif $BRIDGE $INTERFACE\n") - elif nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_ROUTED: - if not nic.ip: - raise errors.HypervisorError("nic/%d is routed, but has no ip." % seq) - script.write(" # Route traffic targeted at the IP to the interface\n") - if nic.nicparams[constants.NIC_LINK]: - script.write(" while ip rule del dev $INTERFACE; do :; done\n") - script.write(" ip rule add dev $INTERFACE table $LINK\n") - script.write(" ip route replace $IP table $LINK proto static" - " dev $INTERFACE\n") - else: - script.write(" ip route replace $IP proto static" - " dev $INTERFACE\n") - interface_v4_conf = "/proc/sys/net/ipv4/conf/$INTERFACE" - interface_v6_conf = "/proc/sys/net/ipv6/conf/$INTERFACE" - script.write(" if [ -d %s ]; then\n" % interface_v4_conf) - script.write(" echo 1 > %s/proxy_arp\n" % interface_v4_conf) - script.write(" echo 1 > %s/forwarding\n" % interface_v4_conf) - script.write(" fi\n") - script.write(" if [ -d %s ]; then\n" % interface_v6_conf) - script.write(" echo 1 > %s/proxy_ndp\n" % interface_v6_conf) - script.write(" echo 1 > %s/forwarding\n" % interface_v6_conf) - script.write(" fi\n") - script.write("fi\n\n") - # As much as we'd like to put this in our _ROOT_DIR, that will happen to be - # mounted noexec sometimes, so we'll have to find another place. - (tmpfd, tmpfile_name) = tempfile.mkstemp() - tmpfile = os.fdopen(tmpfd, 'w') - try: - tmpfile.write(script.getvalue()) - finally: - tmpfile.close() - os.chmod(tmpfile_name, 0755) - return tmpfile_name + env["BRIDGE"] = nic.nicparams[constants.NIC_LINK] + + result = utils.RunCmd([constants.KVM_IFUP, tap], env=env) + if result.failed: + raise errors.HypervisorError("Failed to configure interface %s: %s." + " Network configuration script output: %s" % + (tap, result.fail_reason, result.output)) def ListInstances(self): """Get the list of running instances. @@ -250,43 +459,27 @@ class KVMHypervisor(hv_base.BaseHypervisor): """ result = [] for name in os.listdir(self._PIDS_DIR): - filename = utils.PathJoin(self._PIDS_DIR, name) - if utils.IsProcessAlive(utils.ReadPidFile(filename)): + if self._InstancePidAlive(name)[2]: result.append(name) return result def GetInstanceInfo(self, instance_name): """Get instance properties. + @type instance_name: string @param instance_name: the instance name - - @return: tuple (name, id, memory, vcpus, stat, times) + @rtype: tuple of strings + @return: (name, id, memory, vcpus, stat, times) """ _, pid, alive = self._InstancePidAlive(instance_name) if not alive: return None - cmdline_file = utils.PathJoin("/proc", str(pid), "cmdline") - try: - cmdline = utils.ReadFile(cmdline_file) - except EnvironmentError, err: - raise errors.HypervisorError("Failed to list instance %s: %s" % - (instance_name, err)) - - memory = 0 - vcpus = 0 + _, memory, vcpus = self._InstancePidInfo(pid) stat = "---b-" times = "0" - arg_list = cmdline.split('\x00') - while arg_list: - arg = arg_list.pop(0) - if arg == '-m': - memory = int(arg_list.pop(0)) - elif arg == '-smp': - vcpus = int(arg_list.pop(0)) - return (instance_name, pid, memory, vcpus, stat, times) def GetAllInstancesInfo(self): @@ -297,15 +490,12 @@ class KVMHypervisor(hv_base.BaseHypervisor): """ data = [] for name in os.listdir(self._PIDS_DIR): - filename = utils.PathJoin(self._PIDS_DIR, name) - if utils.IsProcessAlive(utils.ReadPidFile(filename)): - try: - info = self.GetInstanceInfo(name) - except errors.HypervisorError: - continue - if info: - data.append(info) - + try: + info = self.GetInstanceInfo(name) + except errors.HypervisorError: + continue + if info: + data.append(info) return data def _GenerateKVMRuntime(self, instance, block_devices): @@ -329,9 +519,10 @@ class KVMHypervisor(hv_base.BaseHypervisor): boot_cdrom = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_CDROM boot_network = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_NETWORK - security_model = hvp[constants.HV_SECURITY_MODEL] - if security_model == constants.HT_SM_USER: - kvm_cmd.extend(['-runas', hvp[constants.HV_SECURITY_DOMAIN]]) + if hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED: + kvm_cmd.extend(["-enable-kvm"]) + elif hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_DISABLED: + kvm_cmd.extend(["-disable-kvm"]) if boot_network: kvm_cmd.extend(['-boot', 'n']) @@ -354,7 +545,10 @@ class KVMHypervisor(hv_base.BaseHypervisor): # TODO: handle FD_LOOP and FD_BLKTAP (?) if boot_disk: kvm_cmd.extend(['-boot', 'c']) - boot_val = ',boot=on' + if disk_type != constants.HT_DISK_IDE: + boot_val = ',boot=on' + else: + boot_val = '' # We only boot from the first disk boot_disk = False else: @@ -369,9 +563,14 @@ class KVMHypervisor(hv_base.BaseHypervisor): options = ',format=raw,media=cdrom' if boot_cdrom: kvm_cmd.extend(['-boot', 'd']) - options = '%s,boot=on' % options + if disk_type != constants.HT_DISK_IDE: + options = '%s,boot=on' % options else: - options = '%s,if=virtio' % options + if disk_type == constants.HT_DISK_PARAVIRTUAL: + if_val = ',if=virtio' + else: + if_val = ',if=%s' % disk_type + options = '%s%s' % (options, if_val) drive_val = 'file=%s%s' % (iso_image, options) kvm_cmd.extend(['-drive', drive_val]) @@ -387,17 +586,24 @@ class KVMHypervisor(hv_base.BaseHypervisor): root_append.append('console=ttyS0,38400') kvm_cmd.extend(['-append', ' '.join(root_append)]) + mem_path = hvp[constants.HV_MEM_PATH] + if mem_path: + kvm_cmd.extend(["-mem-path", mem_path, "-mem-prealloc"]) + mouse_type = hvp[constants.HV_USB_MOUSE] + vnc_bind_address = hvp[constants.HV_VNC_BIND_ADDRESS] + if mouse_type: kvm_cmd.extend(['-usb']) kvm_cmd.extend(['-usbdevice', mouse_type]) + elif vnc_bind_address: + kvm_cmd.extend(['-usbdevice', constants.HT_MOUSE_TABLET]) - vnc_bind_address = hvp[constants.HV_VNC_BIND_ADDRESS] if vnc_bind_address: - if utils.IsValidIP(vnc_bind_address): + if netutils.IP4Address.IsValid(vnc_bind_address): if instance.network_port > constants.VNC_BASE_PORT: display = instance.network_port - constants.VNC_BASE_PORT - if vnc_bind_address == '0.0.0.0': + if vnc_bind_address == constants.IP4_ADDRESS_ANY: vnc_arg = ':%d' % (display) else: vnc_arg = '%s:%d' % (vnc_bind_address, display) @@ -443,6 +649,9 @@ class KVMHypervisor(hv_base.BaseHypervisor): if hvp[constants.HV_USE_LOCALTIME]: kvm_cmd.extend(['-localtime']) + if hvp[constants.HV_KVM_USE_CHROOT]: + kvm_cmd.extend(['-chroot', self._InstanceChrootDir(instance.name)]) + # Save the current instance nics, but defer their expansion as parameters, # as we'll need to generate executable temp files for them. kvm_nics = instance.nics @@ -490,6 +699,29 @@ class KVMHypervisor(hv_base.BaseHypervisor): kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics] return (kvm_cmd, kvm_nics, hvparams) + def _RunKVMCmd(self, name, kvm_cmd, tap_fds=None): + """Run the KVM cmd and check for errors + + @type name: string + @param name: instance name + @type kvm_cmd: list of strings + @param kvm_cmd: runcmd input for kvm + @type tap_fds: list of int + @param tap_fds: fds of tap devices opened by Ganeti + + """ + try: + result = utils.RunCmd(kvm_cmd, noclose_fds=tap_fds) + finally: + for fd in tap_fds: + utils_wrapper.CloseFdNoError(fd) + + if result.failed: + raise errors.HypervisorError("Failed to start instance %s: %s (%s)" % + (name, result.fail_reason, result.output)) + if not self._InstancePidAlive(name)[2]: + raise errors.HypervisorError("Failed to start instance %s" % name) + def _ExecuteKVMRuntime(self, instance, kvm_runtime, incoming=None): """Execute a KVM cmd, after completing it with some last minute data @@ -497,50 +729,127 @@ class KVMHypervisor(hv_base.BaseHypervisor): @param incoming: (target_host_ip, port) """ - hvp = instance.hvparams + # Small _ExecuteKVMRuntime hv parameters programming howto: + # - conf_hvp contains the parameters as configured on ganeti. they might + # have changed since the instance started; only use them if the change + # won't affect the inside of the instance (which hasn't been rebooted). + # - up_hvp contains the parameters as they were when the instance was + # started, plus any new parameter which has been added between ganeti + # versions: it is paramount that those default to a value which won't + # affect the inside of the instance as well. + conf_hvp = instance.hvparams name = instance.name self._CheckDown(name) temp_files = [] - kvm_cmd, kvm_nics, hvparams = kvm_runtime + kvm_cmd, kvm_nics, up_hvp = kvm_runtime + up_hvp = objects.FillDict(conf_hvp, up_hvp) + kvm_version = self._GetKVMVersion() + if kvm_version: + _, v_major, v_min, _ = kvm_version + else: + raise errors.HypervisorError("Unable to get KVM version") + + # We know it's safe to run as a different user upon migration, so we'll use + # the latest conf, from conf_hvp. + security_model = conf_hvp[constants.HV_SECURITY_MODEL] + if security_model == constants.HT_SM_USER: + kvm_cmd.extend(["-runas", conf_hvp[constants.HV_SECURITY_DOMAIN]]) + + # We have reasons to believe changing something like the nic driver/type + # upon migration won't exactly fly with the instance kernel, so for nic + # related parameters we'll use up_hvp + tapfds = [] + taps = [] if not kvm_nics: - kvm_cmd.extend(['-net', 'none']) + kvm_cmd.extend(["-net", "none"]) else: - nic_type = hvparams[constants.HV_NIC_TYPE] + vnet_hdr = False + tap_extra = "" + nic_type = up_hvp[constants.HV_NIC_TYPE] if nic_type == constants.HT_NIC_PARAVIRTUAL: - nic_model = "model=virtio" + # From version 0.12.0, kvm uses a new sintax for network configuration. + if (v_major, v_min) >= (0, 12): + nic_model = "virtio-net-pci" + vnet_hdr = True + else: + nic_model = "virtio" + + if up_hvp[constants.HV_VHOST_NET]: + # vhost_net is only available from version 0.13.0 or newer + if (v_major, v_min) >= (0, 13): + tap_extra = ",vhost=on" + else: + raise errors.HypervisorError("vhost_net is configured" + " but it is not available") else: - nic_model = "model=%s" % nic_type + nic_model = nic_type for nic_seq, nic in enumerate(kvm_nics): - nic_val = "nic,vlan=%s,macaddr=%s,%s" % (nic_seq, nic.mac, nic_model) - script = self._WriteNetScript(instance, nic_seq, nic) - kvm_cmd.extend(['-net', nic_val]) - kvm_cmd.extend(['-net', 'tap,vlan=%s,script=%s' % (nic_seq, script)]) - temp_files.append(script) + tapname, tapfd = _OpenTap(vnet_hdr) + tapfds.append(tapfd) + taps.append(tapname) + if (v_major, v_min) >= (0, 12): + nic_val = "%s,mac=%s,netdev=netdev%s" % (nic_model, nic.mac, nic_seq) + tap_val = "type=tap,id=netdev%s,fd=%d%s" % (nic_seq, tapfd, tap_extra) + kvm_cmd.extend(["-netdev", tap_val, "-device", nic_val]) + else: + nic_val = "nic,vlan=%s,macaddr=%s,model=%s" % (nic_seq, + nic.mac, nic_model) + tap_val = "tap,vlan=%s,fd=%d" % (nic_seq, tapfd) + kvm_cmd.extend(["-net", tap_val, "-net", nic_val]) if incoming: target, port = incoming kvm_cmd.extend(['-incoming', 'tcp:%s:%s' % (target, port)]) - vnc_pwd_file = hvp[constants.HV_VNC_PASSWORD_FILE] + # Changing the vnc password doesn't bother the guest that much. At most it + # will surprise people who connect to it. Whether positively or negatively + # it's debatable. + vnc_pwd_file = conf_hvp[constants.HV_VNC_PASSWORD_FILE] vnc_pwd = None if vnc_pwd_file: try: - vnc_pwd = utils.ReadFile(vnc_pwd_file) + vnc_pwd = utils.ReadOneLineFile(vnc_pwd_file, strict=True) except EnvironmentError, err: raise errors.HypervisorError("Failed to open VNC password file %s: %s" % (vnc_pwd_file, err)) - result = utils.RunCmd(kvm_cmd) - if result.failed: - raise errors.HypervisorError("Failed to start instance %s: %s (%s)" % - (name, result.fail_reason, result.output)) + if conf_hvp[constants.HV_KVM_USE_CHROOT]: + utils.EnsureDirs([(self._InstanceChrootDir(name), + constants.SECURE_DIR_MODE)]) - if not self._InstancePidAlive(name)[2]: - raise errors.HypervisorError("Failed to start instance %s" % name) + if not incoming: + # Configure the network now for starting instances, during + # FinalizeMigration for incoming instances + for nic_seq, nic in enumerate(kvm_nics): + self._ConfigureNIC(instance, nic_seq, nic, taps[nic_seq]) + + if security_model == constants.HT_SM_POOL: + ss = ssconf.SimpleStore() + uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\n") + all_uids = set(uidpool.ExpandUidPool(uid_pool)) + uid = uidpool.RequestUnusedUid(all_uids) + try: + username = pwd.getpwuid(uid.GetUid()).pw_name + kvm_cmd.extend(["-runas", username]) + self._RunKVMCmd(name, kvm_cmd, tapfds) + except: + uidpool.ReleaseUid(uid) + raise + else: + uid.Unlock() + utils.WriteFile(self._InstanceUidFile(name), data=str(uid)) + else: + self._RunKVMCmd(name, kvm_cmd, tapfds) + + utils.EnsureDirs([(self._InstanceNICDir(instance.name), + constants.RUN_DIRS_MODE)]) + for nic_seq, tap in enumerate(taps): + utils.WriteFile(self._InstanceNICFile(instance.name, nic_seq), + data=tap) if vnc_pwd: change_cmd = 'change vnc password %s' % vnc_pwd @@ -576,22 +885,49 @@ class KVMHypervisor(hv_base.BaseHypervisor): return result - def StopInstance(self, instance, force=False, retry=False): + @classmethod + def _GetKVMVersion(cls): + """Return the installed KVM version + + @return: (version, v_maj, v_min, v_rev), or None + + """ + result = utils.RunCmd([constants.KVM_PATH, "--help"]) + if result.failed: + return None + match = cls._VERSION_RE.search(result.output.splitlines()[0]) + if not match: + return None + + return (match.group(0), int(match.group(1)), int(match.group(2)), + int(match.group(3))) + + def StopInstance(self, instance, force=False, retry=False, name=None): """Stop an instance. """ - pidfile, pid, alive = self._InstancePidAlive(instance.name) + if name is not None and not force: + raise errors.HypervisorError("Cannot shutdown cleanly by name only") + if name is None: + name = instance.name + acpi = instance.hvparams[constants.HV_ACPI] + else: + acpi = False + _, pid, alive = self._InstancePidAlive(name) if pid > 0 and alive: - if force or not instance.hvparams[constants.HV_ACPI]: + if force or not acpi: utils.KillProcess(pid) else: - self._CallMonitorCommand(instance.name, 'system_powerdown') + self._CallMonitorCommand(name, 'system_powerdown') - if not utils.IsProcessAlive(pid): - self._RemoveInstanceRuntimeFiles(pidfile, instance.name) - return True - else: - return False + def CleanupInstance(self, instance_name): + """Cleanup after a stopped instance + + """ + pidfile, pid, alive = self._InstancePidAlive(instance_name) + if pid > 0 and alive: + raise errors.HypervisorError("Cannot cleanup a live instance") + self._RemoveInstanceRuntimeFiles(pidfile, instance_name) def RebootInstance(self, instance): """Reboot an instance. @@ -646,10 +982,25 @@ class KVMHypervisor(hv_base.BaseHypervisor): Stop the incoming mode KVM. @type instance: L{objects.Instance} - @param instance: instance whose migration is being aborted + @param instance: instance whose migration is being finalized """ if success: + kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info) + kvm_nics = kvm_runtime[1] + + for nic_seq, nic in enumerate(kvm_nics): + try: + tap = utils.ReadFile(self._InstanceNICFile(instance.name, nic_seq)) + except EnvironmentError, err: + logging.warning("Failed to find host interface for %s NIC #%d: %s", + instance.name, nic_seq, str(err)) + continue + try: + self._ConfigureNIC(instance, nic_seq, nic, tap) + except errors.HypervisorError, err: + logging.warning(str(err)) + self._WriteKVMRuntime(instance.name, info) else: self.StopInstance(instance, force=True) @@ -674,13 +1025,17 @@ class KVMHypervisor(hv_base.BaseHypervisor): if not alive: raise errors.HypervisorError("Instance not running, cannot migrate") - if not utils.TcpPing(target, port, live_port_needed=True): - raise errors.HypervisorError("Remote host %s not listening on port" - " %s, cannot migrate" % (target, port)) - if not live: self._CallMonitorCommand(instance_name, 'stop') + migrate_command = ('migrate_set_speed %dm' % + instance.hvparams[constants.HV_MIGRATION_BANDWIDTH]) + self._CallMonitorCommand(instance_name, migrate_command) + + migrate_command = ('migrate_set_downtime %dms' % + instance.hvparams[constants.HV_MIGRATION_DOWNTIME]) + self._CallMonitorCommand(instance_name, migrate_command) + migrate_command = 'migrate -d tcp:%s:%s' % (target, port) self._CallMonitorCommand(instance_name, migrate_command) @@ -735,28 +1090,33 @@ class KVMHypervisor(hv_base.BaseHypervisor): return self.GetLinuxNodeInfo() @classmethod - def GetShellCommandForConsole(cls, instance, hvparams, beparams): + def GetInstanceConsole(cls, instance, hvparams, beparams): """Return a command for connecting to the console of an instance. """ if hvparams[constants.HV_SERIAL_CONSOLE]: - shell_command = ("%s STDIO,%s UNIX-CONNECT:%s" % - (constants.SOCAT_PATH, cls._SocatUnixConsoleParams(), - utils.ShellQuote(cls._InstanceSerial(instance.name)))) - else: - shell_command = "echo 'No serial shell for instance %s'" % instance.name + cmd = [constants.SOCAT_PATH, + "STDIO,%s" % cls._SocatUnixConsoleParams(), + "UNIX-CONNECT:%s" % cls._InstanceSerial(instance.name)] + return objects.InstanceConsole(instance=instance.name, + kind=constants.CONS_SSH, + host=instance.primary_node, + user=constants.GANETI_RUNAS, + command=cmd) vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS] - if vnc_bind_address: - if instance.network_port > constants.VNC_BASE_PORT: - display = instance.network_port - constants.VNC_BASE_PORT - vnc_command = ("echo 'Instance has VNC listening on %s:%d" - " (display: %d)'" % (vnc_bind_address, - instance.network_port, - display)) - shell_command = "%s; %s" % (vnc_command, shell_command) - - return shell_command + if vnc_bind_address and instance.network_port > constants.VNC_BASE_PORT: + display = instance.network_port - constants.VNC_BASE_PORT + return objects.InstanceConsole(instance=instance.name, + kind=constants.CONS_VNC, + host=vnc_bind_address, + port=instance.network_port, + display=display) + + return objects.InstanceConsole(instance=instance.name, + kind=constants.CONS_MESSAGE, + message=("No serial shell for instance %s" % + instance.name)) def Verify(self): """Verify the hypervisor. @@ -769,7 +1129,6 @@ class KVMHypervisor(hv_base.BaseHypervisor): if not os.path.exists(constants.SOCAT_PATH): return "The socat binary ('%s') does not exist." % constants.SOCAT_PATH - @classmethod def CheckParameterSyntax(cls, hvparams): """Check the given parameters for validity. @@ -809,8 +1168,6 @@ class KVMHypervisor(hv_base.BaseHypervisor): if hvparams[constants.HV_SECURITY_DOMAIN]: raise errors.HypervisorError("Cannot have a security domain when the" " security model is 'none' or 'pool'") - if security_model == constants.HT_SM_POOL: - raise errors.HypervisorError("Security model pool is not supported yet") @classmethod def ValidateParameters(cls, hvparams):