cluster verify and instance disks on offline nodes
[ganeti-local] / lib / hypervisor / hv_kvm.py
1 #
2 #
3
4 # Copyright (C) 2008, 2009, 2010 Google Inc.
5 #
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 # General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 # 02110-1301, USA.
20
21
22 """KVM hypervisor
23
24 """
25
26 import errno
27 import os
28 import os.path
29 import re
30 import tempfile
31 import time
32 import logging
33 import pwd
34 import struct
35 import fcntl
36 import shutil
37
38 from ganeti import utils
39 from ganeti import constants
40 from ganeti import errors
41 from ganeti import serializer
42 from ganeti import objects
43 from ganeti import uidpool
44 from ganeti import ssconf
45 from ganeti.hypervisor import hv_base
46 from ganeti import netutils
47 from ganeti.utils import wrapper as utils_wrapper
48
49
50 _KVM_NETWORK_SCRIPT = constants.SYSCONFDIR + "/ganeti/kvm-vif-bridge"
51
52 # TUN/TAP driver constants, taken from <linux/if_tun.h>
53 # They are architecture-independent and already hardcoded in qemu-kvm source,
54 # so we can safely include them here.
55 TUNSETIFF = 0x400454ca
56 TUNGETIFF = 0x800454d2
57 TUNGETFEATURES = 0x800454cf
58 IFF_TAP = 0x0002
59 IFF_NO_PI = 0x1000
60 IFF_VNET_HDR = 0x4000
61
62
63 def _ProbeTapVnetHdr(fd):
64   """Check whether to enable the IFF_VNET_HDR flag.
65
66   To do this, _all_ of the following conditions must be met:
67    1. TUNGETFEATURES ioctl() *must* be implemented
68    2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag
69    3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in
70       drivers/net/tun.c there is no way to test this until after the tap device
71       has been created using TUNSETIFF, and there is no way to change the
72       IFF_VNET_HDR flag after creating the interface, catch-22! However both
73       TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27,
74       thus we can expect TUNGETIFF to be present if TUNGETFEATURES is.
75
76    @type fd: int
77    @param fd: the file descriptor of /dev/net/tun
78
79   """
80   req = struct.pack("I", 0)
81   try:
82     res = fcntl.ioctl(fd, TUNGETFEATURES, req)
83   except EnvironmentError:
84     logging.warning("TUNGETFEATURES ioctl() not implemented")
85     return False
86
87   tunflags = struct.unpack("I", res)[0]
88   if tunflags & IFF_VNET_HDR:
89     return True
90   else:
91     logging.warning("Host does not support IFF_VNET_HDR, not enabling")
92     return False
93
94
95 def _OpenTap(vnet_hdr=True):
96   """Open a new tap device and return its file descriptor.
97
98   This is intended to be used by a qemu-type hypervisor together with the -net
99   tap,fd=<fd> command line parameter.
100
101   @type vnet_hdr: boolean
102   @param vnet_hdr: Enable the VNET Header
103   @return: (ifname, tapfd)
104   @rtype: tuple
105
106   """
107   try:
108     tapfd = os.open("/dev/net/tun", os.O_RDWR)
109   except EnvironmentError:
110     raise errors.HypervisorError("Failed to open /dev/net/tun")
111
112   flags = IFF_TAP | IFF_NO_PI
113
114   if vnet_hdr and _ProbeTapVnetHdr(tapfd):
115     flags |= IFF_VNET_HDR
116
117   # The struct ifreq ioctl request (see netdevice(7))
118   ifr = struct.pack("16sh", "", flags)
119
120   try:
121     res = fcntl.ioctl(tapfd, TUNSETIFF, ifr)
122   except EnvironmentError:
123     raise errors.HypervisorError("Failed to allocate a new TAP device")
124
125   # Get the interface name from the ioctl
126   ifname = struct.unpack("16sh", res)[0].strip("\x00")
127   return (ifname, tapfd)
128
129
130 class KVMHypervisor(hv_base.BaseHypervisor):
131   """KVM hypervisor interface"""
132   CAN_MIGRATE = True
133
134   _ROOT_DIR = constants.RUN_GANETI_DIR + "/kvm-hypervisor"
135   _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids
136   _UIDS_DIR = _ROOT_DIR + "/uid" # contains instances reserved uids
137   _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets
138   _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data
139   _NICS_DIR = _ROOT_DIR + "/nic" # contains instances nic <-> tap associations
140   # KVM instances with chroot enabled are started in empty chroot directories.
141   _CHROOT_DIR = _ROOT_DIR + "/chroot" # for empty chroot directories
142   # After an instance is stopped, its chroot directory is removed.
143   # If the chroot directory is not empty, it can't be removed.
144   # A non-empty chroot directory indicates a possible security incident.
145   # To support forensics, the non-empty chroot directory is quarantined in
146   # a separate directory, called 'chroot-quarantine'.
147   _CHROOT_QUARANTINE_DIR = _ROOT_DIR + "/chroot-quarantine"
148   _DIRS = [_ROOT_DIR, _PIDS_DIR, _UIDS_DIR, _CTRL_DIR, _CONF_DIR, _NICS_DIR,
149            _CHROOT_DIR, _CHROOT_QUARANTINE_DIR]
150
151   PARAMETERS = {
152     constants.HV_KERNEL_PATH: hv_base.OPT_FILE_CHECK,
153     constants.HV_INITRD_PATH: hv_base.OPT_FILE_CHECK,
154     constants.HV_ROOT_PATH: hv_base.NO_CHECK,
155     constants.HV_KERNEL_ARGS: hv_base.NO_CHECK,
156     constants.HV_ACPI: hv_base.NO_CHECK,
157     constants.HV_SERIAL_CONSOLE: hv_base.NO_CHECK,
158     constants.HV_VNC_BIND_ADDRESS:
159       (False, lambda x: (netutils.IP4Address.IsValid(x) or
160                          utils.IsNormAbsPath(x)),
161        "the VNC bind address must be either a valid IP address or an absolute"
162        " pathname", None, None),
163     constants.HV_VNC_TLS: hv_base.NO_CHECK,
164     constants.HV_VNC_X509: hv_base.OPT_DIR_CHECK,
165     constants.HV_VNC_X509_VERIFY: hv_base.NO_CHECK,
166     constants.HV_VNC_PASSWORD_FILE: hv_base.OPT_FILE_CHECK,
167     constants.HV_CDROM_IMAGE_PATH: hv_base.OPT_FILE_CHECK,
168     constants.HV_BOOT_ORDER:
169       hv_base.ParamInSet(True, constants.HT_KVM_VALID_BO_TYPES),
170     constants.HV_NIC_TYPE:
171       hv_base.ParamInSet(True, constants.HT_KVM_VALID_NIC_TYPES),
172     constants.HV_DISK_TYPE:
173       hv_base.ParamInSet(True, constants.HT_KVM_VALID_DISK_TYPES),
174     constants.HV_USB_MOUSE:
175       hv_base.ParamInSet(False, constants.HT_KVM_VALID_MOUSE_TYPES),
176     constants.HV_MIGRATION_PORT: hv_base.NET_PORT_CHECK,
177     constants.HV_MIGRATION_BANDWIDTH: hv_base.NO_CHECK,
178     constants.HV_MIGRATION_DOWNTIME: hv_base.NO_CHECK,
179     constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK,
180     constants.HV_USE_LOCALTIME: hv_base.NO_CHECK,
181     constants.HV_DISK_CACHE:
182       hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES),
183     constants.HV_SECURITY_MODEL:
184       hv_base.ParamInSet(True, constants.HT_KVM_VALID_SM_TYPES),
185     constants.HV_SECURITY_DOMAIN: hv_base.NO_CHECK,
186     constants.HV_KVM_FLAG:
187       hv_base.ParamInSet(False, constants.HT_KVM_FLAG_VALUES),
188     constants.HV_VHOST_NET: hv_base.NO_CHECK,
189     constants.HV_KVM_USE_CHROOT: hv_base.NO_CHECK,
190     constants.HV_MEM_PATH: hv_base.OPT_DIR_CHECK,
191     }
192
193   _MIGRATION_STATUS_RE = re.compile('Migration\s+status:\s+(\w+)',
194                                     re.M | re.I)
195   _MIGRATION_INFO_MAX_BAD_ANSWERS = 5
196   _MIGRATION_INFO_RETRY_DELAY = 2
197
198   _VERSION_RE = re.compile(r"\b(\d+)\.(\d+)\.(\d+)\b")
199
200   ANCILLARY_FILES = [
201     _KVM_NETWORK_SCRIPT,
202     ]
203
204   def __init__(self):
205     hv_base.BaseHypervisor.__init__(self)
206     # Let's make sure the directories we need exist, even if the RUN_DIR lives
207     # in a tmpfs filesystem or has been otherwise wiped out.
208     dirs = [(dname, constants.RUN_DIRS_MODE) for dname in self._DIRS]
209     utils.EnsureDirs(dirs)
210
211   @classmethod
212   def _InstancePidFile(cls, instance_name):
213     """Returns the instance pidfile.
214
215     """
216     return utils.PathJoin(cls._PIDS_DIR, instance_name)
217
218   @classmethod
219   def _InstanceUidFile(cls, instance_name):
220     """Returns the instance uidfile.
221
222     """
223     return utils.PathJoin(cls._UIDS_DIR, instance_name)
224
225   @classmethod
226   def _InstancePidInfo(cls, pid):
227     """Check pid file for instance information.
228
229     Check that a pid file is associated with an instance, and retrieve
230     information from its command line.
231
232     @type pid: string or int
233     @param pid: process id of the instance to check
234     @rtype: tuple
235     @return: (instance_name, memory, vcpus)
236     @raise errors.HypervisorError: when an instance cannot be found
237
238     """
239     alive = utils.IsProcessAlive(pid)
240     if not alive:
241       raise errors.HypervisorError("Cannot get info for pid %s" % pid)
242
243     cmdline_file = utils.PathJoin("/proc", str(pid), "cmdline")
244     try:
245       cmdline = utils.ReadFile(cmdline_file)
246     except EnvironmentError, err:
247       raise errors.HypervisorError("Can't open cmdline file for pid %s: %s" %
248                                    (pid, err))
249
250     instance = None
251     memory = 0
252     vcpus = 0
253
254     arg_list = cmdline.split('\x00')
255     while arg_list:
256       arg =  arg_list.pop(0)
257       if arg == "-name":
258         instance = arg_list.pop(0)
259       elif arg == "-m":
260         memory = int(arg_list.pop(0))
261       elif arg == "-smp":
262         vcpus = int(arg_list.pop(0))
263
264     if instance is None:
265       raise errors.HypervisorError("Pid %s doesn't contain a ganeti kvm"
266                                    " instance" % pid)
267
268     return (instance, memory, vcpus)
269
270   def _InstancePidAlive(self, instance_name):
271     """Returns the instance pidfile, pid, and liveness.
272
273     @type instance_name: string
274     @param instance_name: instance name
275     @rtype: tuple
276     @return: (pid file name, pid, liveness)
277
278     """
279     pidfile = self._InstancePidFile(instance_name)
280     pid = utils.ReadPidFile(pidfile)
281
282     alive = False
283     try:
284       cmd_instance = self._InstancePidInfo(pid)[0]
285       alive = (cmd_instance == instance_name)
286     except errors.HypervisorError:
287       pass
288
289     return (pidfile, pid, alive)
290
291   def _CheckDown(self, instance_name):
292     """Raises an error unless the given instance is down.
293
294     """
295     alive = self._InstancePidAlive(instance_name)[2]
296     if alive:
297       raise errors.HypervisorError("Failed to start instance %s: %s" %
298                                    (instance_name, "already running"))
299
300   @classmethod
301   def _InstanceMonitor(cls, instance_name):
302     """Returns the instance monitor socket name
303
304     """
305     return utils.PathJoin(cls._CTRL_DIR, "%s.monitor" % instance_name)
306
307   @classmethod
308   def _InstanceSerial(cls, instance_name):
309     """Returns the instance serial socket name
310
311     """
312     return utils.PathJoin(cls._CTRL_DIR, "%s.serial" % instance_name)
313
314   @staticmethod
315   def _SocatUnixConsoleParams():
316     """Returns the correct parameters for socat
317
318     If we have a new-enough socat we can use raw mode with an escape character.
319
320     """
321     if constants.SOCAT_USE_ESCAPE:
322       return "raw,echo=0,escape=%s" % constants.SOCAT_ESCAPE_CODE
323     else:
324       return "echo=0,icanon=0"
325
326   @classmethod
327   def _InstanceKVMRuntime(cls, instance_name):
328     """Returns the instance KVM runtime filename
329
330     """
331     return utils.PathJoin(cls._CONF_DIR, "%s.runtime" % instance_name)
332
333   @classmethod
334   def _InstanceChrootDir(cls, instance_name):
335     """Returns the name of the KVM chroot dir of the instance
336
337     """
338     return utils.PathJoin(cls._CHROOT_DIR, instance_name)
339
340   @classmethod
341   def _InstanceNICDir(cls, instance_name):
342     """Returns the name of the directory holding the tap device files for a
343     given instance.
344
345     """
346     return utils.PathJoin(cls._NICS_DIR, instance_name)
347
348   @classmethod
349   def _InstanceNICFile(cls, instance_name, seq):
350     """Returns the name of the file containing the tap device for a given NIC
351
352     """
353     return utils.PathJoin(cls._InstanceNICDir(instance_name), str(seq))
354
355   @classmethod
356   def _TryReadUidFile(cls, uid_file):
357     """Try to read a uid file
358
359     """
360     if os.path.exists(uid_file):
361       try:
362         uid = int(utils.ReadOneLineFile(uid_file))
363         return uid
364       except EnvironmentError:
365         logging.warning("Can't read uid file", exc_info=True)
366       except (TypeError, ValueError):
367         logging.warning("Can't parse uid file contents", exc_info=True)
368     return None
369
370   @classmethod
371   def _RemoveInstanceRuntimeFiles(cls, pidfile, instance_name):
372     """Removes an instance's rutime sockets/files/dirs.
373
374     """
375     utils.RemoveFile(pidfile)
376     utils.RemoveFile(cls._InstanceMonitor(instance_name))
377     utils.RemoveFile(cls._InstanceSerial(instance_name))
378     utils.RemoveFile(cls._InstanceKVMRuntime(instance_name))
379     uid_file = cls._InstanceUidFile(instance_name)
380     uid = cls._TryReadUidFile(uid_file)
381     utils.RemoveFile(uid_file)
382     if uid is not None:
383       uidpool.ReleaseUid(uid)
384     try:
385       shutil.rmtree(cls._InstanceNICDir(instance_name))
386     except OSError, err:
387       if err.errno != errno.ENOENT:
388         raise
389     try:
390       chroot_dir = cls._InstanceChrootDir(instance_name)
391       utils.RemoveDir(chroot_dir)
392     except OSError, err:
393       if err.errno == errno.ENOTEMPTY:
394         # The chroot directory is expected to be empty, but it isn't.
395         new_chroot_dir = tempfile.mkdtemp(dir=cls._CHROOT_QUARANTINE_DIR,
396                                           prefix="%s-%s-" %
397                                           (instance_name,
398                                            utils.TimestampForFilename()))
399         logging.warning("The chroot directory of instance %s can not be"
400                         " removed as it is not empty. Moving it to the"
401                         " quarantine instead. Please investigate the"
402                         " contents (%s) and clean up manually",
403                         instance_name, new_chroot_dir)
404         utils.RenameFile(chroot_dir, new_chroot_dir)
405       else:
406         raise
407
408   @staticmethod
409   def _ConfigureNIC(instance, seq, nic, tap):
410     """Run the network configuration script for a specified NIC
411
412     @param instance: instance we're acting on
413     @type instance: instance object
414     @param seq: nic sequence number
415     @type seq: int
416     @param nic: nic we're acting on
417     @type nic: nic object
418     @param tap: the host's tap interface this NIC corresponds to
419     @type tap: str
420
421     """
422
423     if instance.tags:
424       tags = " ".join(instance.tags)
425     else:
426       tags = ""
427
428     env = {
429       "PATH": "%s:/sbin:/usr/sbin" % os.environ["PATH"],
430       "INSTANCE": instance.name,
431       "MAC": nic.mac,
432       "MODE": nic.nicparams[constants.NIC_MODE],
433       "INTERFACE": tap,
434       "INTERFACE_INDEX": str(seq),
435       "TAGS": tags,
436     }
437
438     if nic.ip:
439       env["IP"] = nic.ip
440
441     if nic.nicparams[constants.NIC_LINK]:
442       env["LINK"] = nic.nicparams[constants.NIC_LINK]
443
444     if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
445       env["BRIDGE"] = nic.nicparams[constants.NIC_LINK]
446
447     result = utils.RunCmd([constants.KVM_IFUP, tap], env=env)
448     if result.failed:
449       raise errors.HypervisorError("Failed to configure interface %s: %s."
450                                    " Network configuration script output: %s" %
451                                    (tap, result.fail_reason, result.output))
452
453   def ListInstances(self):
454     """Get the list of running instances.
455
456     We can do this by listing our live instances directory and
457     checking whether the associated kvm process is still alive.
458
459     """
460     result = []
461     for name in os.listdir(self._PIDS_DIR):
462       if self._InstancePidAlive(name)[2]:
463         result.append(name)
464     return result
465
466   def GetInstanceInfo(self, instance_name):
467     """Get instance properties.
468
469     @type instance_name: string
470     @param instance_name: the instance name
471     @rtype: tuple of strings
472     @return: (name, id, memory, vcpus, stat, times)
473
474     """
475     _, pid, alive = self._InstancePidAlive(instance_name)
476     if not alive:
477       return None
478
479     _, memory, vcpus = self._InstancePidInfo(pid)
480     stat = "---b-"
481     times = "0"
482
483     return (instance_name, pid, memory, vcpus, stat, times)
484
485   def GetAllInstancesInfo(self):
486     """Get properties of all instances.
487
488     @return: list of tuples (name, id, memory, vcpus, stat, times)
489
490     """
491     data = []
492     for name in os.listdir(self._PIDS_DIR):
493       try:
494         info = self.GetInstanceInfo(name)
495       except errors.HypervisorError:
496         continue
497       if info:
498         data.append(info)
499     return data
500
501   def _GenerateKVMRuntime(self, instance, block_devices):
502     """Generate KVM information to start an instance.
503
504     """
505     pidfile  = self._InstancePidFile(instance.name)
506     kvm = constants.KVM_PATH
507     kvm_cmd = [kvm]
508     # used just by the vnc server, if enabled
509     kvm_cmd.extend(['-name', instance.name])
510     kvm_cmd.extend(['-m', instance.beparams[constants.BE_MEMORY]])
511     kvm_cmd.extend(['-smp', instance.beparams[constants.BE_VCPUS]])
512     kvm_cmd.extend(['-pidfile', pidfile])
513     kvm_cmd.extend(['-daemonize'])
514     if not instance.hvparams[constants.HV_ACPI]:
515       kvm_cmd.extend(['-no-acpi'])
516
517     hvp = instance.hvparams
518     boot_disk = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_DISK
519     boot_cdrom = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_CDROM
520     boot_network = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_NETWORK
521
522     if hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED:
523       kvm_cmd.extend(["-enable-kvm"])
524     elif hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_DISABLED:
525       kvm_cmd.extend(["-disable-kvm"])
526
527     if boot_network:
528       kvm_cmd.extend(['-boot', 'n'])
529
530     disk_type = hvp[constants.HV_DISK_TYPE]
531     if disk_type == constants.HT_DISK_PARAVIRTUAL:
532       if_val = ',if=virtio'
533     else:
534       if_val = ',if=%s' % disk_type
535     # Cache mode
536     disk_cache = hvp[constants.HV_DISK_CACHE]
537     if disk_cache != constants.HT_CACHE_DEFAULT:
538       cache_val = ",cache=%s" % disk_cache
539     else:
540       cache_val = ""
541     for cfdev, dev_path in block_devices:
542       if cfdev.mode != constants.DISK_RDWR:
543         raise errors.HypervisorError("Instance has read-only disks which"
544                                      " are not supported by KVM")
545       # TODO: handle FD_LOOP and FD_BLKTAP (?)
546       if boot_disk:
547         kvm_cmd.extend(['-boot', 'c'])
548         if disk_type != constants.HT_DISK_IDE:
549           boot_val = ',boot=on'
550         else:
551           boot_val = ''
552         # We only boot from the first disk
553         boot_disk = False
554       else:
555         boot_val = ''
556
557       drive_val = 'file=%s,format=raw%s%s%s' % (dev_path, if_val, boot_val,
558                                                 cache_val)
559       kvm_cmd.extend(['-drive', drive_val])
560
561     iso_image = hvp[constants.HV_CDROM_IMAGE_PATH]
562     if iso_image:
563       options = ',format=raw,media=cdrom'
564       if boot_cdrom:
565         kvm_cmd.extend(['-boot', 'd'])
566         if disk_type != constants.HT_DISK_IDE:
567           options = '%s,boot=on' % options
568       else:
569         if disk_type == constants.HT_DISK_PARAVIRTUAL:
570           if_val = ',if=virtio'
571         else:
572           if_val = ',if=%s' % disk_type
573         options = '%s%s' % (options, if_val)
574       drive_val = 'file=%s%s' % (iso_image, options)
575       kvm_cmd.extend(['-drive', drive_val])
576
577     kernel_path = hvp[constants.HV_KERNEL_PATH]
578     if kernel_path:
579       kvm_cmd.extend(['-kernel', kernel_path])
580       initrd_path = hvp[constants.HV_INITRD_PATH]
581       if initrd_path:
582         kvm_cmd.extend(['-initrd', initrd_path])
583       root_append = ['root=%s' % hvp[constants.HV_ROOT_PATH],
584                      hvp[constants.HV_KERNEL_ARGS]]
585       if hvp[constants.HV_SERIAL_CONSOLE]:
586         root_append.append('console=ttyS0,38400')
587       kvm_cmd.extend(['-append', ' '.join(root_append)])
588
589     mem_path = hvp[constants.HV_MEM_PATH]
590     if mem_path:
591       kvm_cmd.extend(["-mem-path", mem_path, "-mem-prealloc"])
592
593     mouse_type = hvp[constants.HV_USB_MOUSE]
594     vnc_bind_address = hvp[constants.HV_VNC_BIND_ADDRESS]
595
596     if mouse_type:
597       kvm_cmd.extend(['-usb'])
598       kvm_cmd.extend(['-usbdevice', mouse_type])
599     elif vnc_bind_address:
600       kvm_cmd.extend(['-usbdevice', constants.HT_MOUSE_TABLET])
601
602     if vnc_bind_address:
603       if netutils.IP4Address.IsValid(vnc_bind_address):
604         if instance.network_port > constants.VNC_BASE_PORT:
605           display = instance.network_port - constants.VNC_BASE_PORT
606           if vnc_bind_address == constants.IP4_ADDRESS_ANY:
607             vnc_arg = ':%d' % (display)
608           else:
609             vnc_arg = '%s:%d' % (vnc_bind_address, display)
610         else:
611           logging.error("Network port is not a valid VNC display (%d < %d)."
612                         " Not starting VNC", instance.network_port,
613                         constants.VNC_BASE_PORT)
614           vnc_arg = 'none'
615
616         # Only allow tls and other option when not binding to a file, for now.
617         # kvm/qemu gets confused otherwise about the filename to use.
618         vnc_append = ''
619         if hvp[constants.HV_VNC_TLS]:
620           vnc_append = '%s,tls' % vnc_append
621           if hvp[constants.HV_VNC_X509_VERIFY]:
622             vnc_append = '%s,x509verify=%s' % (vnc_append,
623                                                hvp[constants.HV_VNC_X509])
624           elif hvp[constants.HV_VNC_X509]:
625             vnc_append = '%s,x509=%s' % (vnc_append,
626                                          hvp[constants.HV_VNC_X509])
627         if hvp[constants.HV_VNC_PASSWORD_FILE]:
628           vnc_append = '%s,password' % vnc_append
629
630         vnc_arg = '%s%s' % (vnc_arg, vnc_append)
631
632       else:
633         vnc_arg = 'unix:%s/%s.vnc' % (vnc_bind_address, instance.name)
634
635       kvm_cmd.extend(['-vnc', vnc_arg])
636     else:
637       kvm_cmd.extend(['-nographic'])
638
639     monitor_dev = ("unix:%s,server,nowait" %
640                    self._InstanceMonitor(instance.name))
641     kvm_cmd.extend(['-monitor', monitor_dev])
642     if hvp[constants.HV_SERIAL_CONSOLE]:
643       serial_dev = ('unix:%s,server,nowait' %
644                     self._InstanceSerial(instance.name))
645       kvm_cmd.extend(['-serial', serial_dev])
646     else:
647       kvm_cmd.extend(['-serial', 'none'])
648
649     if hvp[constants.HV_USE_LOCALTIME]:
650       kvm_cmd.extend(['-localtime'])
651
652     if hvp[constants.HV_KVM_USE_CHROOT]:
653       kvm_cmd.extend(['-chroot', self._InstanceChrootDir(instance.name)])
654
655     # Save the current instance nics, but defer their expansion as parameters,
656     # as we'll need to generate executable temp files for them.
657     kvm_nics = instance.nics
658     hvparams = hvp
659
660     return (kvm_cmd, kvm_nics, hvparams)
661
662   def _WriteKVMRuntime(self, instance_name, data):
663     """Write an instance's KVM runtime
664
665     """
666     try:
667       utils.WriteFile(self._InstanceKVMRuntime(instance_name),
668                       data=data)
669     except EnvironmentError, err:
670       raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err)
671
672   def _ReadKVMRuntime(self, instance_name):
673     """Read an instance's KVM runtime
674
675     """
676     try:
677       file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name))
678     except EnvironmentError, err:
679       raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err)
680     return file_content
681
682   def _SaveKVMRuntime(self, instance, kvm_runtime):
683     """Save an instance's KVM runtime
684
685     """
686     kvm_cmd, kvm_nics, hvparams = kvm_runtime
687     serialized_nics = [nic.ToDict() for nic in kvm_nics]
688     serialized_form = serializer.Dump((kvm_cmd, serialized_nics, hvparams))
689     self._WriteKVMRuntime(instance.name, serialized_form)
690
691   def _LoadKVMRuntime(self, instance, serialized_runtime=None):
692     """Load an instance's KVM runtime
693
694     """
695     if not serialized_runtime:
696       serialized_runtime = self._ReadKVMRuntime(instance.name)
697     loaded_runtime = serializer.Load(serialized_runtime)
698     kvm_cmd, serialized_nics, hvparams = loaded_runtime
699     kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics]
700     return (kvm_cmd, kvm_nics, hvparams)
701
702   def _RunKVMCmd(self, name, kvm_cmd, tap_fds=None):
703     """Run the KVM cmd and check for errors
704
705     @type name: string
706     @param name: instance name
707     @type kvm_cmd: list of strings
708     @param kvm_cmd: runcmd input for kvm
709     @type tap_fds: list of int
710     @param tap_fds: fds of tap devices opened by Ganeti
711
712     """
713     try:
714       result = utils.RunCmd(kvm_cmd, noclose_fds=tap_fds)
715     finally:
716       for fd in tap_fds:
717         utils_wrapper.CloseFdNoError(fd)
718
719     if result.failed:
720       raise errors.HypervisorError("Failed to start instance %s: %s (%s)" %
721                                    (name, result.fail_reason, result.output))
722     if not self._InstancePidAlive(name)[2]:
723       raise errors.HypervisorError("Failed to start instance %s" % name)
724
725   def _ExecuteKVMRuntime(self, instance, kvm_runtime, incoming=None):
726     """Execute a KVM cmd, after completing it with some last minute data
727
728     @type incoming: tuple of strings
729     @param incoming: (target_host_ip, port)
730
731     """
732     # Small _ExecuteKVMRuntime hv parameters programming howto:
733     #  - conf_hvp contains the parameters as configured on ganeti. they might
734     #    have changed since the instance started; only use them if the change
735     #    won't affect the inside of the instance (which hasn't been rebooted).
736     #  - up_hvp contains the parameters as they were when the instance was
737     #    started, plus any new parameter which has been added between ganeti
738     #    versions: it is paramount that those default to a value which won't
739     #    affect the inside of the instance as well.
740     conf_hvp = instance.hvparams
741     name = instance.name
742     self._CheckDown(name)
743
744     temp_files = []
745
746     kvm_cmd, kvm_nics, up_hvp = kvm_runtime
747     up_hvp = objects.FillDict(conf_hvp, up_hvp)
748
749     kvm_version = self._GetKVMVersion()
750     if kvm_version:
751       _, v_major, v_min, _ = kvm_version
752     else:
753       raise errors.HypervisorError("Unable to get KVM version")
754
755     # We know it's safe to run as a different user upon migration, so we'll use
756     # the latest conf, from conf_hvp.
757     security_model = conf_hvp[constants.HV_SECURITY_MODEL]
758     if security_model == constants.HT_SM_USER:
759       kvm_cmd.extend(["-runas", conf_hvp[constants.HV_SECURITY_DOMAIN]])
760
761     # We have reasons to believe changing something like the nic driver/type
762     # upon migration won't exactly fly with the instance kernel, so for nic
763     # related parameters we'll use up_hvp
764     tapfds = []
765     taps = []
766     if not kvm_nics:
767       kvm_cmd.extend(["-net", "none"])
768     else:
769       vnet_hdr = False
770       tap_extra = ""
771       nic_type = up_hvp[constants.HV_NIC_TYPE]
772       if nic_type == constants.HT_NIC_PARAVIRTUAL:
773         # From version 0.12.0, kvm uses a new sintax for network configuration.
774         if (v_major, v_min) >= (0, 12):
775           nic_model = "virtio-net-pci"
776           vnet_hdr = True
777         else:
778           nic_model = "virtio"
779
780         if up_hvp[constants.HV_VHOST_NET]:
781           # vhost_net is only available from version 0.13.0 or newer
782           if (v_major, v_min) >= (0, 13):
783             tap_extra = ",vhost=on"
784           else:
785             raise errors.HypervisorError("vhost_net is configured"
786                                         " but it is not available")
787       else:
788         nic_model = nic_type
789
790       for nic_seq, nic in enumerate(kvm_nics):
791         tapname, tapfd = _OpenTap(vnet_hdr)
792         tapfds.append(tapfd)
793         taps.append(tapname)
794         if (v_major, v_min) >= (0, 12):
795           nic_val = "%s,mac=%s,netdev=netdev%s" % (nic_model, nic.mac, nic_seq)
796           tap_val = "type=tap,id=netdev%s,fd=%d%s" % (nic_seq, tapfd, tap_extra)
797           kvm_cmd.extend(["-netdev", tap_val, "-device", nic_val])
798         else:
799           nic_val = "nic,vlan=%s,macaddr=%s,model=%s" % (nic_seq,
800                                                          nic.mac, nic_model)
801           tap_val = "tap,vlan=%s,fd=%d" % (nic_seq, tapfd)
802           kvm_cmd.extend(["-net", tap_val, "-net", nic_val])
803
804     if incoming:
805       target, port = incoming
806       kvm_cmd.extend(['-incoming', 'tcp:%s:%s' % (target, port)])
807
808     # Changing the vnc password doesn't bother the guest that much. At most it
809     # will surprise people who connect to it. Whether positively or negatively
810     # it's debatable.
811     vnc_pwd_file = conf_hvp[constants.HV_VNC_PASSWORD_FILE]
812     vnc_pwd = None
813     if vnc_pwd_file:
814       try:
815         vnc_pwd = utils.ReadOneLineFile(vnc_pwd_file, strict=True)
816       except EnvironmentError, err:
817         raise errors.HypervisorError("Failed to open VNC password file %s: %s"
818                                      % (vnc_pwd_file, err))
819
820     if conf_hvp[constants.HV_KVM_USE_CHROOT]:
821       utils.EnsureDirs([(self._InstanceChrootDir(name),
822                          constants.SECURE_DIR_MODE)])
823
824     if not incoming:
825       # Configure the network now for starting instances, during
826       # FinalizeMigration for incoming instances
827       for nic_seq, nic in enumerate(kvm_nics):
828         self._ConfigureNIC(instance, nic_seq, nic, taps[nic_seq])
829
830     if security_model == constants.HT_SM_POOL:
831       ss = ssconf.SimpleStore()
832       uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\n")
833       all_uids = set(uidpool.ExpandUidPool(uid_pool))
834       uid = uidpool.RequestUnusedUid(all_uids)
835       try:
836         username = pwd.getpwuid(uid.GetUid()).pw_name
837         kvm_cmd.extend(["-runas", username])
838         self._RunKVMCmd(name, kvm_cmd, tapfds)
839       except:
840         uidpool.ReleaseUid(uid)
841         raise
842       else:
843         uid.Unlock()
844         utils.WriteFile(self._InstanceUidFile(name), data=str(uid))
845     else:
846       self._RunKVMCmd(name, kvm_cmd, tapfds)
847
848     utils.EnsureDirs([(self._InstanceNICDir(instance.name),
849                      constants.RUN_DIRS_MODE)])
850     for nic_seq, tap in enumerate(taps):
851       utils.WriteFile(self._InstanceNICFile(instance.name, nic_seq),
852                       data=tap)
853
854     if vnc_pwd:
855       change_cmd = 'change vnc password %s' % vnc_pwd
856       self._CallMonitorCommand(instance.name, change_cmd)
857
858     for filename in temp_files:
859       utils.RemoveFile(filename)
860
861   def StartInstance(self, instance, block_devices):
862     """Start an instance.
863
864     """
865     self._CheckDown(instance.name)
866     kvm_runtime = self._GenerateKVMRuntime(instance, block_devices)
867     self._SaveKVMRuntime(instance, kvm_runtime)
868     self._ExecuteKVMRuntime(instance, kvm_runtime)
869
870   def _CallMonitorCommand(self, instance_name, command):
871     """Invoke a command on the instance monitor.
872
873     """
874     socat = ("echo %s | %s STDIO UNIX-CONNECT:%s" %
875              (utils.ShellQuote(command),
876               constants.SOCAT_PATH,
877               utils.ShellQuote(self._InstanceMonitor(instance_name))))
878     result = utils.RunCmd(socat)
879     if result.failed:
880       msg = ("Failed to send command '%s' to instance %s."
881              " output: %s, error: %s, fail_reason: %s" %
882              (command, instance_name,
883               result.stdout, result.stderr, result.fail_reason))
884       raise errors.HypervisorError(msg)
885
886     return result
887
888   @classmethod
889   def _GetKVMVersion(cls):
890     """Return the installed KVM version
891
892     @return: (version, v_maj, v_min, v_rev), or None
893
894     """
895     result = utils.RunCmd([constants.KVM_PATH, "--help"])
896     if result.failed:
897       return None
898     match = cls._VERSION_RE.search(result.output.splitlines()[0])
899     if not match:
900       return None
901
902     return (match.group(0), int(match.group(1)), int(match.group(2)),
903             int(match.group(3)))
904
905   def StopInstance(self, instance, force=False, retry=False, name=None):
906     """Stop an instance.
907
908     """
909     if name is not None and not force:
910       raise errors.HypervisorError("Cannot shutdown cleanly by name only")
911     if name is None:
912       name = instance.name
913       acpi = instance.hvparams[constants.HV_ACPI]
914     else:
915       acpi = False
916     _, pid, alive = self._InstancePidAlive(name)
917     if pid > 0 and alive:
918       if force or not acpi:
919         utils.KillProcess(pid)
920       else:
921         self._CallMonitorCommand(name, 'system_powerdown')
922
923   def CleanupInstance(self, instance_name):
924     """Cleanup after a stopped instance
925
926     """
927     pidfile, pid, alive = self._InstancePidAlive(instance_name)
928     if pid > 0 and alive:
929       raise errors.HypervisorError("Cannot cleanup a live instance")
930     self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
931
932   def RebootInstance(self, instance):
933     """Reboot an instance.
934
935     """
936     # For some reason if we do a 'send-key ctrl-alt-delete' to the control
937     # socket the instance will stop, but now power up again. So we'll resort
938     # to shutdown and restart.
939     _, _, alive = self._InstancePidAlive(instance.name)
940     if not alive:
941       raise errors.HypervisorError("Failed to reboot instance %s:"
942                                    " not running" % instance.name)
943     # StopInstance will delete the saved KVM runtime so:
944     # ...first load it...
945     kvm_runtime = self._LoadKVMRuntime(instance)
946     # ...now we can safely call StopInstance...
947     if not self.StopInstance(instance):
948       self.StopInstance(instance, force=True)
949     # ...and finally we can save it again, and execute it...
950     self._SaveKVMRuntime(instance, kvm_runtime)
951     self._ExecuteKVMRuntime(instance, kvm_runtime)
952
953   def MigrationInfo(self, instance):
954     """Get instance information to perform a migration.
955
956     @type instance: L{objects.Instance}
957     @param instance: instance to be migrated
958     @rtype: string
959     @return: content of the KVM runtime file
960
961     """
962     return self._ReadKVMRuntime(instance.name)
963
964   def AcceptInstance(self, instance, info, target):
965     """Prepare to accept an instance.
966
967     @type instance: L{objects.Instance}
968     @param instance: instance to be accepted
969     @type info: string
970     @param info: content of the KVM runtime file on the source node
971     @type target: string
972     @param target: target host (usually ip), on this node
973
974     """
975     kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
976     incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT])
977     self._ExecuteKVMRuntime(instance, kvm_runtime, incoming=incoming_address)
978
979   def FinalizeMigration(self, instance, info, success):
980     """Finalize an instance migration.
981
982     Stop the incoming mode KVM.
983
984     @type instance: L{objects.Instance}
985     @param instance: instance whose migration is being finalized
986
987     """
988     if success:
989       kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
990       kvm_nics = kvm_runtime[1]
991
992       for nic_seq, nic in enumerate(kvm_nics):
993         try:
994           tap = utils.ReadFile(self._InstanceNICFile(instance.name, nic_seq))
995         except EnvironmentError, err:
996           logging.warning("Failed to find host interface for %s NIC #%d: %s",
997                           instance.name, nic_seq, str(err))
998           continue
999         try:
1000           self._ConfigureNIC(instance, nic_seq, nic, tap)
1001         except errors.HypervisorError, err:
1002           logging.warning(str(err))
1003
1004       self._WriteKVMRuntime(instance.name, info)
1005     else:
1006       self.StopInstance(instance, force=True)
1007
1008   def MigrateInstance(self, instance, target, live):
1009     """Migrate an instance to a target node.
1010
1011     The migration will not be attempted if the instance is not
1012     currently running.
1013
1014     @type instance: L{objects.Instance}
1015     @param instance: the instance to be migrated
1016     @type target: string
1017     @param target: ip address of the target node
1018     @type live: boolean
1019     @param live: perform a live migration
1020
1021     """
1022     instance_name = instance.name
1023     port = instance.hvparams[constants.HV_MIGRATION_PORT]
1024     pidfile, pid, alive = self._InstancePidAlive(instance_name)
1025     if not alive:
1026       raise errors.HypervisorError("Instance not running, cannot migrate")
1027
1028     if not live:
1029       self._CallMonitorCommand(instance_name, 'stop')
1030
1031     migrate_command = ('migrate_set_speed %dm' %
1032         instance.hvparams[constants.HV_MIGRATION_BANDWIDTH])
1033     self._CallMonitorCommand(instance_name, migrate_command)
1034
1035     migrate_command = ('migrate_set_downtime %dms' %
1036         instance.hvparams[constants.HV_MIGRATION_DOWNTIME])
1037     self._CallMonitorCommand(instance_name, migrate_command)
1038
1039     migrate_command = 'migrate -d tcp:%s:%s' % (target, port)
1040     self._CallMonitorCommand(instance_name, migrate_command)
1041
1042     info_command = 'info migrate'
1043     done = False
1044     broken_answers = 0
1045     while not done:
1046       result = self._CallMonitorCommand(instance_name, info_command)
1047       match = self._MIGRATION_STATUS_RE.search(result.stdout)
1048       if not match:
1049         broken_answers += 1
1050         if not result.stdout:
1051           logging.info("KVM: empty 'info migrate' result")
1052         else:
1053           logging.warning("KVM: unknown 'info migrate' result: %s",
1054                           result.stdout)
1055         time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1056       else:
1057         status = match.group(1)
1058         if status == 'completed':
1059           done = True
1060         elif status == 'active':
1061           # reset the broken answers count
1062           broken_answers = 0
1063           time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1064         elif status == 'failed' or status == 'cancelled':
1065           if not live:
1066             self._CallMonitorCommand(instance_name, 'cont')
1067           raise errors.HypervisorError("Migration %s at the kvm level" %
1068                                        status)
1069         else:
1070           logging.warning("KVM: unknown migration status '%s'", status)
1071           broken_answers += 1
1072           time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
1073       if broken_answers >= self._MIGRATION_INFO_MAX_BAD_ANSWERS:
1074         raise errors.HypervisorError("Too many 'info migrate' broken answers")
1075
1076     utils.KillProcess(pid)
1077     self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
1078
1079   def GetNodeInfo(self):
1080     """Return information about the node.
1081
1082     This is just a wrapper over the base GetLinuxNodeInfo method.
1083
1084     @return: a dict with the following keys (values in MiB):
1085           - memory_total: the total memory size on the node
1086           - memory_free: the available memory on the node for instances
1087           - memory_dom0: the memory used by the node itself, if available
1088
1089     """
1090     return self.GetLinuxNodeInfo()
1091
1092   @classmethod
1093   def GetInstanceConsole(cls, instance, hvparams, beparams):
1094     """Return a command for connecting to the console of an instance.
1095
1096     """
1097     if hvparams[constants.HV_SERIAL_CONSOLE]:
1098       cmd = [constants.SOCAT_PATH,
1099              "STDIO,%s" % cls._SocatUnixConsoleParams(),
1100              "UNIX-CONNECT:%s" % cls._InstanceSerial(instance.name)]
1101       return objects.InstanceConsole(instance=instance.name,
1102                                      kind=constants.CONS_SSH,
1103                                      host=instance.primary_node,
1104                                      user=constants.GANETI_RUNAS,
1105                                      command=cmd)
1106
1107     vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS]
1108     if vnc_bind_address and instance.network_port > constants.VNC_BASE_PORT:
1109       display = instance.network_port - constants.VNC_BASE_PORT
1110       return objects.InstanceConsole(instance=instance.name,
1111                                      kind=constants.CONS_VNC,
1112                                      host=vnc_bind_address,
1113                                      port=instance.network_port,
1114                                      display=display)
1115
1116     return objects.InstanceConsole(instance=instance.name,
1117                                    kind=constants.CONS_MESSAGE,
1118                                    message=("No serial shell for instance %s" %
1119                                             instance.name))
1120
1121   def Verify(self):
1122     """Verify the hypervisor.
1123
1124     Check that the binary exists.
1125
1126     """
1127     if not os.path.exists(constants.KVM_PATH):
1128       return "The kvm binary ('%s') does not exist." % constants.KVM_PATH
1129     if not os.path.exists(constants.SOCAT_PATH):
1130       return "The socat binary ('%s') does not exist." % constants.SOCAT_PATH
1131
1132   @classmethod
1133   def CheckParameterSyntax(cls, hvparams):
1134     """Check the given parameters for validity.
1135
1136     @type hvparams:  dict
1137     @param hvparams: dictionary with parameter names/value
1138     @raise errors.HypervisorError: when a parameter is not valid
1139
1140     """
1141     super(KVMHypervisor, cls).CheckParameterSyntax(hvparams)
1142
1143     kernel_path = hvparams[constants.HV_KERNEL_PATH]
1144     if kernel_path:
1145       if not hvparams[constants.HV_ROOT_PATH]:
1146         raise errors.HypervisorError("Need a root partition for the instance,"
1147                                      " if a kernel is defined")
1148
1149     if (hvparams[constants.HV_VNC_X509_VERIFY] and
1150         not hvparams[constants.HV_VNC_X509]):
1151       raise errors.HypervisorError("%s must be defined, if %s is" %
1152                                    (constants.HV_VNC_X509,
1153                                     constants.HV_VNC_X509_VERIFY))
1154
1155     boot_order = hvparams[constants.HV_BOOT_ORDER]
1156     if (boot_order == constants.HT_BO_CDROM and
1157         not hvparams[constants.HV_CDROM_IMAGE_PATH]):
1158       raise errors.HypervisorError("Cannot boot from cdrom without an"
1159                                    " ISO path")
1160
1161     security_model = hvparams[constants.HV_SECURITY_MODEL]
1162     if security_model == constants.HT_SM_USER:
1163       if not hvparams[constants.HV_SECURITY_DOMAIN]:
1164         raise errors.HypervisorError("A security domain (user to run kvm as)"
1165                                      " must be specified")
1166     elif (security_model == constants.HT_SM_NONE or
1167           security_model == constants.HT_SM_POOL):
1168       if hvparams[constants.HV_SECURITY_DOMAIN]:
1169         raise errors.HypervisorError("Cannot have a security domain when the"
1170                                      " security model is 'none' or 'pool'")
1171
1172   @classmethod
1173   def ValidateParameters(cls, hvparams):
1174     """Check the given parameters for validity.
1175
1176     @type hvparams:  dict
1177     @param hvparams: dictionary with parameter names/value
1178     @raise errors.HypervisorError: when a parameter is not valid
1179
1180     """
1181     super(KVMHypervisor, cls).ValidateParameters(hvparams)
1182
1183     security_model = hvparams[constants.HV_SECURITY_MODEL]
1184     if security_model == constants.HT_SM_USER:
1185       username = hvparams[constants.HV_SECURITY_DOMAIN]
1186       try:
1187         pwd.getpwnam(username)
1188       except KeyError:
1189         raise errors.HypervisorError("Unknown security domain user %s"
1190                                      % username)
1191
1192   @classmethod
1193   def PowercycleNode(cls):
1194     """KVM powercycle, just a wrapper over Linux powercycle.
1195
1196     """
1197     cls.LinuxPowercycle()