4 # Copyright (C) 2008 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
32 from cStringIO import StringIO
34 from ganeti import utils
35 from ganeti import constants
36 from ganeti import errors
37 from ganeti import serializer
38 from ganeti import objects
39 from ganeti.hypervisor import hv_base
42 class KVMHypervisor(hv_base.BaseHypervisor):
43 """KVM hypervisor interface"""
45 _ROOT_DIR = constants.RUN_GANETI_DIR + "/kvm-hypervisor"
46 _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids
47 _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets
48 _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data
49 _DIRS = [_ROOT_DIR, _PIDS_DIR, _CTRL_DIR, _CONF_DIR]
52 constants.HV_KERNEL_PATH,
53 constants.HV_INITRD_PATH,
54 constants.HV_ROOT_PATH,
56 constants.HV_SERIAL_CONSOLE,
57 constants.HV_VNC_BIND_ADDRESS,
60 _MIGRATION_STATUS_RE = re.compile('Migration\s+status:\s+(\w+)',
64 hv_base.BaseHypervisor.__init__(self)
65 # Let's make sure the directories we need exist, even if the RUN_DIR lives
66 # in a tmpfs filesystem or has been otherwise wiped out.
67 for mydir in self._DIRS:
68 if not os.path.exists(mydir):
71 def _InstancePidAlive(self, instance_name):
72 """Returns the instance pid and pidfile
75 pidfile = "%s/%s" % (self._PIDS_DIR, instance_name)
76 pid = utils.ReadPidFile(pidfile)
77 alive = utils.IsProcessAlive(pid)
79 return (pidfile, pid, alive)
82 def _InstanceMonitor(cls, instance_name):
83 """Returns the instance monitor socket name
86 return '%s/%s.monitor' % (cls._CTRL_DIR, instance_name)
89 def _InstanceSerial(cls, instance_name):
90 """Returns the instance serial socket name
93 return '%s/%s.serial' % (cls._CTRL_DIR, instance_name)
96 def _InstanceKVMRuntime(cls, instance_name):
97 """Returns the instance KVM runtime filename
100 return '%s/%s.runtime' % (cls._CONF_DIR, instance_name)
102 def _WriteNetScript(self, instance, seq, nic):
103 """Write a script to connect a net interface to the proper bridge.
105 This can be used by any qemu-type hypervisor.
107 @param instance: instance we're acting on
108 @type instance: instance object
109 @param seq: nic sequence number
111 @param nic: nic we're acting on
112 @type nic: nic object
113 @return: netscript file name
118 script.write("#!/bin/sh\n")
119 script.write("# this is autogenerated by Ganeti, please do not edit\n#\n")
120 script.write("export INSTANCE=%s\n" % instance.name)
121 script.write("export MAC=%s\n" % nic.mac)
122 script.write("export IP=%s\n" % nic.ip)
123 script.write("export BRIDGE=%s\n" % nic.bridge)
124 script.write("export INTERFACE=$1\n")
125 # TODO: make this configurable at ./configure time
126 script.write("if [ -x /etc/ganeti/kvm-vif-bridge ]; then\n")
127 script.write(" # Execute the user-specific vif file\n")
128 script.write(" /etc/ganeti/kvm-vif-bridge\n")
129 script.write("else\n")
130 script.write(" # Connect the interface to the bridge\n")
131 script.write(" /sbin/ifconfig $INTERFACE 0.0.0.0 up\n")
132 script.write(" /usr/sbin/brctl addif $BRIDGE $INTERFACE\n")
133 script.write("fi\n\n")
134 # As much as we'd like to put this in our _ROOT_DIR, that will happen to be
135 # mounted noexec sometimes, so we'll have to find another place.
136 (tmpfd, tmpfile_name) = tempfile.mkstemp()
137 tmpfile = os.fdopen(tmpfd, 'w')
138 tmpfile.write(script.getvalue())
140 os.chmod(tmpfile_name, 0755)
143 def ListInstances(self):
144 """Get the list of running instances.
146 We can do this by listing our live instances directory and
147 checking whether the associated kvm process is still alive.
151 for name in os.listdir(self._PIDS_DIR):
152 filename = "%s/%s" % (self._PIDS_DIR, name)
153 if utils.IsProcessAlive(utils.ReadPidFile(filename)):
157 def GetInstanceInfo(self, instance_name):
158 """Get instance properties.
160 @param instance_name: the instance name
162 @return: tuple (name, id, memory, vcpus, stat, times)
165 pidfile, pid, alive = self._InstancePidAlive(instance_name)
169 cmdline_file = "/proc/%s/cmdline" % pid
171 fh = open(cmdline_file, 'r')
176 except EnvironmentError, err:
177 raise errors.HypervisorError("Failed to list instance %s: %s" %
178 (instance_name, err))
185 arg_list = cmdline.split('\x00')
187 arg = arg_list.pop(0)
189 memory = arg_list.pop(0)
191 vcpus = arg_list.pop(0)
193 return (instance_name, pid, memory, vcpus, stat, times)
195 def GetAllInstancesInfo(self):
196 """Get properties of all instances.
198 @return: list of tuples (name, id, memory, vcpus, stat, times)
202 for name in os.listdir(self._PIDS_DIR):
203 filename = "%s/%s" % (self._PIDS_DIR, name)
204 if utils.IsProcessAlive(utils.ReadPidFile(filename)):
205 data.append(self.GetInstanceInfo(name))
209 def _GenerateKVMRuntime(self, instance, block_devices, extra_args):
210 """Generate KVM information to start an instance.
213 pidfile, pid, alive = self._InstancePidAlive(instance.name)
214 kvm = constants.KVM_PATH
216 kvm_cmd.extend(['-m', instance.beparams[constants.BE_MEMORY]])
217 kvm_cmd.extend(['-smp', instance.beparams[constants.BE_VCPUS]])
218 kvm_cmd.extend(['-pidfile', pidfile])
219 # used just by the vnc server, if enabled
220 kvm_cmd.extend(['-name', instance.name])
221 kvm_cmd.extend(['-daemonize'])
222 if not instance.hvparams[constants.HV_ACPI]:
223 kvm_cmd.extend(['-no-acpi'])
226 for cfdev, dev_path in block_devices:
227 if cfdev.mode != constants.DISK_RDWR:
228 raise errors.HypervisorError("Instance has read-only disks which"
229 " are not supported by KVM")
230 # TODO: handle FD_LOOP and FD_BLKTAP (?)
232 boot_val = ',boot=on'
237 # TODO: handle different if= types
238 if_val = ',if=virtio'
240 drive_val = 'file=%s,format=raw%s%s' % (dev_path, if_val, boot_val)
241 kvm_cmd.extend(['-drive', drive_val])
243 kernel_path = instance.hvparams[constants.HV_KERNEL_PATH]
245 kvm_cmd.extend(['-kernel', kernel_path])
246 initrd_path = instance.hvparams[constants.HV_INITRD_PATH]
248 kvm_cmd.extend(['-initrd', initrd_path])
249 root_append = 'root=%s ro' % instance.hvparams[constants.HV_ROOT_PATH]
250 if instance.hvparams[constants.HV_SERIAL_CONSOLE]:
251 kvm_cmd.extend(['-append', 'console=ttyS0,38400 %s' % root_append])
253 kvm_cmd.extend(['-append', root_append])
256 #"hvm_cdrom_image_path",
258 kvm_cmd.extend(['-nographic'])
259 # FIXME: handle vnc, if needed
260 # How do we decide whether to have it or not?? :(
263 monitor_dev = 'unix:%s,server,nowait' % \
264 self._InstanceMonitor(instance.name)
265 kvm_cmd.extend(['-monitor', monitor_dev])
266 if instance.hvparams[constants.HV_SERIAL_CONSOLE]:
267 serial_dev = 'unix:%s,server,nowait' % self._InstanceSerial(instance.name)
268 kvm_cmd.extend(['-serial', serial_dev])
270 kvm_cmd.extend(['-serial', 'none'])
272 # Save the current instance nics, but defer their expansion as parameters,
273 # as we'll need to generate executable temp files for them.
274 kvm_nics = instance.nics
276 return (kvm_cmd, kvm_nics)
278 def _WriteKVMRuntime(self, instance_name, data):
279 """Write an instance's KVM runtime
283 utils.WriteFile(self._InstanceKVMRuntime(instance_name),
285 except EnvironmentError, err:
286 raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err)
288 def _ReadKVMRuntime(self, instance_name):
289 """Read an instance's KVM runtime
293 file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name))
294 except EnvironmentError, err:
295 raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err)
298 def _SaveKVMRuntime(self, instance, kvm_runtime):
299 """Save an instance's KVM runtime
302 kvm_cmd, kvm_nics = kvm_runtime
303 serialized_nics = [nic.ToDict() for nic in kvm_nics]
304 serialized_form = serializer.Dump((kvm_cmd, serialized_nics))
305 self._WriteKVMRuntime(instance.name, serialized_form)
307 def _LoadKVMRuntime(self, instance, serialized_runtime=None):
308 """Load an instance's KVM runtime
311 if not serialized_runtime:
312 serialized_runtime = self._ReadKVMRuntime(instance.name)
313 loaded_runtime = serializer.Load(serialized_runtime)
314 kvm_cmd, serialized_nics = loaded_runtime
315 kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics]
316 return (kvm_cmd, kvm_nics)
318 def _ExecuteKVMRuntime(self, instance, kvm_runtime, incoming=None):
319 """Execute a KVM cmd, after completing it with some last minute data
321 @type incoming: tuple of strings
322 @param incoming: (target_host_ip, port)
325 pidfile, pid, alive = self._InstancePidAlive(instance.name)
327 raise errors.HypervisorError("Failed to start instance %s: %s" %
328 (instance.name, "already running"))
332 kvm_cmd, kvm_nics = kvm_runtime
335 kvm_cmd.extend(['-net', 'none'])
337 for nic_seq, nic in enumerate(kvm_nics):
338 nic_val = "nic,macaddr=%s,model=virtio" % nic.mac
339 script = self._WriteNetScript(instance, nic_seq, nic)
340 kvm_cmd.extend(['-net', nic_val])
341 kvm_cmd.extend(['-net', 'tap,script=%s' % script])
342 temp_files.append(script)
345 target, port = incoming
346 kvm_cmd.extend(['-incoming', 'tcp:%s:%s' % (target, port)])
348 result = utils.RunCmd(kvm_cmd)
350 raise errors.HypervisorError("Failed to start instance %s: %s (%s)" %
351 (instance.name, result.fail_reason,
354 if not utils.IsProcessAlive(utils.ReadPidFile(pidfile)):
355 raise errors.HypervisorError("Failed to start instance %s: %s" %
358 for filename in temp_files:
359 utils.RemoveFile(filename)
361 def StartInstance(self, instance, block_devices, extra_args):
362 """Start an instance.
365 pidfile, pid, alive = self._InstancePidAlive(instance.name)
367 raise errors.HypervisorError("Failed to start instance %s: %s" %
368 (instance.name, "already running"))
370 kvm_runtime = self._GenerateKVMRuntime(instance, block_devices, extra_args)
371 self._SaveKVMRuntime(instance, kvm_runtime)
372 self._ExecuteKVMRuntime(instance, kvm_runtime)
374 def _CallMonitorCommand(self, instance_name, command):
375 """Invoke a command on the instance monitor.
378 socat = ("echo %s | %s STDIO UNIX-CONNECT:%s" %
379 (utils.ShellQuote(command),
380 constants.SOCAT_PATH,
381 utils.ShellQuote(self._InstanceMonitor(instance_name))))
382 result = utils.RunCmd(socat)
384 msg = ("Failed to send command '%s' to instance %s."
385 " output: %s, error: %s, fail_reason: %s" %
386 (instance.name, result.stdout, result.stderr, result.fail_reason))
387 raise errors.HypervisorError(msg)
391 def _RetryInstancePowerdown(self, instance, pid, timeout=30):
392 """Wait for an instance to power down.
395 # Wait up to $timeout seconds
396 end = time.time() + timeout
398 while time.time() < end and utils.IsProcessAlive(pid):
399 self._CallMonitorCommand(instance.name, 'system_powerdown')
401 # Make wait time longer for next try
405 def StopInstance(self, instance, force=False):
409 pidfile, pid, alive = self._InstancePidAlive(instance.name)
410 if pid > 0 and alive:
411 if force or not instance.hvparams[constants.HV_ACPI]:
412 utils.KillProcess(pid)
414 self._RetryInstancePowerdown(instance, pid)
416 if not utils.IsProcessAlive(pid):
417 utils.RemoveFile(pidfile)
418 utils.RemoveFile(self._InstanceMonitor(instance.name))
419 utils.RemoveFile(self._InstanceSerial(instance.name))
420 utils.RemoveFile(self._InstanceKVMRuntime(instance.name))
425 def RebootInstance(self, instance):
426 """Reboot an instance.
429 # For some reason if we do a 'send-key ctrl-alt-delete' to the control
430 # socket the instance will stop, but now power up again. So we'll resort
431 # to shutdown and restart.
432 pidfile, pid, alive = self._InstancePidAlive(instance.name)
434 raise errors.HypervisorError("Failed to reboot instance %s: not running" %
436 # StopInstance will delete the saved KVM runtime so:
437 # ...first load it...
438 kvm_runtime = self._LoadKVMRuntime(instance)
439 # ...now we can safely call StopInstance...
440 if not self.StopInstance(instance):
441 self.StopInstance(instance, force=True)
442 # ...and finally we can save it again, and execute it...
443 self._SaveKVMRuntime(instance, kvm_runtime)
444 self._ExecuteKVMRuntime(instance, kvm_runtime)
446 def MigrationInfo(self, instance):
447 """Get instance information to perform a migration.
449 @type instance: L{objects.Instance}
450 @param instance: instance to be migrated
452 @return: content of the KVM runtime file
455 return self._ReadKVMRuntime(instance.name)
457 def AcceptInstance(self, instance, info, target):
458 """Prepare to accept an instance.
460 @type instance: L{objects.Instance}
461 @param instance: instance to be accepted
463 @param info: content of the KVM runtime file on the source node
465 @param target: target host (usually ip), on this node
468 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
469 incoming_address = (target, constants.KVM_MIGRATION_PORT)
470 self._ExecuteKVMRuntime(instance, kvm_runtime, incoming=incoming_address)
472 def FinalizeMigration(self, instance, info, success):
473 """Finalize an instance migration.
475 Stop the incoming mode KVM.
477 @type instance: L{objects.Instance}
478 @param instance: instance whose migration is being aborted
482 self._WriteKVMRuntime(instance.name, info)
484 self.StopInstance(instance, force=True)
486 def MigrateInstance(self, instance_name, target, live):
487 """Migrate an instance to a target node.
489 The migration will not be attempted if the instance is not
492 @type instance_name: string
493 @param instance_name: name of the instance to be migrated
495 @param target: ip address of the target node
497 @param live: perform a live migration
500 pidfile, pid, alive = self._InstancePidAlive(instance_name)
502 raise errors.HypervisorError("Instance not running, cannot migrate")
505 self._CallMonitorCommand(instance_name, 'stop')
507 migrate_command = ('migrate -d tcp:%s:%s' %
508 (target, constants.KVM_MIGRATION_PORT))
509 self._CallMonitorCommand(instance_name, migrate_command)
511 info_command = 'info migrate'
514 result = self._CallMonitorCommand(instance_name, info_command)
515 match = self._MIGRATION_STATUS_RE.search(result.stdout)
517 raise errors.HypervisorError("Unknown 'info migrate' result: %s" %
520 status = match.group(1)
521 if status == 'completed':
523 elif status == 'active':
525 elif status == 'failed' or status == 'cancelled':
527 self._CallMonitorCommand(instance_name, 'cont')
528 raise errors.HypervisorError("Migration %s at the kvm level" %
531 logging.info("KVM: unknown migration status '%s'" % status)
534 utils.KillProcess(pid)
535 utils.RemoveFile(pidfile)
536 utils.RemoveFile(self._InstanceMonitor(instance_name))
537 utils.RemoveFile(self._InstanceSerial(instance_name))
538 utils.RemoveFile(self._InstanceKVMRuntime(instance_name))
540 def GetNodeInfo(self):
541 """Return information about the node.
543 @return: a dict with the following keys (values in MiB):
544 - memory_total: the total memory size on the node
545 - memory_free: the available memory on the node for instances
546 - memory_dom0: the memory used by the node itself, if available
549 # global ram usage from the xm info command
552 # note: in xen 3, memory has changed to total_memory
554 fh = file("/proc/meminfo")
556 data = fh.readlines()
559 except EnvironmentError, err:
560 raise errors.HypervisorError("Failed to list node info: %s" % err)
565 splitfields = line.split(":", 1)
567 if len(splitfields) > 1:
568 key = splitfields[0].strip()
569 val = splitfields[1].strip()
570 if key == 'MemTotal':
571 result['memory_total'] = int(val.split()[0])/1024
572 elif key in ('MemFree', 'Buffers', 'Cached'):
573 sum_free += int(val.split()[0])/1024
574 elif key == 'Active':
575 result['memory_dom0'] = int(val.split()[0])/1024
576 result['memory_free'] = sum_free
580 fh = open("/proc/cpuinfo")
582 cpu_total = len(re.findall("(?m)^processor\s*:\s*[0-9]+\s*$",
586 except EnvironmentError, err:
587 raise errors.HypervisorError("Failed to list node info: %s" % err)
588 result['cpu_total'] = cpu_total
593 def GetShellCommandForConsole(cls, instance, hvparams, beparams):
594 """Return a command for connecting to the console of an instance.
597 if hvparams[constants.HV_SERIAL_CONSOLE]:
598 # FIXME: The socat shell is not perfect. In particular the way we start
599 # it ctrl+c will close it, rather than being passed to the other end.
600 # On the other hand if we pass the option 'raw' (or ignbrk=1) there
601 # will be no way of exiting socat (except killing it from another shell)
602 # and ctrl+c doesn't work anyway, printing ^C rather than being
603 # interpreted by kvm. For now we'll leave it this way, which at least
604 # allows a minimal interaction and changes on the machine.
605 shell_command = ("%s STDIO,echo=0,icanon=0 UNIX-CONNECT:%s" %
606 (constants.SOCAT_PATH,
607 utils.ShellQuote(cls._InstanceSerial(instance.name))))
609 shell_command = "echo 'No serial shell for instance %s'" % instance.name
613 """Verify the hypervisor.
615 Check that the binary exists.
618 if not os.path.exists(constants.KVM_PATH):
619 return "The kvm binary ('%s') does not exist." % constants.KVM_PATH
620 if not os.path.exists(constants.SOCAT_PATH):
621 return "The socat binary ('%s') does not exist." % constants.SOCAT_PATH
625 def CheckParameterSyntax(cls, hvparams):
626 """Check the given parameters for validity.
628 For the KVM hypervisor, this only check the existence of the
632 @param hvparams: dictionary with parameter names/value
633 @raise errors.HypervisorError: when a parameter is not valid
636 super(KVMHypervisor, cls).CheckParameterSyntax(hvparams)
638 kernel_path = hvparams[constants.HV_KERNEL_PATH]
640 if not os.path.isabs(hvparams[constants.HV_KERNEL_PATH]):
641 raise errors.HypervisorError("The kernel path must be an absolute path"
644 if not hvparams[constants.HV_ROOT_PATH]:
645 raise errors.HypervisorError("Need a root partition for the instance"
646 ", if a kernel is defined")
648 if hvparams[constants.HV_INITRD_PATH]:
649 if not os.path.isabs(hvparams[constants.HV_INITRD_PATH]):
650 raise errors.HypervisorError("The initrd path must an absolute path"
653 vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS]
655 if not utils.IsValidIP(vnc_bind_address):
656 raise errors.OpPrereqError("given VNC bind address '%s' doesn't look"
657 " like a valid IP address" %
660 def ValidateParameters(self, hvparams):
661 """Check the given parameters for validity.
663 For the KVM hypervisor, this checks the existence of the
667 super(KVMHypervisor, self).ValidateParameters(hvparams)
669 kernel_path = hvparams[constants.HV_KERNEL_PATH]
670 if kernel_path and not os.path.isfile(kernel_path):
671 raise errors.HypervisorError("Instance kernel '%s' not found or"
672 " not a file" % kernel_path)
673 initrd_path = hvparams[constants.HV_INITRD_PATH]
674 if initrd_path and not os.path.isfile(initrd_path):
675 raise errors.HypervisorError("Instance initrd '%s' not found or"
676 " not a file" % initrd_path)