4 # Copyright (C) 2008 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
32 from cStringIO import StringIO
34 from ganeti import utils
35 from ganeti import constants
36 from ganeti import errors
37 from ganeti import serializer
38 from ganeti import objects
39 from ganeti.hypervisor import hv_base
42 class KVMHypervisor(hv_base.BaseHypervisor):
43 """KVM hypervisor interface"""
45 _ROOT_DIR = constants.RUN_GANETI_DIR + "/kvm-hypervisor"
46 _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids
47 _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets
48 _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data
49 _DIRS = [_ROOT_DIR, _PIDS_DIR, _CTRL_DIR, _CONF_DIR]
52 constants.HV_KERNEL_PATH,
53 constants.HV_INITRD_PATH,
57 _MIGRATION_STATUS_RE = re.compile('Migration\s+status:\s+(\w+)',
61 hv_base.BaseHypervisor.__init__(self)
62 # Let's make sure the directories we need exist, even if the RUN_DIR lives
63 # in a tmpfs filesystem or has been otherwise wiped out.
64 for mydir in self._DIRS:
65 if not os.path.exists(mydir):
68 def _InstancePidAlive(self, instance_name):
69 """Returns the instance pid and pidfile
72 pidfile = "%s/%s" % (self._PIDS_DIR, instance_name)
73 pid = utils.ReadPidFile(pidfile)
74 alive = utils.IsProcessAlive(pid)
76 return (pidfile, pid, alive)
78 def _InstanceMonitor(self, instance_name):
79 """Returns the instance monitor socket name
82 return '%s/%s.monitor' % (self._CTRL_DIR, instance_name)
84 def _InstanceSerial(self, instance_name):
85 """Returns the instance serial socket name
88 return '%s/%s.serial' % (self._CTRL_DIR, instance_name)
90 def _InstanceKVMRuntime(self, instance_name):
91 """Returns the instance KVM runtime filename
94 return '%s/%s.runtime' % (self._CONF_DIR, instance_name)
96 def _WriteNetScript(self, instance, seq, nic):
97 """Write a script to connect a net interface to the proper bridge.
99 This can be used by any qemu-type hypervisor.
101 @param instance: instance we're acting on
102 @type instance: instance object
103 @param seq: nic sequence number
105 @param nic: nic we're acting on
106 @type nic: nic object
107 @return: netscript file name
112 script.write("#!/bin/sh\n")
113 script.write("# this is autogenerated by Ganeti, please do not edit\n#\n")
114 script.write("export INSTANCE=%s\n" % instance.name)
115 script.write("export MAC=%s\n" % nic.mac)
116 script.write("export IP=%s\n" % nic.ip)
117 script.write("export BRIDGE=%s\n" % nic.bridge)
118 script.write("export INTERFACE=$1\n")
119 # TODO: make this configurable at ./configure time
120 script.write("if [ -x /etc/ganeti/kvm-vif-bridge ]; then\n")
121 script.write(" # Execute the user-specific vif file\n")
122 script.write(" /etc/ganeti/kvm-vif-bridge\n")
123 script.write("else\n")
124 script.write(" # Connect the interface to the bridge\n")
125 script.write(" /sbin/ifconfig $INTERFACE 0.0.0.0 up\n")
126 script.write(" /usr/sbin/brctl addif $BRIDGE $INTERFACE\n")
127 script.write("fi\n\n")
128 # As much as we'd like to put this in our _ROOT_DIR, that will happen to be
129 # mounted noexec sometimes, so we'll have to find another place.
130 (tmpfd, tmpfile_name) = tempfile.mkstemp()
131 tmpfile = os.fdopen(tmpfd, 'w')
132 tmpfile.write(script.getvalue())
134 os.chmod(tmpfile_name, 0755)
137 def ListInstances(self):
138 """Get the list of running instances.
140 We can do this by listing our live instances directory and
141 checking whether the associated kvm process is still alive.
145 for name in os.listdir(self._PIDS_DIR):
146 filename = "%s/%s" % (self._PIDS_DIR, name)
147 if utils.IsProcessAlive(utils.ReadPidFile(filename)):
151 def GetInstanceInfo(self, instance_name):
152 """Get instance properties.
154 @param instance_name: the instance name
156 @return: tuple (name, id, memory, vcpus, stat, times)
159 pidfile, pid, alive = self._InstancePidAlive(instance_name)
163 cmdline_file = "/proc/%s/cmdline" % pid
165 fh = open(cmdline_file, 'r')
170 except EnvironmentError, err:
171 raise errors.HypervisorError("Failed to list instance %s: %s" %
172 (instance_name, err))
179 arg_list = cmdline.split('\x00')
181 arg = arg_list.pop(0)
183 memory = arg_list.pop(0)
185 vcpus = arg_list.pop(0)
187 return (instance_name, pid, memory, vcpus, stat, times)
189 def GetAllInstancesInfo(self):
190 """Get properties of all instances.
192 @return: list of tuples (name, id, memory, vcpus, stat, times)
196 for name in os.listdir(self._PIDS_DIR):
197 filename = "%s/%s" % (self._PIDS_DIR, name)
198 if utils.IsProcessAlive(utils.ReadPidFile(filename)):
199 data.append(self.GetInstanceInfo(name))
203 def _GenerateKVMRuntime(self, instance, block_devices, extra_args):
204 """Generate KVM information to start an instance.
207 pidfile, pid, alive = self._InstancePidAlive(instance.name)
208 kvm = constants.KVM_PATH
210 kvm_cmd.extend(['-m', instance.beparams[constants.BE_MEMORY]])
211 kvm_cmd.extend(['-smp', instance.beparams[constants.BE_VCPUS]])
212 kvm_cmd.extend(['-pidfile', pidfile])
213 # used just by the vnc server, if enabled
214 kvm_cmd.extend(['-name', instance.name])
215 kvm_cmd.extend(['-daemonize'])
216 if not instance.hvparams[constants.HV_ACPI]:
217 kvm_cmd.extend(['-no-acpi'])
220 for cfdev, dev_path in block_devices:
221 # TODO: handle FD_LOOP and FD_BLKTAP (?)
223 boot_val = ',boot=on'
228 # TODO: handle different if= types
229 if_val = ',if=virtio'
231 drive_val = 'file=%s,format=raw%s%s' % (dev_path, if_val, boot_val)
232 kvm_cmd.extend(['-drive', drive_val])
234 kvm_cmd.extend(['-kernel', instance.hvparams[constants.HV_KERNEL_PATH]])
236 initrd_path = instance.hvparams[constants.HV_INITRD_PATH]
238 kvm_cmd.extend(['-initrd', initrd_path])
240 kvm_cmd.extend(['-append', 'console=ttyS0,38400 root=/dev/vda'])
243 #"hvm_cdrom_image_path",
245 kvm_cmd.extend(['-nographic'])
246 # FIXME: handle vnc, if needed
247 # How do we decide whether to have it or not?? :(
250 monitor_dev = 'unix:%s,server,nowait' % \
251 self._InstanceMonitor(instance.name)
252 kvm_cmd.extend(['-monitor', monitor_dev])
253 serial_dev = 'unix:%s,server,nowait' % self._InstanceSerial(instance.name)
254 kvm_cmd.extend(['-serial', serial_dev])
256 # Save the current instance nics, but defer their expansion as parameters,
257 # as we'll need to generate executable temp files for them.
258 kvm_nics = instance.nics
260 return (kvm_cmd, kvm_nics)
262 def _WriteKVMRuntime(self, instance_name, data):
263 """Write an instance's KVM runtime
267 utils.WriteFile(self._InstanceKVMRuntime(instance_name),
269 except EnvironmentError, err:
270 raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err)
272 def _ReadKVMRuntime(self, instance_name):
273 """Read an instance's KVM runtime
277 file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name))
278 except EnvironmentError, err:
279 raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err)
282 def _SaveKVMRuntime(self, instance, kvm_runtime):
283 """Save an instance's KVM runtime
286 kvm_cmd, kvm_nics = kvm_runtime
287 serialized_nics = [nic.ToDict() for nic in kvm_nics]
288 serialized_form = serializer.Dump((kvm_cmd, serialized_nics))
289 self._WriteKVMRuntime(instance.name, serialized_form)
291 def _LoadKVMRuntime(self, instance, serialized_runtime=None):
292 """Load an instance's KVM runtime
295 if not serialized_runtime:
296 serialized_runtime = self._ReadKVMRuntime(instance.name)
297 loaded_runtime = serializer.Load(serialized_runtime)
298 kvm_cmd, serialized_nics = loaded_runtime
299 kvm_nics = [objects.NIC.FromDict(snic) for snic in serialized_nics]
300 return (kvm_cmd, kvm_nics)
302 def _ExecuteKVMRuntime(self, instance, kvm_runtime, incoming=None):
303 """Execute a KVM cmd, after completing it with some last minute data
305 @type incoming: tuple of strings
306 @param incoming: (target_host_ip, port)
309 pidfile, pid, alive = self._InstancePidAlive(instance.name)
311 raise errors.HypervisorError("Failed to start instance %s: %s" %
312 (instance.name, "already running"))
316 kvm_cmd, kvm_nics = kvm_runtime
319 kvm_cmd.extend(['-net', 'none'])
321 for nic_seq, nic in enumerate(kvm_nics):
322 nic_val = "nic,macaddr=%s,model=virtio" % nic.mac
323 script = self._WriteNetScript(instance, nic_seq, nic)
324 kvm_cmd.extend(['-net', nic_val])
325 kvm_cmd.extend(['-net', 'tap,script=%s' % script])
326 temp_files.append(script)
329 target, port = incoming
330 kvm_cmd.extend(['-incoming', 'tcp:%s:%s' % (target, port)])
332 result = utils.RunCmd(kvm_cmd)
334 raise errors.HypervisorError("Failed to start instance %s: %s (%s)" %
335 (instance.name, result.fail_reason,
338 if not utils.IsProcessAlive(utils.ReadPidFile(pidfile)):
339 raise errors.HypervisorError("Failed to start instance %s: %s" %
342 for filename in temp_files:
343 utils.RemoveFile(filename)
345 def StartInstance(self, instance, block_devices, extra_args):
346 """Start an instance.
349 pidfile, pid, alive = self._InstancePidAlive(instance.name)
351 raise errors.HypervisorError("Failed to start instance %s: %s" %
352 (instance.name, "already running"))
354 kvm_runtime = self._GenerateKVMRuntime(instance, block_devices, extra_args)
355 self._SaveKVMRuntime(instance, kvm_runtime)
356 self._ExecuteKVMRuntime(instance, kvm_runtime)
358 def _CallMonitorCommand(self, instance_name, command):
359 """Invoke a command on the instance monitor.
362 socat = ("echo %s | %s STDIO UNIX-CONNECT:%s" %
363 (utils.ShellQuote(command),
364 constants.SOCAT_PATH,
365 utils.ShellQuote(self._InstanceMonitor(instance_name))))
366 result = utils.RunCmd(socat)
368 msg = ("Failed to send command '%s' to instance %s."
369 " output: %s, error: %s, fail_reason: %s" %
370 (instance.name, result.stdout, result.stderr, result.fail_reason))
371 raise errors.HypervisorError(msg)
375 def _RetryInstancePowerdown(self, instance, pid, timeout=30):
376 """Wait for an instance to power down.
379 # Wait up to $timeout seconds
380 end = time.time() + timeout
382 while time.time() < end and utils.IsProcessAlive(pid):
383 self._CallMonitorCommand(instance.name, 'system_powerdown')
385 # Make wait time longer for next try
389 def StopInstance(self, instance, force=False):
393 pidfile, pid, alive = self._InstancePidAlive(instance.name)
394 if pid > 0 and alive:
395 if force or not instance.hvparams[constants.HV_ACPI]:
396 utils.KillProcess(pid)
398 self._RetryInstancePowerdown(instance, pid)
400 if not utils.IsProcessAlive(pid):
401 utils.RemoveFile(pidfile)
402 utils.RemoveFile(self._InstanceMonitor(instance.name))
403 utils.RemoveFile(self._InstanceSerial(instance.name))
404 utils.RemoveFile(self._InstanceKVMRuntime(instance.name))
409 def RebootInstance(self, instance):
410 """Reboot an instance.
413 # For some reason if we do a 'send-key ctrl-alt-delete' to the control
414 # socket the instance will stop, but now power up again. So we'll resort
415 # to shutdown and restart.
416 pidfile, pid, alive = self._InstancePidAlive(instance.name)
418 raise errors.HypervisorError("Failed to reboot instance %s: not running" %
420 # StopInstance will delete the saved KVM runtime so:
421 # ...first load it...
422 kvm_runtime = self._LoadKVMRuntime(instance)
423 # ...now we can safely call StopInstance...
424 if not self.StopInstance(instance):
425 self.StopInstance(instance, force=True)
426 # ...and finally we can save it again, and execute it...
427 self._SaveKVMRuntime(instance, kvm_runtime)
428 self._ExecuteKVMRuntime(instance, kvm_runtime)
430 def MigrationInfo(self, instance):
431 """Get instance information to perform a migration.
433 @type instance: L{objects.Instance}
434 @param instance: instance to be migrated
436 @return: content of the KVM runtime file
439 return self._ReadKVMRuntime(instance.name)
441 def AcceptInstance(self, instance, info, target):
442 """Prepare to accept an instance.
444 @type instance: L{objects.Instance}
445 @param instance: instance to be accepted
447 @param info: content of the KVM runtime file on the source node
449 @param target: target host (usually ip), on this node
452 kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
453 incoming_address = (target, constants.KVM_MIGRATION_PORT)
454 self._ExecuteKVMRuntime(instance, kvm_runtime, incoming=incoming_address)
456 def FinalizeMigration(self, instance, info, success):
457 """Finalize an instance migration.
459 Stop the incoming mode KVM.
461 @type instance: L{objects.Instance}
462 @param instance: instance whose migration is being aborted
466 self._WriteKVMRuntime(instance.name, info)
468 self.StopInstance(instance, force=True)
470 def MigrateInstance(self, instance_name, target, live):
471 """Migrate an instance to a target node.
473 The migration will not be attempted if the instance is not
476 @type instance_name: string
477 @param instance_name: name of the instance to be migrated
479 @param target: ip address of the target node
481 @param live: perform a live migration
484 pidfile, pid, alive = self._InstancePidAlive(instance_name)
486 raise errors.HypervisorError("Instance not running, cannot migrate")
489 self._CallMonitorCommand(instance_name, 'stop')
491 migrate_command = ('migrate -d tcp:%s:%s' %
492 (target, constants.KVM_MIGRATION_PORT))
493 self._CallMonitorCommand(instance_name, migrate_command)
495 info_command = 'info migrate'
498 result = self._CallMonitorCommand(instance_name, info_command)
499 match = self._MIGRATION_STATUS_RE.search(result.stdout)
501 raise errors.HypervisorError("Unknown 'info migrate' result: %s" %
504 status = match.group(1)
505 if status == 'completed':
507 elif status == 'active':
509 elif status == 'failed' or status == 'cancelled':
511 self._CallMonitorCommand(instance_name, 'cont')
512 raise errors.HypervisorError("Migration %s at the kvm level" %
515 logging.info("KVM: unknown migration status '%s'" % status)
518 utils.KillProcess(pid)
519 utils.RemoveFile(pidfile)
520 utils.RemoveFile(self._InstanceMonitor(instance_name))
521 utils.RemoveFile(self._InstanceSerial(instance_name))
522 utils.RemoveFile(self._InstanceKVMRuntime(instance_name))
524 def GetNodeInfo(self):
525 """Return information about the node.
527 @return: a dict with the following keys (values in MiB):
528 - memory_total: the total memory size on the node
529 - memory_free: the available memory on the node for instances
530 - memory_dom0: the memory used by the node itself, if available
533 # global ram usage from the xm info command
536 # note: in xen 3, memory has changed to total_memory
538 fh = file("/proc/meminfo")
540 data = fh.readlines()
543 except EnvironmentError, err:
544 raise errors.HypervisorError("Failed to list node info: %s" % err)
549 splitfields = line.split(":", 1)
551 if len(splitfields) > 1:
552 key = splitfields[0].strip()
553 val = splitfields[1].strip()
554 if key == 'MemTotal':
555 result['memory_total'] = int(val.split()[0])/1024
556 elif key in ('MemFree', 'Buffers', 'Cached'):
557 sum_free += int(val.split()[0])/1024
558 elif key == 'Active':
559 result['memory_dom0'] = int(val.split()[0])/1024
560 result['memory_free'] = sum_free
564 fh = open("/proc/cpuinfo")
566 cpu_total = len(re.findall("(?m)^processor\s*:\s*[0-9]+\s*$",
570 except EnvironmentError, err:
571 raise errors.HypervisorError("Failed to list node info: %s" % err)
572 result['cpu_total'] = cpu_total
577 def GetShellCommandForConsole(instance):
578 """Return a command for connecting to the console of an instance.
581 # TODO: we can either try the serial socket or suggest vnc
582 return "echo Console not available for the kvm hypervisor yet"
585 """Verify the hypervisor.
587 Check that the binary exists.
590 if not os.path.exists(constants.KVM_PATH):
591 return "The kvm binary ('%s') does not exist." % constants.KVM_PATH
592 if not os.path.exists(constants.SOCAT_PATH):
593 return "The socat binary ('%s') does not exist." % constants.SOCAT_PATH
597 def CheckParameterSyntax(cls, hvparams):
598 """Check the given parameters for validity.
600 For the KVM hypervisor, this only check the existence of the
604 @param hvparams: dictionary with parameter names/value
605 @raise errors.HypervisorError: when a parameter is not valid
608 super(KVMHypervisor, cls).CheckParameterSyntax(hvparams)
610 if not hvparams[constants.HV_KERNEL_PATH]:
611 raise errors.HypervisorError("Need a kernel for the instance")
613 if not os.path.isabs(hvparams[constants.HV_KERNEL_PATH]):
614 raise errors.HypervisorError("The kernel path must be an absolute path")
616 if hvparams[constants.HV_INITRD_PATH]:
617 if not os.path.isabs(hvparams[constants.HV_INITRD_PATH]):
618 raise errors.HypervisorError("The initrd path must be an absolute path"
621 def ValidateParameters(self, hvparams):
622 """Check the given parameters for validity.
624 For the KVM hypervisor, this checks the existence of the
628 super(KVMHypervisor, self).ValidateParameters(hvparams)
630 kernel_path = hvparams[constants.HV_KERNEL_PATH]
631 if not os.path.isfile(kernel_path):
632 raise errors.HypervisorError("Instance kernel '%s' not found or"
633 " not a file" % kernel_path)
634 initrd_path = hvparams[constants.HV_INITRD_PATH]
635 if initrd_path and not os.path.isfile(initrd_path):
636 raise errors.HypervisorError("Instance initrd '%s' not found or"
637 " not a file" % initrd_path)