hv_base.ParamInSet(True, constants.HT_KVM_VALID_DISK_TYPES),
constants.HV_USB_MOUSE:
hv_base.ParamInSet(False, constants.HT_KVM_VALID_MOUSE_TYPES),
+ constants.HV_MIGRATION_PORT: hv_base.NET_PORT_CHECK,
+ constants.HV_USE_LOCALTIME: hv_base.NO_CHECK,
+ constants.HV_DISK_CACHE:
+ hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES),
}
_MIGRATION_STATUS_RE = re.compile('Migration\s+status:\s+(\w+)',
re.M | re.I)
+ _MIGRATION_INFO_MAX_BAD_ANSWERS = 5
+ _MIGRATION_INFO_RETRY_DELAY = 2
_KVM_NETWORK_SCRIPT = constants.SYSCONFDIR + "/ganeti/kvm-vif-bridge"
dirs = [(dname, constants.RUN_DIRS_MODE) for dname in self._DIRS]
utils.EnsureDirs(dirs)
+ @classmethod
+ def _InstancePidFile(cls, instance_name):
+ """Returns the instance pidfile.
+
+ """
+ return utils.PathJoin(cls._PIDS_DIR, instance_name)
+
def _InstancePidAlive(self, instance_name):
"""Returns the instance pid and pidfile
"""
- pidfile = "%s/%s" % (self._PIDS_DIR, instance_name)
+ pidfile = self._InstancePidFile(instance_name)
pid = utils.ReadPidFile(pidfile)
alive = utils.IsProcessAlive(pid)
return (pidfile, pid, alive)
+ def _CheckDown(self, instance_name):
+ """Raises an error unless the given instance is down.
+
+ """
+ alive = self._InstancePidAlive(instance_name)[2]
+ if alive:
+ raise errors.HypervisorError("Failed to start instance %s: %s" %
+ (instance_name, "already running"))
+
@classmethod
def _InstanceMonitor(cls, instance_name):
"""Returns the instance monitor socket name
"""
- return '%s/%s.monitor' % (cls._CTRL_DIR, instance_name)
+ return utils.PathJoin(cls._CTRL_DIR, "%s.monitor" % instance_name)
@classmethod
def _InstanceSerial(cls, instance_name):
"""Returns the instance serial socket name
"""
- return '%s/%s.serial' % (cls._CTRL_DIR, instance_name)
+ return utils.PathJoin(cls._CTRL_DIR, "%s.serial" % instance_name)
+
+ @staticmethod
+ def _SocatUnixConsoleParams():
+ """Returns the correct parameters for socat
+
+ If we have a new-enough socat we can use raw mode with an escape character.
+
+ """
+ if constants.SOCAT_USE_ESCAPE:
+ return "raw,echo=0,escape=%s" % constants.SOCAT_ESCAPE_CODE
+ else:
+ return "echo=0,icanon=0"
@classmethod
def _InstanceKVMRuntime(cls, instance_name):
"""Returns the instance KVM runtime filename
"""
- return '%s/%s.runtime' % (cls._CONF_DIR, instance_name)
+ return utils.PathJoin(cls._CONF_DIR, "%s.runtime" % instance_name)
@classmethod
def _RemoveInstanceRuntimeFiles(cls, pidfile, instance_name):
script = StringIO()
script.write("#!/bin/sh\n")
script.write("# this is autogenerated by Ganeti, please do not edit\n#\n")
+ script.write("PATH=$PATH:/sbin:/usr/sbin\n")
script.write("export INSTANCE=%s\n" % instance.name)
script.write("export MAC=%s\n" % nic.mac)
if nic.ip:
script.write(" # Execute the user-specific vif file\n")
script.write(" %s\n" % self._KVM_NETWORK_SCRIPT)
script.write("else\n")
- script.write(" /sbin/ifconfig $INTERFACE 0.0.0.0 up\n")
+ script.write(" ifconfig $INTERFACE 0.0.0.0 up\n")
if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
script.write(" # Connect the interface to the bridge\n")
- script.write(" /usr/sbin/brctl addif $BRIDGE $INTERFACE\n")
+ script.write(" brctl addif $BRIDGE $INTERFACE\n")
elif nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_ROUTED:
+ if not nic.ip:
+ raise errors.HypervisorError("nic/%d is routed, but has no ip." % seq)
script.write(" # Route traffic targeted at the IP to the interface\n")
if nic.nicparams[constants.NIC_LINK]:
- script.write(" while /sbin/ip rule del dev $INTERFACE; do :; done\n")
- script.write(" /sbin/ip rule add dev $INTERFACE table $LINK\n")
- script.write(" /sbin/ip route replace $IP/32 table $LINK"
+ script.write(" while ip rule del dev $INTERFACE; do :; done\n")
+ script.write(" ip rule add dev $INTERFACE table $LINK\n")
+ script.write(" ip route replace $IP table $LINK proto static"
" dev $INTERFACE\n")
else:
- script.write(" /sbin/ip route replace $IP/32 dev $INTERFACE\n")
- interface_proxy_arp = "/proc/sys/net/ipv4/conf/$INTERFACE/proxy_arp"
- interface_forwarding = "/proc/sys/net/ipv4/conf/$INTERFACE/forwarding"
- script.write(" /bin/echo 1 > %s\n" % interface_proxy_arp)
- script.write(" /bin/echo 1 > %s\n" % interface_forwarding)
+ script.write(" ip route replace $IP proto static"
+ " dev $INTERFACE\n")
+ interface_v4_conf = "/proc/sys/net/ipv4/conf/$INTERFACE"
+ interface_v6_conf = "/proc/sys/net/ipv6/conf/$INTERFACE"
+ script.write(" if [ -d %s ]; then\n" % interface_v4_conf)
+ script.write(" echo 1 > %s/proxy_arp\n" % interface_v4_conf)
+ script.write(" echo 1 > %s/forwarding\n" % interface_v4_conf)
+ script.write(" fi\n")
+ script.write(" if [ -d %s ]; then\n" % interface_v6_conf)
+ script.write(" echo 1 > %s/proxy_ndp\n" % interface_v6_conf)
+ script.write(" echo 1 > %s/forwarding\n" % interface_v6_conf)
+ script.write(" fi\n")
script.write("fi\n\n")
# As much as we'd like to put this in our _ROOT_DIR, that will happen to be
# mounted noexec sometimes, so we'll have to find another place.
"""
result = []
for name in os.listdir(self._PIDS_DIR):
- filename = "%s/%s" % (self._PIDS_DIR, name)
+ filename = utils.PathJoin(self._PIDS_DIR, name)
if utils.IsProcessAlive(utils.ReadPidFile(filename)):
result.append(name)
return result
@return: tuple (name, id, memory, vcpus, stat, times)
"""
- pidfile, pid, alive = self._InstancePidAlive(instance_name)
+ _, pid, alive = self._InstancePidAlive(instance_name)
if not alive:
return None
- cmdline_file = "/proc/%s/cmdline" % pid
+ cmdline_file = utils.PathJoin("/proc", str(pid), "cmdline")
try:
cmdline = utils.ReadFile(cmdline_file)
except EnvironmentError, err:
"""
data = []
for name in os.listdir(self._PIDS_DIR):
- filename = "%s/%s" % (self._PIDS_DIR, name)
+ filename = utils.PathJoin(self._PIDS_DIR, name)
if utils.IsProcessAlive(utils.ReadPidFile(filename)):
try:
info = self.GetInstanceInfo(name)
- except errors.HypervisorError, err:
+ except errors.HypervisorError:
continue
if info:
data.append(info)
"""Generate KVM information to start an instance.
"""
- pidfile, pid, alive = self._InstancePidAlive(instance.name)
+ pidfile = self._InstancePidFile(instance.name)
kvm = constants.KVM_PATH
kvm_cmd = [kvm]
kvm_cmd.extend(['-m', instance.beparams[constants.BE_MEMORY]])
if_val = ',if=virtio'
else:
if_val = ',if=%s' % disk_type
+ # Cache mode
+ disk_cache = hvp[constants.HV_DISK_CACHE]
+ if disk_cache != constants.HT_CACHE_DEFAULT:
+ cache_val = ",cache=%s" % disk_cache
+ else:
+ cache_val = ""
for cfdev, dev_path in block_devices:
if cfdev.mode != constants.DISK_RDWR:
raise errors.HypervisorError("Instance has read-only disks which"
else:
boot_val = ''
- drive_val = 'file=%s,format=raw%s%s' % (dev_path, if_val, boot_val)
+ drive_val = 'file=%s,format=raw%s%s%s' % (dev_path, if_val, boot_val,
+ cache_val)
kvm_cmd.extend(['-drive', drive_val])
iso_image = hvp[constants.HV_CDROM_IMAGE_PATH]
vnc_arg = '%s:%d' % (vnc_bind_address, display)
else:
logging.error("Network port is not a valid VNC display (%d < %d)."
- " Not starting VNC" %
- (instance.network_port,
- constants.VNC_BASE_PORT))
+ " Not starting VNC", instance.network_port,
+ constants.VNC_BASE_PORT)
vnc_arg = 'none'
# Only allow tls and other option when not binding to a file, for now.
else:
kvm_cmd.extend(['-serial', 'none'])
+ if hvp[constants.HV_USE_LOCALTIME]:
+ kvm_cmd.extend(['-localtime'])
+
# Save the current instance nics, but defer their expansion as parameters,
# as we'll need to generate executable temp files for them.
kvm_nics = instance.nics
@param incoming: (target_host_ip, port)
"""
- pidfile, pid, alive = self._InstancePidAlive(instance.name)
hvp = instance.hvparams
- if alive:
- raise errors.HypervisorError("Failed to start instance %s: %s" %
- (instance.name, "already running"))
+ name = instance.name
+ self._CheckDown(name)
temp_files = []
result = utils.RunCmd(kvm_cmd)
if result.failed:
raise errors.HypervisorError("Failed to start instance %s: %s (%s)" %
- (instance.name, result.fail_reason,
- result.output))
+ (name, result.fail_reason, result.output))
- if not utils.IsProcessAlive(utils.ReadPidFile(pidfile)):
- raise errors.HypervisorError("Failed to start instance %s: %s" %
- (instance.name))
+ if not self._InstancePidAlive(name)[2]:
+ raise errors.HypervisorError("Failed to start instance %s" % name)
if vnc_pwd:
change_cmd = 'change vnc password %s' % vnc_pwd
"""Start an instance.
"""
- pidfile, pid, alive = self._InstancePidAlive(instance.name)
- if alive:
- raise errors.HypervisorError("Failed to start instance %s: %s" %
- (instance.name, "already running"))
-
+ self._CheckDown(instance.name)
kvm_runtime = self._GenerateKVMRuntime(instance, block_devices)
self._SaveKVMRuntime(instance, kvm_runtime)
self._ExecuteKVMRuntime(instance, kvm_runtime)
# For some reason if we do a 'send-key ctrl-alt-delete' to the control
# socket the instance will stop, but now power up again. So we'll resort
# to shutdown and restart.
- pidfile, pid, alive = self._InstancePidAlive(instance.name)
+ _, _, alive = self._InstancePidAlive(instance.name)
if not alive:
- raise errors.HypervisorError("Failed to reboot instance %s: not running" %
- (instance.name))
+ raise errors.HypervisorError("Failed to reboot instance %s:"
+ " not running" % instance.name)
# StopInstance will delete the saved KVM runtime so:
# ...first load it...
kvm_runtime = self._LoadKVMRuntime(instance)
"""
kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info)
- incoming_address = (target, constants.KVM_MIGRATION_PORT)
+ incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT])
self._ExecuteKVMRuntime(instance, kvm_runtime, incoming=incoming_address)
def FinalizeMigration(self, instance, info, success):
else:
self.StopInstance(instance, force=True)
- def MigrateInstance(self, instance_name, target, live):
+ def MigrateInstance(self, instance, target, live):
"""Migrate an instance to a target node.
The migration will not be attempted if the instance is not
currently running.
- @type instance_name: string
- @param instance_name: name of the instance to be migrated
+ @type instance: L{objects.Instance}
+ @param instance: the instance to be migrated
@type target: string
@param target: ip address of the target node
@type live: boolean
@param live: perform a live migration
"""
+ instance_name = instance.name
+ port = instance.hvparams[constants.HV_MIGRATION_PORT]
pidfile, pid, alive = self._InstancePidAlive(instance_name)
if not alive:
raise errors.HypervisorError("Instance not running, cannot migrate")
+ if not utils.TcpPing(target, port, live_port_needed=True):
+ raise errors.HypervisorError("Remote host %s not listening on port"
+ " %s, cannot migrate" % (target, port))
+
if not live:
self._CallMonitorCommand(instance_name, 'stop')
- migrate_command = ('migrate -d tcp:%s:%s' %
- (target, constants.KVM_MIGRATION_PORT))
+ migrate_command = 'migrate -d tcp:%s:%s' % (target, port)
self._CallMonitorCommand(instance_name, migrate_command)
info_command = 'info migrate'
done = False
+ broken_answers = 0
while not done:
result = self._CallMonitorCommand(instance_name, info_command)
match = self._MIGRATION_STATUS_RE.search(result.stdout)
if not match:
- raise errors.HypervisorError("Unknown 'info migrate' result: %s" %
- result.stdout)
+ broken_answers += 1
+ if not result.stdout:
+ logging.info("KVM: empty 'info migrate' result")
+ else:
+ logging.warning("KVM: unknown 'info migrate' result: %s",
+ result.stdout)
+ time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
else:
status = match.group(1)
if status == 'completed':
done = True
elif status == 'active':
- time.sleep(2)
+ # reset the broken answers count
+ broken_answers = 0
+ time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
elif status == 'failed' or status == 'cancelled':
if not live:
self._CallMonitorCommand(instance_name, 'cont')
raise errors.HypervisorError("Migration %s at the kvm level" %
status)
else:
- logging.info("KVM: unknown migration status '%s'" % status)
- time.sleep(2)
+ logging.warning("KVM: unknown migration status '%s'", status)
+ broken_answers += 1
+ time.sleep(self._MIGRATION_INFO_RETRY_DELAY)
+ if broken_answers >= self._MIGRATION_INFO_MAX_BAD_ANSWERS:
+ raise errors.HypervisorError("Too many 'info migrate' broken answers")
utils.KillProcess(pid)
self._RemoveInstanceRuntimeFiles(pidfile, instance_name)
"""
if hvparams[constants.HV_SERIAL_CONSOLE]:
- # FIXME: The socat shell is not perfect. In particular the way we start
- # it ctrl+c will close it, rather than being passed to the other end.
- # On the other hand if we pass the option 'raw' (or ignbrk=1) there
- # will be no way of exiting socat (except killing it from another shell)
- # and ctrl+c doesn't work anyway, printing ^C rather than being
- # interpreted by kvm. For now we'll leave it this way, which at least
- # allows a minimal interaction and changes on the machine.
- shell_command = ("%s STDIO,echo=0,icanon=0 UNIX-CONNECT:%s" %
- (constants.SOCAT_PATH,
+ shell_command = ("%s STDIO,%s UNIX-CONNECT:%s" %
+ (constants.SOCAT_PATH, cls._SocatUnixConsoleParams(),
utils.ShellQuote(cls._InstanceSerial(instance.name))))
else:
shell_command = "echo 'No serial shell for instance %s'" % instance.name