(instance.name, target_node))
+class LUMigrateInstance(LogicalUnit):
+ """Migrate an instance.
+
+ This is migration without shutting down, compared to the failover,
+ which is done with shutdown.
+
+ """
+ HPATH = "instance-migrate"
+ HTYPE = constants.HTYPE_INSTANCE
+ _OP_REQP = ["instance_name", "live", "cleanup"]
+
+ REQ_BGL = False
+
+ def ExpandNames(self):
+ self._ExpandAndLockInstance()
+ self.needed_locks[locking.LEVEL_NODE] = []
+ self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE
+
+ def DeclareLocks(self, level):
+ if level == locking.LEVEL_NODE:
+ self._LockInstancesNodes()
+
+ def BuildHooksEnv(self):
+ """Build hooks env.
+
+ This runs on master, primary and secondary nodes of the instance.
+
+ """
+ env = _BuildInstanceHookEnvByObject(self, self.instance)
+ nl = [self.cfg.GetMasterNode()] + list(self.instance.secondary_nodes)
+ return env, nl, nl
+
+ def CheckPrereq(self):
+ """Check prerequisites.
+
+ This checks that the instance is in the cluster.
+
+ """
+ instance = self.cfg.GetInstanceInfo(
+ self.cfg.ExpandInstanceName(self.op.instance_name))
+ if instance is None:
+ raise errors.OpPrereqError("Instance '%s' not known" %
+ self.op.instance_name)
+
+ if instance.disk_template != constants.DT_DRBD8:
+ raise errors.OpPrereqError("Instance's disk layout is not"
+ " drbd8, cannot migrate.")
+
+ secondary_nodes = instance.secondary_nodes
+ if not secondary_nodes:
+ raise errors.ProgrammerError("no secondary node but using "
+ "drbd8 disk template")
+
+ i_be = self.cfg.GetClusterInfo().FillBE(instance)
+
+ target_node = secondary_nodes[0]
+ # check memory requirements on the secondary node
+ _CheckNodeFreeMemory(self, target_node, "migrating instance %s" %
+ instance.name, i_be[constants.BE_MEMORY],
+ instance.hypervisor)
+
+ # check bridge existance
+ brlist = [nic.bridge for nic in instance.nics]
+ result = self.rpc.call_bridges_exist(target_node, brlist)
+ if result.failed or not result.data:
+ raise errors.OpPrereqError("One or more target bridges %s does not"
+ " exist on destination node '%s'" %
+ (brlist, target_node))
+
+ if not self.op.cleanup:
+ result = self.rpc.call_instance_migratable(instance.primary_node,
+ instance)
+ msg = result.RemoteFailMsg()
+ if msg:
+ raise errors.OpPrereqError("Can't migrate: %s - please use failover" %
+ msg)
+
+ self.instance = instance
+
+ def _WaitUntilSync(self):
+ """Poll with custom rpc for disk sync.
+
+ This uses our own step-based rpc call.
+
+ """
+ self.feedback_fn("* wait until resync is done")
+ all_done = False
+ while not all_done:
+ all_done = True
+ result = self.rpc.call_drbd_wait_sync(self.all_nodes,
+ self.nodes_ip,
+ self.instance.disks)
+ min_percent = 100
+ for node, nres in result.items():
+ msg = nres.RemoteFailMsg()
+ if msg:
+ raise errors.OpExecError("Cannot resync disks on node %s: %s" %
+ (node, msg))
+ node_done, node_percent = nres.data[1]
+ all_done = all_done and node_done
+ if node_percent is not None:
+ min_percent = min(min_percent, node_percent)
+ if not all_done:
+ if min_percent < 100:
+ self.feedback_fn(" - progress: %.1f%%" % min_percent)
+ time.sleep(2)
+
+ def _EnsureSecondary(self, node):
+ """Demote a node to secondary.
+
+ """
+ self.feedback_fn("* switching node %s to secondary mode" % node)
+
+ for dev in self.instance.disks:
+ self.cfg.SetDiskID(dev, node)
+
+ result = self.rpc.call_blockdev_close(node, self.instance.name,
+ self.instance.disks)
+ msg = result.RemoteFailMsg()
+ if msg:
+ raise errors.OpExecError("Cannot change disk to secondary on node %s,"
+ " error %s" % (node, msg))
+
+ def _GoStandalone(self):
+ """Disconnect from the network.
+
+ """
+ self.feedback_fn("* changing into standalone mode")
+ result = self.rpc.call_drbd_disconnect_net(self.all_nodes, self.nodes_ip,
+ self.instance.disks)
+ for node, nres in result.items():
+ msg = nres.RemoteFailMsg()
+ if msg:
+ raise errors.OpExecError("Cannot disconnect disks node %s,"
+ " error %s" % (node, msg))
+
+ def _GoReconnect(self, multimaster):
+ """Reconnect to the network.
+
+ """
+ if multimaster:
+ msg = "dual-master"
+ else:
+ msg = "single-master"
+ self.feedback_fn("* changing disks into %s mode" % msg)
+ result = self.rpc.call_drbd_attach_net(self.all_nodes, self.nodes_ip,
+ self.instance.disks,
+ self.instance.name, multimaster)
+ for node, nres in result.items():
+ msg = nres.RemoteFailMsg()
+ if msg:
+ raise errors.OpExecError("Cannot change disks config on node %s,"
+ " error: %s" % (node, msg))
+
+ def _ExecCleanup(self):
+ """Try to cleanup after a failed migration.
+
+ The cleanup is done by:
+ - check that the instance is running only on one node
+ (and update the config if needed)
+ - change disks on its secondary node to secondary
+ - wait until disks are fully synchronized
+ - disconnect from the network
+ - change disks into single-master mode
+ - wait again until disks are fully synchronized
+
+ """
+ instance = self.instance
+ target_node = self.target_node
+ source_node = self.source_node
+
+ # check running on only one node
+ self.feedback_fn("* checking where the instance actually runs"
+ " (if this hangs, the hypervisor might be in"
+ " a bad state)")
+ ins_l = self.rpc.call_instance_list(self.all_nodes, [instance.hypervisor])
+ for node, result in ins_l.items():
+ result.Raise()
+ if not isinstance(result.data, list):
+ raise errors.OpExecError("Can't contact node '%s'" % node)
+
+ runningon_source = instance.name in ins_l[source_node].data
+ runningon_target = instance.name in ins_l[target_node].data
+
+ if runningon_source and runningon_target:
+ raise errors.OpExecError("Instance seems to be running on two nodes,"
+ " or the hypervisor is confused. You will have"
+ " to ensure manually that it runs only on one"
+ " and restart this operation.")
+
+ if not (runningon_source or runningon_target):
+ raise errors.OpExecError("Instance does not seem to be running at all."
+ " In this case, it's safer to repair by"
+ " running 'gnt-instance stop' to ensure disk"
+ " shutdown, and then restarting it.")
+
+ if runningon_target:
+ # the migration has actually succeeded, we need to update the config
+ self.feedback_fn("* instance running on secondary node (%s),"
+ " updating config" % target_node)
+ instance.primary_node = target_node
+ self.cfg.Update(instance)
+ demoted_node = source_node
+ else:
+ self.feedback_fn("* instance confirmed to be running on its"
+ " primary node (%s)" % source_node)
+ demoted_node = target_node
+
+ self._EnsureSecondary(demoted_node)
+ try:
+ self._WaitUntilSync()
+ except errors.OpExecError:
+ # we ignore here errors, since if the device is standalone, it
+ # won't be able to sync
+ pass
+ self._GoStandalone()
+ self._GoReconnect(False)
+ self._WaitUntilSync()
+
+ self.feedback_fn("* done")
+
+ def _ExecMigration(self):
+ """Migrate an instance.
+
+ The migrate is done by:
+ - change the disks into dual-master mode
+ - wait until disks are fully synchronized again
+ - migrate the instance
+ - change disks on the new secondary node (the old primary) to secondary
+ - wait until disks are fully synchronized
+ - change disks into single-master mode
+
+ """
+ instance = self.instance
+ target_node = self.target_node
+ source_node = self.source_node
+
+ self.feedback_fn("* checking disk consistency between source and target")
+ for dev in instance.disks:
+ if not _CheckDiskConsistency(self, dev, target_node, False):
+ raise errors.OpExecError("Disk %s is degraded or not fully"
+ " synchronized on target node,"
+ " aborting migrate." % dev.iv_name)
+
+ self._EnsureSecondary(target_node)
+ self._GoStandalone()
+ self._GoReconnect(True)
+ self._WaitUntilSync()
+
+ self.feedback_fn("* migrating instance to %s" % target_node)
+ time.sleep(10)
+ result = self.rpc.call_instance_migrate(source_node, instance,
+ self.nodes_ip[target_node],
+ self.op.live)
+ msg = result.RemoteFailMsg()
+ if msg:
+ logging.error("Instance migration failed, trying to revert"
+ " disk status: %s", msg)
+ try:
+ self._EnsureSecondary(target_node)
+ self._GoStandalone()
+ self._GoReconnect(False)
+ self._WaitUntilSync()
+ except errors.OpExecError, err:
+ self.LogWarning("Migration failed and I can't reconnect the"
+ " drives: error '%s'\n"
+ "Please look and recover the instance status" %
+ str(err))
+
+ raise errors.OpExecError("Could not migrate instance %s: %s" %
+ (instance.name, msg))
+ time.sleep(10)
+
+ instance.primary_node = target_node
+ # distribute new instance config to the other nodes
+ self.cfg.Update(instance)
+
+ self._EnsureSecondary(source_node)
+ self._WaitUntilSync()
+ self._GoStandalone()
+ self._GoReconnect(False)
+ self._WaitUntilSync()
+
+ self.feedback_fn("* done")
+
+ def Exec(self, feedback_fn):
+ """Perform the migration.
+
+ """
+ self.feedback_fn = feedback_fn
+
+ self.source_node = self.instance.primary_node
+ self.target_node = self.instance.secondary_nodes[0]
+ self.all_nodes = [self.source_node, self.target_node]
+ self.nodes_ip = {
+ self.source_node: self.cfg.GetNodeInfo(self.source_node).secondary_ip,
+ self.target_node: self.cfg.GetNodeInfo(self.target_node).secondary_ip,
+ }
+ if self.op.cleanup:
+ return self._ExecCleanup()
+ else:
+ return self._ExecMigration()
+
+
def _CreateBlockDevOnPrimary(lu, node, instance, device, info):
"""Create a tree of block devices on the primary node.
"""
if device.children:
for child in device.children:
- if not _CreateBlockDevOnPrimary(lu, node, instance, child, info):
- return False
+ _CreateBlockDevOnPrimary(lu, node, instance, child, info)
lu.cfg.SetDiskID(device, node)
new_id = lu.rpc.call_blockdev_create(node, device, device.size,
instance.name, True, info)
if new_id.failed or not new_id.data:
- return False
+ raise errors.OpExecError("Can't create block device %s on primary"
+ " node %s" % (device, node))
if device.physical_id is None:
device.physical_id = new_id
- return True
def _CreateBlockDevOnSecondary(lu, node, instance, device, force, info):
"""
if device.CreateOnSecondary():
force = True
+
if device.children:
for child in device.children:
- if not _CreateBlockDevOnSecondary(lu, node, instance,
- child, force, info):
- return False
+ _CreateBlockDevOnSecondary(lu, node, instance, child, force, info)
if not force:
- return True
+ return
+
lu.cfg.SetDiskID(device, node)
new_id = lu.rpc.call_blockdev_create(node, device, device.size,
instance.name, False, info)
if new_id.failed or not new_id.data:
- return False
+ raise errors.OpExecError("Can't create block device %s on secondary"
+ " node %s" % (device, node))
if device.physical_id is None:
device.physical_id = new_id
- return True
def _GenerateUniqueNames(lu, exts):
minors = lu.cfg.AllocateDRBDMinor(
[primary_node, remote_node] * len(disk_info), instance_name)
- names = _GenerateUniqueNames(lu,
- [".disk%d_%s" % (i, s)
- for i in range(disk_count)
- for s in ("data", "meta")
- ])
+ names = []
+ for lv_prefix in _GenerateUniqueNames(lu, [".disk%d" % i
+ for i in range(disk_count)]):
+ names.append(lv_prefix + "_data")
+ names.append(lv_prefix + "_meta")
for idx, disk in enumerate(disk_info):
disk_index = idx + base_index
disk_dev = _GenerateDRBD8Branch(lu, primary_node, remote_node,
file_storage_dir)
if result.failed or not result.data:
- logging.error("Could not connect to node '%s'", instance.primary_node)
- return False
+ raise errors.OpExecError("Could not connect to node '%s'" %
+ instance.primary_node)
if not result.data[0]:
- logging.error("Failed to create directory '%s'", file_storage_dir)
- return False
+ raise errors.OpExecError("Failed to create directory '%s'" %
+ file_storage_dir)
# Note: this needs to be kept in sync with adding of disks in
# LUSetInstanceParams
device.iv_name, instance.name)
#HARDCODE
for secondary_node in instance.secondary_nodes:
- if not _CreateBlockDevOnSecondary(lu, secondary_node, instance,
- device, False, info):
- logging.error("Failed to create volume %s (%s) on secondary node %s!",
- device.iv_name, device, secondary_node)
- return False
+ _CreateBlockDevOnSecondary(lu, secondary_node, instance,
+ device, False, info)
#HARDCODE
- if not _CreateBlockDevOnPrimary(lu, instance.primary_node,
- instance, device, info):
- logging.error("Failed to create volume %s on primary!", device.iv_name)
- return False
-
- return True
+ _CreateBlockDevOnPrimary(lu, instance.primary_node,
+ instance, device, info)
def _RemoveDisks(lu, instance):
)
feedback_fn("* creating instance disks...")
- if not _CreateDisks(self, iobj):
- _RemoveDisks(self, iobj)
- self.cfg.ReleaseDRBDMinors(instance)
- raise errors.OpExecError("Device creation failed, reverting...")
+ try:
+ _CreateDisks(self, iobj)
+ except errors.OpExecError:
+ self.LogWarning("Device creation failed, reverting...")
+ try:
+ _RemoveDisks(self, iobj)
+ finally:
+ self.cfg.ReleaseDRBDMinors(instance)
+ raise
feedback_fn("adding instance %s to cluster config" % instance)
# _Create...OnPrimary (which forces the creation), even if we
# are talking about the secondary node
for new_lv in new_lvs:
- if not _CreateBlockDevOnPrimary(self, tgt_node, instance, new_lv,
- _GetInstanceInfoText(instance)):
- raise errors.OpExecError("Failed to create new LV named '%s' on"
- " node '%s'" %
- (new_lv.logical_id[1], tgt_node))
+ _CreateBlockDevOnPrimary(self, tgt_node, instance, new_lv,
+ _GetInstanceInfoText(instance))
# Step: for each lv, detach+rename*2+attach
self.proc.LogStep(4, steps_total, "change drbd configuration")
old_node = self.tgt_node
new_node = self.new_node
pri_node = instance.primary_node
+ nodes_ip = {
+ old_node: self.cfg.GetNodeInfo(old_node).secondary_ip,
+ new_node: self.cfg.GetNodeInfo(new_node).secondary_ip,
+ pri_node: self.cfg.GetNodeInfo(pri_node).secondary_ip,
+ }
# Step: check device activation
self.proc.LogStep(1, steps_total, "check device existence")
# _Create...OnPrimary (which forces the creation), even if we
# are talking about the secondary node
for new_lv in dev.children:
- if not _CreateBlockDevOnPrimary(self, new_node, instance, new_lv,
- _GetInstanceInfoText(instance)):
- raise errors.OpExecError("Failed to create new LV named '%s' on"
- " node '%s'" %
- (new_lv.logical_id[1], new_node))
+ _CreateBlockDevOnPrimary(self, new_node, instance, new_lv,
+ _GetInstanceInfoText(instance))
# Step 4: dbrd minors and drbd setups changes
# after this, we must manually remove the drbd minors on both the
for idx, (dev, new_minor) in enumerate(zip(instance.disks, minors)):
size = dev.size
info("activating a new drbd on %s for disk/%d" % (new_node, idx))
- # create new devices on new_node
- if pri_node == dev.logical_id[0]:
- new_logical_id = (pri_node, new_node,
- dev.logical_id[2], dev.logical_id[3], new_minor,
- dev.logical_id[5])
+ # create new devices on new_node; note that we create two IDs:
+ # one without port, so the drbd will be activated without
+ # networking information on the new node at this stage, and one
+ # with network, for the latter activation in step 4
+ (o_node1, o_node2, o_port, o_minor1, o_minor2, o_secret) = dev.logical_id
+ if pri_node == o_node1:
+ p_minor = o_minor1
else:
- new_logical_id = (new_node, pri_node,
- dev.logical_id[2], new_minor, dev.logical_id[4],
- dev.logical_id[5])
- iv_names[idx] = (dev, dev.children, new_logical_id)
+ p_minor = o_minor2
+
+ new_alone_id = (pri_node, new_node, None, p_minor, new_minor, o_secret)
+ new_net_id = (pri_node, new_node, o_port, p_minor, new_minor, o_secret)
+
+ iv_names[idx] = (dev, dev.children, new_net_id)
logging.debug("Allocated new_minor: %s, new_logical_id: %s", new_minor,
- new_logical_id)
+ new_net_id)
new_drbd = objects.Disk(dev_type=constants.LD_DRBD8,
- logical_id=new_logical_id,
+ logical_id=new_alone_id,
children=dev.children)
- if not _CreateBlockDevOnSecondary(self, new_node, instance,
- new_drbd, False,
- _GetInstanceInfoText(instance)):
+ try:
+ _CreateBlockDevOnSecondary(self, new_node, instance, new_drbd, False,
+ _GetInstanceInfoText(instance))
+ except error.BlockDeviceError:
self.cfg.ReleaseDRBDMinors(instance.name)
- raise errors.OpExecError("Failed to create new DRBD on"
- " node '%s'" % new_node)
+ raise
for idx, dev in enumerate(instance.disks):
# we have new devices, shutdown the drbd on the old secondary
hint="Please cleanup this device manually as soon as possible")
info("detaching primary drbds from the network (=> standalone)")
- done = 0
- for idx, dev in enumerate(instance.disks):
- cfg.SetDiskID(dev, pri_node)
- # set the network part of the physical (unique in bdev terms) id
- # to None, meaning detach from network
- dev.physical_id = (None, None, None, None) + dev.physical_id[4:]
- # and 'find' the device, which will 'fix' it to match the
- # standalone state
- result = self.rpc.call_blockdev_find(pri_node, dev)
- if not result.failed and result.data:
- done += 1
- else:
- warning("Failed to detach drbd disk/%d from network, unusual case" %
- idx)
+ result = self.rpc.call_drbd_disconnect_net([pri_node], nodes_ip,
+ instance.disks)[pri_node]
- if not done:
- # no detaches succeeded (very unlikely)
+ msg = result.RemoteFailMsg()
+ if msg:
+ # detaches didn't succeed (unlikely)
self.cfg.ReleaseDRBDMinors(instance.name)
- raise errors.OpExecError("Can't detach at least one DRBD from old node")
+ raise errors.OpExecError("Can't detach the disks from the network on"
+ " old node: %s" % (msg,))
# if we managed to detach at least one, we update all the disks of
# the instance to point to the new secondary
# and now perform the drbd attach
info("attaching primary drbds to new secondary (standalone => connected)")
- for idx, dev in enumerate(instance.disks):
- info("attaching primary drbd for disk/%d to new secondary node" % idx)
- # since the attach is smart, it's enough to 'find' the device,
- # it will automatically activate the network, if the physical_id
- # is correct
- cfg.SetDiskID(dev, pri_node)
- logging.debug("Disk to attach: %s", dev)
- result = self.rpc.call_blockdev_find(pri_node, dev)
- if result.failed or not result.data:
- warning("can't attach drbd disk/%d to new secondary!" % idx,
- "please do a gnt-instance info to see the status of disks")
+ result = self.rpc.call_drbd_attach_net([pri_node, new_node], nodes_ip,
+ instance.disks, instance.name,
+ False)
+ for to_node, to_result in result.items():
+ msg = to_result.RemoteFailMsg()
+ if msg:
+ warning("can't attach drbd disks on node %s: %s", to_node, msg,
+ hint="please do a gnt-instance info to see the"
+ " status of disks")
# this can fail as the old devices are degraded and _WaitForSync
# does a combined result over all disks, so we don't check its
" missing on its primary node" % miss_mem)
if be_new[constants.BE_AUTO_BALANCE]:
- for node, nres in instance.secondary_nodes.iteritems():
+ for node, nres in nodeinfo.iteritems():
+ if node not in instance.secondary_nodes:
+ continue
if nres.failed or not isinstance(nres.data, dict):
self.warn.append("Can't get info from secondary node %s" % node)
elif be_new[constants.BE_MEMORY] > nres.data['memory_free']:
# Note: this needs to be kept in sync with _CreateDisks
#HARDCODE
for secondary_node in instance.secondary_nodes:
- if not _CreateBlockDevOnSecondary(self, secondary_node, instance,
- new_disk, False, info):
+ try:
+ _CreateBlockDevOnSecondary(self, secondary_node, instance,
+ new_disk, False, info)
+ except error.OpExecError, err:
self.LogWarning("Failed to create volume %s (%s) on"
- " secondary node %s!",
- new_disk.iv_name, new_disk, secondary_node)
+ " secondary node %s: %s",
+ new_disk.iv_name, new_disk, secondary_node, err)
#HARDCODE
- if not _CreateBlockDevOnPrimary(self, instance.primary_node,
- instance, new_disk, info):
- self.LogWarning("Failed to create volume %s on primary!",
- new_disk.iv_name)
+ try:
+ _CreateBlockDevOnPrimary(self, instance.primary_node,
+ instance, new_disk, info)
+ except errors.OpExecError, err:
+ self.LogWarning("Failed to create volume %s on primary: %s",
+ new_disk.iv_name, err)
result.append(("disk/%d" % disk_idx_base, "add:size=%s,mode=%s" %
(new_disk.size, new_disk.mode)))
else:
if self.dst_node is None:
# This is wrong node name, not a non-locked node
raise errors.OpPrereqError("Wrong node name %s" % self.op.target_node)
- _CheckNodeOnline(self, self.op.target_node)
+ _CheckNodeOnline(self, self.dst_node.name)
# instance disk type verification
for disk in self.instance.disks:
snap_disks = []
+ # set the disks ID correctly since call_instance_start needs the
+ # correct drbd minor to create the symlinks
+ for disk in instance.disks:
+ self.cfg.SetDiskID(disk, src_node)
+
try:
for disk in instance.disks:
# new_dev_name will be a snapshot of an lvm leaf of the one we passed