Synnefo » snf-ganeti » ganeti-local

#

#

# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful, but

# WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

# General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

# 02110-1301, USA.

"""Logical units dealing with instances."""

import OpenSSL

import copy

import itertools

import logging

import operator

import os

import time

from ganeti import compat

from ganeti import constants

from ganeti import errors

from ganeti import ht

from ganeti import hypervisor

from ganeti import locking

from ganeti.masterd import iallocator

from ganeti import masterd

from ganeti import netutils

from ganeti import objects

from ganeti import opcodes

from ganeti import pathutils

from ganeti import qlang

from ganeti import rpc

from ganeti import utils

from ganeti import query

from ganeti.cmdlib.base import NoHooksLU, LogicalUnit, _QueryBase, \

  ResultWithJobs, Tasklet

from ganeti.cmdlib.common import INSTANCE_ONLINE, INSTANCE_DOWN, \

  INSTANCE_NOT_RUNNING, CAN_CHANGE_INSTANCE_OFFLINE, _CheckNodeOnline, \

  _ShareAll, _GetDefaultIAllocator, _CheckInstanceNodeGroups, \

  _LoadNodeEvacResult, _CheckIAllocatorOrNode, _CheckParamsNotGlobal, \

  _IsExclusiveStorageEnabledNode, _CheckHVParams, _CheckOSParams, \

  _GetWantedInstances, _CheckInstancesNodeGroups, _AnnotateDiskParams, \

  _GetUpdatedParams, _ExpandInstanceName, _FindFaultyInstanceDisks, \

  _ComputeIPolicySpecViolation, _ComputeIPolicyInstanceViolation, \

  _CheckInstanceState, _ExpandNodeName

from ganeti.cmdlib.instance_utils import _AssembleInstanceDisks, \

  _BuildInstanceHookEnvByObject, _GetClusterDomainSecret, \

  _BuildInstanceHookEnv, _NICListToTuple, _NICToTuple, _CheckNodeNotDrained, \

  _RemoveDisks, _StartInstanceDisks, _ShutdownInstanceDisks, \

  _RemoveInstance, _ExpandCheckDisks

import ganeti.masterd.instance

_DISK_TEMPLATE_NAME_PREFIX = {

  constants.DT_PLAIN: "",

  constants.DT_RBD: ".rbd",

  constants.DT_EXT: ".ext",

  }

_DISK_TEMPLATE_DEVICE_TYPE = {

  constants.DT_PLAIN: constants.LD_LV,

  constants.DT_FILE: constants.LD_FILE,

  constants.DT_SHARED_FILE: constants.LD_FILE,

  constants.DT_BLOCK: constants.LD_BLOCKDEV,

  constants.DT_RBD: constants.LD_RBD,

  constants.DT_EXT: constants.LD_EXT,

  }

#: Type description for changes as returned by L{ApplyContainerMods}'s

#: callbacks

_TApplyContModsCbChanges = \

  ht.TMaybeListOf(ht.TAnd(ht.TIsLength(2), ht.TItems([

    ht.TNonEmptyString,

    ht.TAny,

    ])))

def _CopyLockList(names):

  """Makes a copy of a list of lock names.

  Handles L{locking.ALL_SET} correctly.

  """

  if names == locking.ALL_SET:

    return locking.ALL_SET

  else:

    return names[:]

def _ReleaseLocks(lu, level, names=None, keep=None):

  """Releases locks owned by an LU.

  @type lu: L{LogicalUnit}

  @param level: Lock level

  @type names: list or None

  @param names: Names of locks to release

  @type keep: list or None

  @param keep: Names of locks to retain

  """

  assert not (keep is not None and names is not None), \

    "Only one of the 'names' and the 'keep' parameters can be given"

  if names is not None:

    should_release = names.__contains__

  elif keep:

    should_release = lambda name: name not in keep

  else:

    should_release = None

  owned = lu.owned_locks(level)

  if not owned:

    # Not owning any lock at this level, do nothing

    pass

  elif should_release:

    retain = []

    release = []

    # Determine which locks to release

    for name in owned:

      if should_release(name):

        release.append(name)

      else:

        retain.append(name)

    assert len(lu.owned_locks(level)) == (len(retain) + len(release))

    # Release just some locks

    lu.glm.release(level, names=release)

    assert frozenset(lu.owned_locks(level)) == frozenset(retain)

  else:

    # Release everything

    lu.glm.release(level)

    assert not lu.glm.is_owned(level), "No locks should be owned"

def _CheckHostnameSane(lu, name):

  """Ensures that a given hostname resolves to a 'sane' name.

  The given name is required to be a prefix of the resolved hostname,

  to prevent accidental mismatches.

  @param lu: the logical unit on behalf of which we're checking

  @param name: the name we should resolve and check

  @return: the resolved hostname object

  """

  hostname = netutils.GetHostname(name=name)

  if hostname.name != name:

    lu.LogInfo("Resolved given name '%s' to '%s'", name, hostname.name)

  if not utils.MatchNameComponent(name, [hostname.name]):

    raise errors.OpPrereqError(("Resolved hostname '%s' does not look the"

                                " same as given hostname '%s'") %

                               (hostname.name, name), errors.ECODE_INVAL)

  return hostname

def _CheckOpportunisticLocking(op):

  """Generate error if opportunistic locking is not possible.

  """

  if op.opportunistic_locking and not op.iallocator:

    raise errors.OpPrereqError("Opportunistic locking is only available in"

                               " combination with an instance allocator",

                               errors.ECODE_INVAL)

def _CreateInstanceAllocRequest(op, disks, nics, beparams, node_whitelist):

  """Wrapper around IAReqInstanceAlloc.

  @param op: The instance opcode

  @param disks: The computed disks

  @param nics: The computed nics

  @param beparams: The full filled beparams

  @param node_whitelist: List of nodes which should appear as online to the

    allocator (unless the node is already marked offline)

  @returns: A filled L{iallocator.IAReqInstanceAlloc}

  """

  spindle_use = beparams[constants.BE_SPINDLE_USE]

  return iallocator.IAReqInstanceAlloc(name=op.instance_name,

                                       disk_template=op.disk_template,

                                       tags=op.tags,

                                       os=op.os_type,

                                       vcpus=beparams[constants.BE_VCPUS],

                                       memory=beparams[constants.BE_MAXMEM],

                                       spindle_use=spindle_use,

                                       disks=disks,

                                       nics=[n.ToDict() for n in nics],

                                       hypervisor=op.hypervisor,

                                       node_whitelist=node_whitelist)

def _ComputeFullBeParams(op, cluster):

  """Computes the full beparams.

  @param op: The instance opcode

  @param cluster: The cluster config object

  @return: The fully filled beparams

  """

  default_beparams = cluster.beparams[constants.PP_DEFAULT]

  for param, value in op.beparams.iteritems():

    if value == constants.VALUE_AUTO:

      op.beparams[param] = default_beparams[param]

  objects.UpgradeBeParams(op.beparams)

  utils.ForceDictType(op.beparams, constants.BES_PARAMETER_TYPES)

  return cluster.SimpleFillBE(op.beparams)

def _ComputeNics(op, cluster, default_ip, cfg, ec_id):

  """Computes the nics.

  @param op: The instance opcode

  @param cluster: Cluster configuration object

  @param default_ip: The default ip to assign

  @param cfg: An instance of the configuration object

  @param ec_id: Execution context ID

  @returns: The build up nics

  """

  nics = []

  for nic in op.nics:

    nic_mode_req = nic.get(constants.INIC_MODE, None)

    nic_mode = nic_mode_req

    if nic_mode is None or nic_mode == constants.VALUE_AUTO:

      nic_mode = cluster.nicparams[constants.PP_DEFAULT][constants.NIC_MODE]

    net = nic.get(constants.INIC_NETWORK, None)

    link = nic.get(constants.NIC_LINK, None)

    ip = nic.get(constants.INIC_IP, None)

    if net is None or net.lower() == constants.VALUE_NONE:

      net = None

    else:

      if nic_mode_req is not None or link is not None:

        raise errors.OpPrereqError("If network is given, no mode or link"

                                   " is allowed to be passed",

                                   errors.ECODE_INVAL)

    # ip validity checks

    if ip is None or ip.lower() == constants.VALUE_NONE:

      nic_ip = None

    elif ip.lower() == constants.VALUE_AUTO:

      if not op.name_check:

        raise errors.OpPrereqError("IP address set to auto but name checks"

                                   " have been skipped",

                                   errors.ECODE_INVAL)

      nic_ip = default_ip

    else:

      # We defer pool operations until later, so that the iallocator has

      # filled in the instance's node(s) dimara

      if ip.lower() == constants.NIC_IP_POOL:

        if net is None:

          raise errors.OpPrereqError("if ip=pool, parameter network"

                                     " must be passed too",

                                     errors.ECODE_INVAL)

      elif not netutils.IPAddress.IsValid(ip):

        raise errors.OpPrereqError("Invalid IP address '%s'" % ip,

                                   errors.ECODE_INVAL)

      nic_ip = ip

    # TODO: check the ip address for uniqueness

    if nic_mode == constants.NIC_MODE_ROUTED and not nic_ip:

      raise errors.OpPrereqError("Routed nic mode requires an ip address",

                                 errors.ECODE_INVAL)

    # MAC address verification

    mac = nic.get(constants.INIC_MAC, constants.VALUE_AUTO)

    if mac not in (constants.VALUE_AUTO, constants.VALUE_GENERATE):

      mac = utils.NormalizeAndValidateMac(mac)

      try:

        # TODO: We need to factor this out

        cfg.ReserveMAC(mac, ec_id)

      except errors.ReservationError:

        raise errors.OpPrereqError("MAC address %s already in use"

                                   " in cluster" % mac,

                                   errors.ECODE_NOTUNIQUE)

    #  Build nic parameters

    nicparams = {}

    if nic_mode_req:

      nicparams[constants.NIC_MODE] = nic_mode

    if link:

      nicparams[constants.NIC_LINK] = link

    check_params = cluster.SimpleFillNIC(nicparams)

    objects.NIC.CheckParameterSyntax(check_params)

    net_uuid = cfg.LookupNetwork(net)

    name = nic.get(constants.INIC_NAME, None)

    if name is not None and name.lower() == constants.VALUE_NONE:

      name = None

    nic_obj = objects.NIC(mac=mac, ip=nic_ip, name=name,

                          network=net_uuid, nicparams=nicparams)

    nic_obj.uuid = cfg.GenerateUniqueID(ec_id)

    nics.append(nic_obj)

  return nics

def _CheckForConflictingIp(lu, ip, node):

  """In case of conflicting IP address raise error.

  @type ip: string

  @param ip: IP address

  @type node: string

  @param node: node name

  """

  (conf_net, _) = lu.cfg.CheckIPInNodeGroup(ip, node)

  if conf_net is not None:

    raise errors.OpPrereqError(("The requested IP address (%s) belongs to"

                                " network %s, but the target NIC does not." %

                                (ip, conf_net)),

                               errors.ECODE_STATE)

  return (None, None)

def _CheckRADOSFreeSpace():

  """Compute disk size requirements inside the RADOS cluster.

  """

  # For the RADOS cluster we assume there is always enough space.

  pass

def _WaitForSync(lu, instance, disks=None, oneshot=False):

  """Sleep and poll for an instance's disk to sync.

  """

  if not instance.disks or disks is not None and not disks:

    return True

  disks = _ExpandCheckDisks(instance, disks)

  if not oneshot:

    lu.LogInfo("Waiting for instance %s to sync disks", instance.name)

  node = instance.primary_node

  for dev in disks:

    lu.cfg.SetDiskID(dev, node)

  # TODO: Convert to utils.Retry

  retries = 0

  degr_retries = 10 # in seconds, as we sleep 1 second each time

  while True:

    max_time = 0

    done = True

    cumul_degraded = False

    rstats = lu.rpc.call_blockdev_getmirrorstatus(node, (disks, instance))

    msg = rstats.fail_msg

    if msg:

      lu.LogWarning("Can't get any data from node %s: %s", node, msg)

      retries += 1

      if retries >= 10:

        raise errors.RemoteError("Can't contact node %s for mirror data,"

                                 " aborting." % node)

      time.sleep(6)

      continue

    rstats = rstats.payload

    retries = 0

    for i, mstat in enumerate(rstats):

      if mstat is None:

        lu.LogWarning("Can't compute data for node %s/%s",

                      node, disks[i].iv_name)

        continue

      cumul_degraded = (cumul_degraded or

                        (mstat.is_degraded and mstat.sync_percent is None))

      if mstat.sync_percent is not None:

        done = False

        if mstat.estimated_time is not None:

          rem_time = ("%s remaining (estimated)" %

                      utils.FormatSeconds(mstat.estimated_time))

          max_time = mstat.estimated_time

        else:

          rem_time = "no time estimate"

        lu.LogInfo("- device %s: %5.2f%% done, %s",

                   disks[i].iv_name, mstat.sync_percent, rem_time)

    # if we're done but degraded, let's do a few small retries, to

    # make sure we see a stable and not transient situation; therefore

    # we force restart of the loop

    if (done or oneshot) and cumul_degraded and degr_retries > 0:

      logging.info("Degraded disks found, %d retries left", degr_retries)

      degr_retries -= 1

      time.sleep(1)

      continue

    if done or oneshot:

      break

    time.sleep(min(60, max_time))

  if done:

    lu.LogInfo("Instance %s's disks are in sync", instance.name)

  return not cumul_degraded

def _ComputeDisks(op, default_vg):

  """Computes the instance disks.

  @param op: The instance opcode

  @param default_vg: The default_vg to assume

  @return: The computed disks

  """

  disks = []

  for disk in op.disks:

    mode = disk.get(constants.IDISK_MODE, constants.DISK_RDWR)

    if mode not in constants.DISK_ACCESS_SET:

      raise errors.OpPrereqError("Invalid disk access mode '%s'" %

                                 mode, errors.ECODE_INVAL)

    size = disk.get(constants.IDISK_SIZE, None)

    if size is None:

      raise errors.OpPrereqError("Missing disk size", errors.ECODE_INVAL)

    try:

      size = int(size)

    except (TypeError, ValueError):

      raise errors.OpPrereqError("Invalid disk size '%s'" % size,

                                 errors.ECODE_INVAL)

    ext_provider = disk.get(constants.IDISK_PROVIDER, None)

    if ext_provider and op.disk_template != constants.DT_EXT:

      raise errors.OpPrereqError("The '%s' option is only valid for the %s"

                                 " disk template, not %s" %

                                 (constants.IDISK_PROVIDER, constants.DT_EXT,

                                  op.disk_template), errors.ECODE_INVAL)

    data_vg = disk.get(constants.IDISK_VG, default_vg)

    name = disk.get(constants.IDISK_NAME, None)

    if name is not None and name.lower() == constants.VALUE_NONE:

      name = None

    new_disk = {

      constants.IDISK_SIZE: size,

      constants.IDISK_MODE: mode,

      constants.IDISK_VG: data_vg,

      constants.IDISK_NAME: name,

      }

    if constants.IDISK_METAVG in disk:

      new_disk[constants.IDISK_METAVG] = disk[constants.IDISK_METAVG]

    if constants.IDISK_ADOPT in disk:

      new_disk[constants.IDISK_ADOPT] = disk[constants.IDISK_ADOPT]

    # For extstorage, demand the `provider' option and add any

    # additional parameters (ext-params) to the dict

    if op.disk_template == constants.DT_EXT:

      if ext_provider:

        new_disk[constants.IDISK_PROVIDER] = ext_provider

        for key in disk:

          if key not in constants.IDISK_PARAMS:

            new_disk[key] = disk[key]

      else:

        raise errors.OpPrereqError("Missing provider for template '%s'" %

                                   constants.DT_EXT, errors.ECODE_INVAL)

    disks.append(new_disk)

  return disks

def _ComputeDiskSizePerVG(disk_template, disks):

  """Compute disk size requirements in the volume group

  """

  def _compute(disks, payload):

    """Universal algorithm.

    """

    vgs = {}

    for disk in disks:

      vgs[disk[constants.IDISK_VG]] = \

        vgs.get(constants.IDISK_VG, 0) + disk[constants.IDISK_SIZE] + payload

    return vgs

  # Required free disk space as a function of disk and swap space

  req_size_dict = {

    constants.DT_DISKLESS: {},

    constants.DT_PLAIN: _compute(disks, 0),

    # 128 MB are added for drbd metadata for each disk

    constants.DT_DRBD8: _compute(disks, constants.DRBD_META_SIZE),

    constants.DT_FILE: {},

    constants.DT_SHARED_FILE: {},

    }

  if disk_template not in req_size_dict:

    raise errors.ProgrammerError("Disk template '%s' size requirement"

                                 " is unknown" % disk_template)

  return req_size_dict[disk_template]

def _CheckNodesFreeDiskOnVG(lu, nodenames, vg, requested):

  """Checks if nodes have enough free disk space in the specified VG.

  This function checks if all given nodes have the needed amount of

  free disk. In case any node has less disk or we cannot get the

  information from the node, this function raises an OpPrereqError

  exception.

  @type lu: C{LogicalUnit}

  @param lu: a logical unit from which we get configuration data

  @type nodenames: C{list}

  @param nodenames: the list of node names to check

  @type vg: C{str}

  @param vg: the volume group to check

  @type requested: C{int}

  @param requested: the amount of disk in MiB to check for

  @raise errors.OpPrereqError: if the node doesn't have enough disk,

      or we cannot check the node

  """

  es_flags = rpc.GetExclusiveStorageForNodeNames(lu.cfg, nodenames)

  nodeinfo = lu.rpc.call_node_info(nodenames, [vg], None, es_flags)

  for node in nodenames:

    info = nodeinfo[node]

    info.Raise("Cannot get current information from node %s" % node,

               prereq=True, ecode=errors.ECODE_ENVIRON)

    (_, (vg_info, ), _) = info.payload

    vg_free = vg_info.get("vg_free", None)

    if not isinstance(vg_free, int):

      raise errors.OpPrereqError("Can't compute free disk space on node"

                                 " %s for vg %s, result was '%s'" %

                                 (node, vg, vg_free), errors.ECODE_ENVIRON)

    if requested > vg_free:

      raise errors.OpPrereqError("Not enough disk space on target node %s"

                                 " vg %s: required %d MiB, available %d MiB" %

                                 (node, vg, requested, vg_free),

                                 errors.ECODE_NORES)

def _CheckNodesFreeDiskPerVG(lu, nodenames, req_sizes):

  """Checks if nodes have enough free disk space in all the VGs.

  This function checks if all given nodes have the needed amount of

  free disk. In case any node has less disk or we cannot get the

  information from the node, this function raises an OpPrereqError

  exception.

  @type lu: C{LogicalUnit}

  @param lu: a logical unit from which we get configuration data

  @type nodenames: C{list}

  @param nodenames: the list of node names to check

  @type req_sizes: C{dict}

  @param req_sizes: the hash of vg and corresponding amount of disk in

      MiB to check for

  @raise errors.OpPrereqError: if the node doesn't have enough disk,

      or we cannot check the node

  """

  for vg, req_size in req_sizes.items():

    _CheckNodesFreeDiskOnVG(lu, nodenames, vg, req_size)

def _CheckNodeVmCapable(lu, node):

  """Ensure that a given node is vm capable.

  @param lu: the LU on behalf of which we make the check

  @param node: the node to check

  @raise errors.OpPrereqError: if the node is not vm capable

  """

  if not lu.cfg.GetNodeInfo(node).vm_capable:

    raise errors.OpPrereqError("Can't use non-vm_capable node %s" % node,

                               errors.ECODE_STATE)

def _ComputeIPolicyInstanceSpecViolation(

  ipolicy, instance_spec, disk_template,

  _compute_fn=_ComputeIPolicySpecViolation):

  """Compute if instance specs meets the specs of ipolicy.

  @type ipolicy: dict

  @param ipolicy: The ipolicy to verify against

  @param instance_spec: dict

  @param instance_spec: The instance spec to verify

  @type disk_template: string

  @param disk_template: the disk template of the instance

  @param _compute_fn: The function to verify ipolicy (unittest only)

  @see: L{_ComputeIPolicySpecViolation}

  """

  mem_size = instance_spec.get(constants.ISPEC_MEM_SIZE, None)

  cpu_count = instance_spec.get(constants.ISPEC_CPU_COUNT, None)

  disk_count = instance_spec.get(constants.ISPEC_DISK_COUNT, 0)

  disk_sizes = instance_spec.get(constants.ISPEC_DISK_SIZE, [])

  nic_count = instance_spec.get(constants.ISPEC_NIC_COUNT, 0)

  spindle_use = instance_spec.get(constants.ISPEC_SPINDLE_USE, None)

  return _compute_fn(ipolicy, mem_size, cpu_count, disk_count, nic_count,

                     disk_sizes, spindle_use, disk_template)

def _CheckOSVariant(os_obj, name):

  """Check whether an OS name conforms to the os variants specification.

  @type os_obj: L{objects.OS}

  @param os_obj: OS object to check

  @type name: string

  @param name: OS name passed by the user, to check for validity

  """

  variant = objects.OS.GetVariant(name)

  if not os_obj.supported_variants:

    if variant:

      raise errors.OpPrereqError("OS '%s' doesn't support variants ('%s'"

                                 " passed)" % (os_obj.name, variant),

                                 errors.ECODE_INVAL)

    return

  if not variant:

    raise errors.OpPrereqError("OS name must include a variant",

                               errors.ECODE_INVAL)

  if variant not in os_obj.supported_variants:

    raise errors.OpPrereqError("Unsupported OS variant", errors.ECODE_INVAL)

def _CheckNodeHasOS(lu, node, os_name, force_variant):

  """Ensure that a node supports a given OS.

  @param lu: the LU on behalf of which we make the check

  @param node: the node to check

  @param os_name: the OS to query about

  @param force_variant: whether to ignore variant errors

  @raise errors.OpPrereqError: if the node is not supporting the OS

  """

  result = lu.rpc.call_os_get(node, os_name)

  result.Raise("OS '%s' not in supported OS list for node %s" %

               (os_name, node),

               prereq=True, ecode=errors.ECODE_INVAL)

  if not force_variant:

    _CheckOSVariant(result.payload, os_name)

def _CheckNicsBridgesExist(lu, target_nics, target_node):

  """Check that the brigdes needed by a list of nics exist.

  """

  cluster = lu.cfg.GetClusterInfo()

  paramslist = [cluster.SimpleFillNIC(nic.nicparams) for nic in target_nics]

  brlist = [params[constants.NIC_LINK] for params in paramslist

            if params[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED]

  if brlist:

    result = lu.rpc.call_bridges_exist(target_node, brlist)

    result.Raise("Error checking bridges on destination node '%s'" %

                 target_node, prereq=True, ecode=errors.ECODE_ENVIRON)

def _CheckNodeFreeMemory(lu, node, reason, requested, hypervisor_name):

  """Checks if a node has enough free memory.

  This function checks if a given node has the needed amount of free

  memory. In case the node has less memory or we cannot get the

  information from the node, this function raises an OpPrereqError

  exception.

  @type lu: C{LogicalUnit}

  @param lu: a logical unit from which we get configuration data

  @type node: C{str}

  @param node: the node to check

  @type reason: C{str}

  @param reason: string to use in the error message

  @type requested: C{int}

  @param requested: the amount of memory in MiB to check for

  @type hypervisor_name: C{str}

  @param hypervisor_name: the hypervisor to ask for memory stats

  @rtype: integer

  @return: node current free memory

  @raise errors.OpPrereqError: if the node doesn't have enough memory, or

      we cannot check the node

  """

  nodeinfo = lu.rpc.call_node_info([node], None, [hypervisor_name], False)

  nodeinfo[node].Raise("Can't get data from node %s" % node,

                       prereq=True, ecode=errors.ECODE_ENVIRON)

  (_, _, (hv_info, )) = nodeinfo[node].payload

  free_mem = hv_info.get("memory_free", None)

  if not isinstance(free_mem, int):

    raise errors.OpPrereqError("Can't compute free memory on node %s, result"

                               " was '%s'" % (node, free_mem),

                               errors.ECODE_ENVIRON)

  if requested > free_mem:

    raise errors.OpPrereqError("Not enough memory on node %s for %s:"

                               " needed %s MiB, available %s MiB" %

                               (node, reason, requested, free_mem),

                               errors.ECODE_NORES)

  return free_mem

def _GenerateUniqueNames(lu, exts):

  """Generate a suitable LV name.

  This will generate a logical volume name for the given instance.

  """

  results = []

  for val in exts:

    new_id = lu.cfg.GenerateUniqueID(lu.proc.GetECId())

    results.append("%s%s" % (new_id, val))

  return results

def _GenerateDRBD8Branch(lu, primary, secondary, size, vgnames, names,

                         iv_name, p_minor, s_minor):

  """Generate a drbd8 device complete with its children.

  """

  assert len(vgnames) == len(names) == 2

  port = lu.cfg.AllocatePort()

  shared_secret = lu.cfg.GenerateDRBDSecret(lu.proc.GetECId())

  dev_data = objects.Disk(dev_type=constants.LD_LV, size=size,

                          logical_id=(vgnames[0], names[0]),

                          params={})

  dev_data.uuid = lu.cfg.GenerateUniqueID(lu.proc.GetECId())

  dev_meta = objects.Disk(dev_type=constants.LD_LV,

                          size=constants.DRBD_META_SIZE,

                          logical_id=(vgnames[1], names[1]),

                          params={})

  dev_meta.uuid = lu.cfg.GenerateUniqueID(lu.proc.GetECId())

  drbd_dev = objects.Disk(dev_type=constants.LD_DRBD8, size=size,

                          logical_id=(primary, secondary, port,

                                      p_minor, s_minor,

                                      shared_secret),

                          children=[dev_data, dev_meta],

                          iv_name=iv_name, params={})

  drbd_dev.uuid = lu.cfg.GenerateUniqueID(lu.proc.GetECId())

  return drbd_dev

def _GenerateDiskTemplate(

  lu, template_name, instance_name, primary_node, secondary_nodes,

  disk_info, file_storage_dir, file_driver, base_index,

  feedback_fn, full_disk_params, _req_file_storage=opcodes.RequireFileStorage,

  _req_shr_file_storage=opcodes.RequireSharedFileStorage):

  """Generate the entire disk layout for a given template type.

  """

  vgname = lu.cfg.GetVGName()

  disk_count = len(disk_info)

  disks = []

  if template_name == constants.DT_DISKLESS:

    pass

  elif template_name == constants.DT_DRBD8:

    if len(secondary_nodes) != 1:

      raise errors.ProgrammerError("Wrong template configuration")

    remote_node = secondary_nodes[0]

    minors = lu.cfg.AllocateDRBDMinor(

      [primary_node, remote_node] * len(disk_info), instance_name)

    (drbd_params, _, _) = objects.Disk.ComputeLDParams(template_name,

                                                       full_disk_params)

    drbd_default_metavg = drbd_params[constants.LDP_DEFAULT_METAVG]

    names = []

    for lv_prefix in _GenerateUniqueNames(lu, [".disk%d" % (base_index + i)

                                               for i in range(disk_count)]):

      names.append(lv_prefix + "_data")

      names.append(lv_prefix + "_meta")

    for idx, disk in enumerate(disk_info):

      disk_index = idx + base_index

      data_vg = disk.get(constants.IDISK_VG, vgname)

      meta_vg = disk.get(constants.IDISK_METAVG, drbd_default_metavg)

      disk_dev = _GenerateDRBD8Branch(lu, primary_node, remote_node,

                                      disk[constants.IDISK_SIZE],

                                      [data_vg, meta_vg],

                                      names[idx * 2:idx * 2 + 2],

                                      "disk/%d" % disk_index,

                                      minors[idx * 2], minors[idx * 2 + 1])

      disk_dev.mode = disk[constants.IDISK_MODE]

      disk_dev.name = disk.get(constants.IDISK_NAME, None)

      disks.append(disk_dev)

  else:

    if secondary_nodes:

      raise errors.ProgrammerError("Wrong template configuration")

    if template_name == constants.DT_FILE:

      _req_file_storage()

    elif template_name == constants.DT_SHARED_FILE:

      _req_shr_file_storage()

    name_prefix = _DISK_TEMPLATE_NAME_PREFIX.get(template_name, None)

    if name_prefix is None:

      names = None

    else:

      names = _GenerateUniqueNames(lu, ["%s.disk%s" %

                                        (name_prefix, base_index + i)

                                        for i in range(disk_count)])

    if template_name == constants.DT_PLAIN:

      def logical_id_fn(idx, _, disk):

        vg = disk.get(constants.IDISK_VG, vgname)

        return (vg, names[idx])

    elif template_name in (constants.DT_FILE, constants.DT_SHARED_FILE):

      logical_id_fn = \

        lambda _, disk_index, disk: (file_driver,

                                     "%s/disk%d" % (file_storage_dir,

                                                    disk_index))

    elif template_name == constants.DT_BLOCK:

      logical_id_fn = \

        lambda idx, disk_index, disk: (constants.BLOCKDEV_DRIVER_MANUAL,

                                       disk[constants.IDISK_ADOPT])

    elif template_name == constants.DT_RBD:

      logical_id_fn = lambda idx, _, disk: ("rbd", names[idx])

    elif template_name == constants.DT_EXT:

      def logical_id_fn(idx, _, disk):

        provider = disk.get(constants.IDISK_PROVIDER, None)

        if provider is None:

          raise errors.ProgrammerError("Disk template is %s, but '%s' is"

                                       " not found", constants.DT_EXT,

                                       constants.IDISK_PROVIDER)

        return (provider, names[idx])

    else:

      raise errors.ProgrammerError("Unknown disk template '%s'" % template_name)

    dev_type = _DISK_TEMPLATE_DEVICE_TYPE[template_name]

    for idx, disk in enumerate(disk_info):

      params = {}

      # Only for the Ext template add disk_info to params

      if template_name == constants.DT_EXT:

        params[constants.IDISK_PROVIDER] = disk[constants.IDISK_PROVIDER]

        for key in disk:

          if key not in constants.IDISK_PARAMS:

            params[key] = disk[key]

      disk_index = idx + base_index

      size = disk[constants.IDISK_SIZE]

      feedback_fn("* disk %s, size %s" %

                  (disk_index, utils.FormatUnit(size, "h")))

      disk_dev = objects.Disk(dev_type=dev_type, size=size,

                              logical_id=logical_id_fn(idx, disk_index, disk),

                              iv_name="disk/%d" % disk_index,

                              mode=disk[constants.IDISK_MODE],

                              params=params)

      disk_dev.name = disk.get(constants.IDISK_NAME, None)

      disk_dev.uuid = lu.cfg.GenerateUniqueID(lu.proc.GetECId())

      disks.append(disk_dev)

  return disks

def _CreateSingleBlockDev(lu, node, instance, device, info, force_open,

                          excl_stor):

  """Create a single block device on a given node.

  This will not recurse over children of the device, so they must be

  created in advance.

  @param lu: the lu on whose behalf we execute

  @param node: the node on which to create the device

  @type instance: L{objects.Instance}

  @param instance: the instance which owns the device

  @type device: L{objects.Disk}

  @param device: the device to create

  @param info: the extra 'metadata' we should attach to the device

      (this will be represented as a LVM tag)

  @type force_open: boolean

  @param force_open: this parameter will be passes to the

      L{backend.BlockdevCreate} function where it specifies

      whether we run on primary or not, and it affects both

      the child assembly and the device own Open() execution

  @type excl_stor: boolean

  @param excl_stor: Whether exclusive_storage is active for the node

  """

  lu.cfg.SetDiskID(device, node)

  result = lu.rpc.call_blockdev_create(node, device, device.size,

                                       instance.name, force_open, info,

                                       excl_stor)

  result.Raise("Can't create block device %s on"

               " node %s for instance %s" % (device, node, instance.name))

  if device.physical_id is None:

    device.physical_id = result.payload

def _CreateBlockDevInner(lu, node, instance, device, force_create,

                         info, force_open, excl_stor):

  """Create a tree of block devices on a given node.

  If this device type has to be created on secondaries, create it and

  all its children.

  If not, just recurse to children keeping the same 'force' value.

  @attention: The device has to be annotated already.

  @param lu: the lu on whose behalf we execute

  @param node: the node on which to create the device

  @type instance: L{objects.Instance}

  @param instance: the instance which owns the device

  @type device: L{objects.Disk}

  @param device: the device to create

  @type force_create: boolean

  @param force_create: whether to force creation of this device; this

      will be change to True whenever we find a device which has

      CreateOnSecondary() attribute

  @param info: the extra 'metadata' we should attach to the device

      (this will be represented as a LVM tag)

  @type force_open: boolean

  @param force_open: this parameter will be passes to the

      L{backend.BlockdevCreate} function where it specifies

      whether we run on primary or not, and it affects both

      the child assembly and the device own Open() execution

  @type excl_stor: boolean

  @param excl_stor: Whether exclusive_storage is active for the node

  @return: list of created devices

  """

  created_devices = []

  try:

    if device.CreateOnSecondary():

      force_create = True

    if device.children:

      for child in device.children:

        devs = _CreateBlockDevInner(lu, node, instance, child, force_create,

                                    info, force_open, excl_stor)

        created_devices.extend(devs)

    if not force_create:

      return created_devices

    _CreateSingleBlockDev(lu, node, instance, device, info, force_open,

                          excl_stor)

    # The device has been completely created, so there is no point in keeping

    # its subdevices in the list. We just add the device itself instead.

    created_devices = [(node, device)]

    return created_devices

  except errors.DeviceCreationError, e:

    e.created_devices.extend(created_devices)

    raise e

  except errors.OpExecError, e:

    raise errors.DeviceCreationError(str(e), created_devices)

def _IsExclusiveStorageEnabledNodeName(cfg, nodename):

  """Whether exclusive_storage is in effect for the given node.

  @type cfg: L{config.ConfigWriter}

  @param cfg: The cluster configuration

  @type nodename: string

  @param nodename: The node

  @rtype: bool

  @return: The effective value of exclusive_storage

  @raise errors.OpPrereqError: if no node exists with the given name

  """

  ni = cfg.GetNodeInfo(nodename)

  if ni is None:

    raise errors.OpPrereqError("Invalid node name %s" % nodename,

                               errors.ECODE_NOENT)

  return _IsExclusiveStorageEnabledNode(cfg, ni)

def _CreateBlockDev(lu, node, instance, device, force_create, info,

                    force_open):

  """Wrapper around L{_CreateBlockDevInner}.

  This method annotates the root device first.

  """

  (disk,) = _AnnotateDiskParams(instance, [device], lu.cfg)

  excl_stor = _IsExclusiveStorageEnabledNodeName(lu.cfg, node)

  return _CreateBlockDevInner(lu, node, instance, disk, force_create, info,

                              force_open, excl_stor)

def _CreateDisks(lu, instance, to_skip=None, target_node=None):

  """Create all disks for an instance.

  This abstracts away some work from AddInstance.

  @type lu: L{LogicalUnit}

  @param lu: the logical unit on whose behalf we execute

  @type instance: L{objects.Instance}

  @param instance: the instance whose disks we should create

  @type to_skip: list

  @param to_skip: list of indices to skip

  @type target_node: string

  @param target_node: if passed, overrides the target node for creation

  @rtype: boolean

  @return: the success of the creation

  """

  info = _GetInstanceInfoText(instance)

  if target_node is None:

    pnode = instance.primary_node

    all_nodes = instance.all_nodes

  else:

    pnode = target_node

    all_nodes = [pnode]

  if instance.disk_template in constants.DTS_FILEBASED:

    file_storage_dir = os.path.dirname(instance.disks[0].logical_id[1])

    result = lu.rpc.call_file_storage_dir_create(pnode, file_storage_dir)

    result.Raise("Failed to create directory '%s' on"

                 " node %s" % (file_storage_dir, pnode))

  disks_created = []

  # Note: this needs to be kept in sync with adding of disks in

  # LUInstanceSetParams

  for idx, device in enumerate(instance.disks):

    if to_skip and idx in to_skip:

      continue

    logging.info("Creating disk %s for instance '%s'", idx, instance.name)

    #HARDCODE

    for node in all_nodes:

      f_create = node == pnode

      try:

        _CreateBlockDev(lu, node, instance, device, f_create, info, f_create)

        disks_created.append((node, device))

      except errors.OpExecError:

        logging.warning("Creating disk %s for instance '%s' failed",

                        idx, instance.name)

      except errors.DeviceCreationError, e:

        logging.warning("Creating disk %s for instance '%s' failed",

                        idx, instance.name)

        disks_created.extend(e.created_devices)

        for (node, disk) in disks_created:

          lu.cfg.SetDiskID(disk, node)

          result = lu.rpc.call_blockdev_remove(node, disk)

          if result.fail_msg:

            logging.warning("Failed to remove newly-created disk %s on node %s:"

                            " %s", device, node, result.fail_msg)

        raise errors.OpExecError(e.message)

def _CalcEta(time_taken, written, total_size):

  """Calculates the ETA based on size written and total size.

  @param time_taken: The time taken so far

  @param written: amount written so far

  @param total_size: The total size of data to be written

  @return: The remaining time in seconds

  """

  avg_time = time_taken / float(written)

  return (total_size - written) * avg_time

def _WipeDisks(lu, instance, disks=None):

  """Wipes instance disks.

  @type lu: L{LogicalUnit}

  @param lu: the logical unit on whose behalf we execute

  @type instance: L{objects.Instance}

  @param instance: the instance whose disks we should create

  @type disks: None or list of tuple of (number, L{objects.Disk}, number)

  @param disks: Disk details; tuple contains disk index, disk object and the

    start offset

  """

  node = instance.primary_node

  if disks is None:

    disks = [(idx, disk, 0)

             for (idx, disk) in enumerate(instance.disks)]

  for (_, device, _) in disks:

    lu.cfg.SetDiskID(device, node)

  logging.info("Pausing synchronization of disks of instance '%s'",

               instance.name)

  result = lu.rpc.call_blockdev_pause_resume_sync(node,

                                                  (map(compat.snd, disks),

                                                   instance),

                                                  True)

  result.Raise("Failed to pause disk synchronization on node '%s'" % node)

  for idx, success in enumerate(result.payload):

    if not success:

      logging.warn("Pausing synchronization of disk %s of instance '%s'"

                   " failed", idx, instance.name)