Synnefo » snf-ganeti » ganeti-local

  class NodeImage(object):

    """A class representing the logical and physical status of a node.

    @ivar volumes: a structure as returned from

        L{ganeti.utils.GetVolumeList} (runtime)

    @ivar instances: a list of running instances (runtime)

    @ivar pinst: list of configured primary instances (config)

    @ivar sinst: list of configured secondary instances (config)

    @ivar sbp: diction of {secondary-node: list of instances} of all peers

        of this node (config)

    @ivar mfree: free memory, as reported by hypervisor (runtime)

    @ivar dfree: free disk, as reported by the node (runtime)

    @ivar offline: the offline status (config)

    @type rpc_fail: boolean

    @ivar rpc_fail: whether the RPC verify call was successfull (overall,

        not whether the individual keys were correct) (runtime)

    @type lvm_fail: boolean

    @ivar lvm_fail: whether the RPC call didn't return valid LVM data

    @type hyp_fail: boolean

    @ivar hyp_fail: whether the RPC call didn't return the instance list

    @type ghost: boolean

    @ivar ghost: whether this is a known node or not (config)

    """

    def __init__(self, offline=False):

      self.volumes = {}

      self.instances = []

      self.pinst = []

      self.sinst = []

      self.sbp = {}

      self.mfree = 0

      self.dfree = 0

      self.offline = offline

      self.rpc_fail = False

      self.lvm_fail = False

      self.hyp_fail = False

      self.ghost = False

  def _VerifyNode(self, ninfo, nresult):

    @type ninfo: L{objects.Node}

    @param ninfo: the node to check

    @param nresult: the results from the node

    @rtype: boolean

    @return: whether overall this call was successful (and we can expect

         reasonable values in the respose)

    node = ninfo.name

    # main result, nresult should be a non-empty dict

    test = not nresult or not isinstance(nresult, dict)

      return False

    remote_version = nresult.get("version", None)

      return False

      return False

    hyp_result = nresult.get(constants.NV_HYPERVISOR, None)

    if isinstance(hyp_result, dict):

      for hv_name, hv_result in hyp_result.iteritems():

        test = hv_result is not None

        _ErrorIf(test, self.ENODEHV, node,

                 "hypervisor %s verify failure: '%s'", hv_name, hv_result)

    test = nresult.get(constants.NV_NODESETUP,

                           ["Missing NODESETUP results"])

    _ErrorIf(test, self.ENODESETUP, node, "node setup error: %s",

             "; ".join(test))

    return True

  def _VerifyNodeTime(self, ninfo, nresult,

                      nvinfo_starttime, nvinfo_endtime):

    """Check the node time.

    @type ninfo: L{objects.Node}

    @param ninfo: the node to check

    @param nresult: the remote results for the node

    @param nvinfo_starttime: the start time of the RPC call

    @param nvinfo_endtime: the end time of the RPC call

    """

    node = ninfo.name

    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103

    ntime = nresult.get(constants.NV_TIME, None)

    try:

      ntime_merged = utils.MergeTime(ntime)

    except (ValueError, TypeError):

      _ErrorIf(True, self.ENODETIME, node, "Node returned invalid time")

      return

    if ntime_merged < (nvinfo_starttime - constants.NODE_MAX_CLOCK_SKEW):

      ntime_diff = "%.01fs" % abs(nvinfo_starttime - ntime_merged)

    elif ntime_merged > (nvinfo_endtime + constants.NODE_MAX_CLOCK_SKEW):

      ntime_diff = "%.01fs" % abs(ntime_merged - nvinfo_endtime)

    else:

      ntime_diff = None

    _ErrorIf(ntime_diff is not None, self.ENODETIME, node,

             "Node time diverges by at least %s from master node time",

             ntime_diff)

  def _VerifyNodeLVM(self, ninfo, nresult, vg_name):

    """Check the node time.

    @type ninfo: L{objects.Node}

    @param ninfo: the node to check

    @param nresult: the remote results for the node

    @param vg_name: the configured VG name

    """

    if vg_name is None:

      return

    node = ninfo.name

    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103

    # checks vg existence and size > 20G

    vglist = nresult.get(constants.NV_VGLIST, None)

    test = not vglist

    _ErrorIf(test, self.ENODELVM, node, "unable to check volume groups")

      vgstatus = utils.CheckVolumeGroupSize(vglist, vg_name,

                                            constants.MIN_VG_SIZE)

      _ErrorIf(vgstatus, self.ENODELVM, node, vgstatus)

    # check pv names

    pvlist = nresult.get(constants.NV_PVLIST, None)

    test = pvlist is None

    _ErrorIf(test, self.ENODELVM, node, "Can't get PV list from node")

    if not test:

      # check that ':' is not present in PV names, since it's a

      # special character for lvcreate (denotes the range of PEs to

      # use on the PV)

      for _, pvname, owner_vg in pvlist:

        test = ":" in pvname

        _ErrorIf(test, self.ENODELVM, node, "Invalid character ':' in PV"

                 " '%s' of VG '%s'", pvname, owner_vg)

  def _VerifyNodeNetwork(self, ninfo, nresult):

    """Check the node time.

    @type ninfo: L{objects.Node}

    @param ninfo: the node to check

    @param nresult: the remote results for the node

    """

    node = ninfo.name

    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103

    test = constants.NV_NODELIST not in nresult

      if nresult[constants.NV_NODELIST]:

        for a_node, a_msg in nresult[constants.NV_NODELIST].items():

    test = constants.NV_NODENETTEST not in nresult

      if nresult[constants.NV_NODENETTEST]:

        nlist = utils.NiceSort(nresult[constants.NV_NODENETTEST].keys())

                   anode, nresult[constants.NV_NODENETTEST][anode])

  def _VerifyInstance(self, instance, instanceconfig, node_image):

      n_img = node_image[node]

      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:

        # ignore missing volumes on offline or broken nodes

        test = volume not in n_img.volumes

      pri_img = node_image[node_current]

      test = instance not in pri_img.instances and not pri_img.offline

    for node, n_img in node_image.items():

        test = instance in n_img.instances

  def _VerifyOrphanVolumes(self, node_vol_should, node_image):

    for node, n_img in node_image.items():

      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:

        # skip non-healthy nodes

        continue

      for volume in n_img.volumes:

  def _VerifyOrphanInstances(self, instancelist, node_image):

    for node, n_img in node_image.items():

      for o_inst in n_img.instances:

  def _VerifyNPlusOneMemory(self, node_image, instance_cfg):

    Check that if one single node dies we can still start all the

    instances it was primary for.

    for node, n_img in node_image.items():

      # This code checks that every node which is now listed as

      # secondary has enough memory to host all instances it is

      # supposed to should a single other node in the cluster fail.

      # WARNING: we currently take into account down instances as well

      # as up ones, considering that even if they're down someone

      # might want to start them even in the event of a node failure.

      for prinode, instances in n_img.sbp.items():

        test = n_img.mfree < needed_mem

  def _VerifyNodeFiles(self, ninfo, nresult, file_list, local_cksum,

                       master_files):

    """Verifies and computes the node required file checksums.

    @type ninfo: L{objects.Node}

    @param ninfo: the node to check

    @param nresult: the remote results for the node

    @param file_list: required list of files

    @param local_cksum: dictionary of local files and their checksums

    @param master_files: list of files that only masters should have

    """

    node = ninfo.name

    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103

    remote_cksum = nresult.get(constants.NV_FILELIST, None)

    test = not isinstance(remote_cksum, dict)

    _ErrorIf(test, self.ENODEFILECHECK, node,

             "node hasn't returned file checksum data")

    if test:

      return

    for file_name in file_list:

      node_is_mc = ninfo.master_candidate

      must_have = (file_name not in master_files) or node_is_mc

      # missing

      test1 = file_name not in remote_cksum

      # invalid checksum

      test2 = not test1 and remote_cksum[file_name] != local_cksum[file_name]

      # existing and good

      test3 = not test1 and remote_cksum[file_name] == local_cksum[file_name]

      _ErrorIf(test1 and must_have, self.ENODEFILECHECK, node,

               "file '%s' missing", file_name)

      _ErrorIf(test2 and must_have, self.ENODEFILECHECK, node,

               "file '%s' has wrong checksum", file_name)

      # not candidate and this is not a must-have file

      _ErrorIf(test2 and not must_have, self.ENODEFILECHECK, node,

               "file '%s' should not exist on non master"

               " candidates (and the file is outdated)", file_name)

      # all good, except non-master/non-must have combination

      _ErrorIf(test3 and not must_have, self.ENODEFILECHECK, node,

               "file '%s' should not exist"

               " on non master candidates", file_name)

  def _VerifyNodeDrbd(self, ninfo, nresult, instanceinfo, drbd_map):

    """Verifies and the node DRBD status.

    @type ninfo: L{objects.Node}

    @param ninfo: the node to check

    @param nresult: the remote results for the node

    @param instanceinfo: the dict of instances

    @param drbd_map: the DRBD map as returned by

        L{ganeti.config.ConfigWriter.ComputeDRBDMap}

    """

    node = ninfo.name

    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103

    # compute the DRBD minors

    node_drbd = {}

    for minor, instance in drbd_map[node].items():

      test = instance not in instanceinfo

      _ErrorIf(test, self.ECLUSTERCFG, None,

               "ghost instance '%s' in temporary DRBD map", instance)

        # ghost instance should not be running, but otherwise we

        # don't give double warnings (both ghost instance and

        # unallocated minor in use)

      if test:

        node_drbd[minor] = (instance, False)

      else:

        instance = instanceinfo[instance]

        node_drbd[minor] = (instance.name, instance.admin_up)

    # and now check them

    used_minors = nresult.get(constants.NV_DRBDLIST, [])

    test = not isinstance(used_minors, (tuple, list))

    _ErrorIf(test, self.ENODEDRBD, node,

             "cannot parse drbd status file: %s", str(used_minors))

    if test:

      # we cannot check drbd status

      return

    for minor, (iname, must_exist) in node_drbd.items():

      test = minor not in used_minors and must_exist

      _ErrorIf(test, self.ENODEDRBD, node,

               "drbd minor %d of instance %s is not active", minor, iname)

    for minor in used_minors:

      test = minor not in node_drbd

      _ErrorIf(test, self.ENODEDRBD, node,

               "unallocated drbd minor %d is in use", minor)

  def _UpdateNodeVolumes(self, ninfo, nresult, nimg, vg_name):

    """Verifies and updates the node volume data.

    This function will update a L{NodeImage}'s internal structures

    with data from the remote call.

    @type ninfo: L{objects.Node}

    @param ninfo: the node to check

    @param nresult: the remote results for the node

    @param nimg: the node image object

    @param vg_name: the configured VG name

    """

    node = ninfo.name

    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103

    nimg.lvm_fail = True

    lvdata = nresult.get(constants.NV_LVLIST, "Missing LV data")

    if vg_name is None:

      pass

    elif isinstance(lvdata, basestring):

      _ErrorIf(True, self.ENODELVM, node, "LVM problem on node: %s",

               utils.SafeEncode(lvdata))

    elif not isinstance(lvdata, dict):

      _ErrorIf(True, self.ENODELVM, node, "rpc call to node failed (lvlist)")

    else:

      nimg.volumes = lvdata

      nimg.lvm_fail = False

  def _UpdateNodeInstances(self, ninfo, nresult, nimg):

    """Verifies and updates the node instance list.

    If the listing was successful, then updates this node's instance

    list. Otherwise, it marks the RPC call as failed for the instance

    list key.

    @type ninfo: L{objects.Node}

    @param ninfo: the node to check

    @param nresult: the remote results for the node

    @param nimg: the node image object

    """

    idata = nresult.get(constants.NV_INSTANCELIST, None)

    test = not isinstance(idata, list)

    self._ErrorIf(test, self.ENODEHV, ninfo.name, "rpc call to node failed"

                  " (instancelist): %s", utils.SafeEncode(str(idata)))

    if test:

      nimg.hyp_fail = True

    else:

      nimg.instances = idata

  def _UpdateNodeInfo(self, ninfo, nresult, nimg, vg_name):

    """Verifies and computes a node information map

    @type ninfo: L{objects.Node}

    @param ninfo: the node to check

    @param nresult: the remote results for the node

    @param nimg: the node image object

    @param vg_name: the configured VG name

    """

    node = ninfo.name

    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103

    # try to read free memory (from the hypervisor)

    hv_info = nresult.get(constants.NV_HVINFO, None)

    test = not isinstance(hv_info, dict) or "memory_free" not in hv_info

    _ErrorIf(test, self.ENODEHV, node, "rpc call to node failed (hvinfo)")

    if not test:

      try:

        nimg.mfree = int(hv_info["memory_free"])

      except (ValueError, TypeError):

        _ErrorIf(True, self.ENODERPC, node,

                 "node returned invalid nodeinfo, check hypervisor")

    # FIXME: devise a free space model for file based instances as well

    if vg_name is not None:

      test = (constants.NV_VGLIST not in nresult or

              vg_name not in nresult[constants.NV_VGLIST])

      _ErrorIf(test, self.ENODELVM, node,

               "node didn't return data for the volume group '%s'"

               " - it is either missing or broken", vg_name)

      if not test:

        try:

          nimg.dfree = int(nresult[constants.NV_VGLIST][vg_name])

        except (ValueError, TypeError):

          _ErrorIf(True, self.ENODERPC, node,

                   "node returned invalid LVM info, check LVM status")

    n_offline = 0 # Count of offline nodes

    n_drained = 0 # Count of nodes being drained

    node_vol_should = {}

    # Build our expected cluster state

    node_image = dict((node.name, self.NodeImage(offline=node.offline))

                      for node in nodeinfo)

    for instance in instancelist:

      inst_config = instanceinfo[instance]

      for nname in inst_config.all_nodes:

        if nname not in node_image:

          # ghost node

          gnode = self.NodeImage()

          gnode.ghost = True

          node_image[nname] = gnode

      inst_config.MapLVsByNode(node_vol_should)

      pnode = inst_config.primary_node

      node_image[pnode].pinst.append(instance)

      for snode in inst_config.secondary_nodes:

        nimg = node_image[snode]

        nimg.sinst.append(instance)

        if pnode not in nimg.sbp:

          nimg.sbp[pnode] = []

        nimg.sbp[pnode].append(instance)

    # At this point, we have the in-memory data structures complete,

    # except for the runtime information, which we'll gather next

      nimg = node_image[node]

        n_offline += 1

        n_drained += 1

        nimg.rpc_fail = True

      nimg.call_ok = self._VerifyNode(node_i, nresult)

      self._VerifyNodeNetwork(node_i, nresult)

      self._VerifyNodeLVM(node_i, nresult, vg_name)

      self._VerifyNodeFiles(node_i, nresult, file_names, local_checksums,

                            master_files)

      self._VerifyNodeDrbd(node_i, nresult, instanceinfo, all_drbd_map)

      self._VerifyNodeTime(node_i, nresult, nvinfo_starttime, nvinfo_endtime)

      self._UpdateNodeVolumes(node_i, nresult, nimg, vg_name)

      self._UpdateNodeInstances(node_i, nresult, nimg)

      self._UpdateNodeInfo(node_i, nresult, nimg, vg_name)

      self._VerifyInstance(instance, inst_config, node_image)

      pnode_img = node_image[pnode]

      _ErrorIf(pnode_img.rpc_fail and not pnode_img.offline,

      if pnode_img.offline:

      if not inst_config.secondary_nodes:

      _ErrorIf(len(inst_config.secondary_nodes) > 1, self.EINSTANCELAYOUT,

               instance, "instance has multiple secondary nodes: %s",

               utils.CommaJoin(inst_config.secondary_nodes),

               code=self.ETYPE_WARNING)

        s_img = node_image[snode]

        _ErrorIf(s_img.rpc_fail and not s_img.offline, self.ENODERPC, snode,

                 "instance %s, connection to secondary node failed", instance)

        if s_img.offline:

      # ... or ghost nodes

      for node in inst_config.all_nodes:

        _ErrorIf(node_image[node].ghost, self.EINSTANCEBADNODE, instance,

                 "instance lives on ghost node %s", node)

    self._VerifyOrphanVolumes(node_vol_should, node_image)

    feedback_fn("* Verifying oprhan instances")

    self._VerifyOrphanInstances(instancelist, node_image)

      self._VerifyNPlusOneMemory(node_image, instanceinfo)

      feedback_fn("  - NOTICE: %d offline node(s) found." % n_offline)

      feedback_fn("  - NOTICE: %d drained node(s) found." % n_drained)