Synnefo » snf-ganeti » ganeti-local

#

#

# Copyright (C) 2006, 2007, 2008 Google Inc.

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful, but

# WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

# General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

# 02110-1301, USA.

"""Module implementing the master-side code."""

# pylint: disable-msg=W0613,W0201

import os

import os.path

import sha

import time

import tempfile

import re

import platform

import logging

import copy

import random

from ganeti import ssh

from ganeti import utils

from ganeti import errors

from ganeti import hypervisor

from ganeti import locking

from ganeti import constants

from ganeti import objects

from ganeti import opcodes

from ganeti import serializer

from ganeti import ssconf

class LogicalUnit(object):

  """Logical Unit base class.

  Subclasses must follow these rules:

    - implement ExpandNames

    - implement CheckPrereq

    - implement Exec

    - implement BuildHooksEnv

    - redefine HPATH and HTYPE

    - optionally redefine their run requirements:

        REQ_BGL: the LU needs to hold the Big Ganeti Lock exclusively

  Note that all commands require root permissions.

  """

  HPATH = None

  HTYPE = None

  _OP_REQP = []

  REQ_BGL = True

  def __init__(self, processor, op, context, rpc):

    """Constructor for LogicalUnit.

    This needs to be overriden in derived classes in order to check op

    validity.

    """

    self.proc = processor

    self.op = op

    self.cfg = context.cfg

    self.context = context

    self.rpc = rpc

    # Dicts used to declare locking needs to mcpu

    self.needed_locks = None

    self.acquired_locks = {}

    self.share_locks = dict(((i, 0) for i in locking.LEVELS))

    self.add_locks = {}

    self.remove_locks = {}

    # Used to force good behavior when calling helper functions

    self.recalculate_locks = {}

    self.__ssh = None

    # logging

    self.LogWarning = processor.LogWarning

    self.LogInfo = processor.LogInfo

    for attr_name in self._OP_REQP:

      attr_val = getattr(op, attr_name, None)

      if attr_val is None:

        raise errors.OpPrereqError("Required parameter '%s' missing" %

                                   attr_name)

    self.CheckArguments()

  def __GetSSH(self):

    """Returns the SshRunner object

    """

    if not self.__ssh:

      self.__ssh = ssh.SshRunner(self.cfg.GetClusterName())

    return self.__ssh

  ssh = property(fget=__GetSSH)

  def CheckArguments(self):

    """Check syntactic validity for the opcode arguments.

    This method is for doing a simple syntactic check and ensure

    validity of opcode parameters, without any cluster-related

    checks. While the same can be accomplished in ExpandNames and/or

    CheckPrereq, doing these separate is better because:

      - ExpandNames is left as as purely a lock-related function

      - CheckPrereq is run after we have aquired locks (and possible

        waited for them)

    The function is allowed to change the self.op attribute so that

    later methods can no longer worry about missing parameters.

    """

    pass

  def ExpandNames(self):

    """Expand names for this LU.

    This method is called before starting to execute the opcode, and it should

    update all the parameters of the opcode to their canonical form (e.g. a

    short node name must be fully expanded after this method has successfully

    completed). This way locking, hooks, logging, ecc. can work correctly.

    LUs which implement this method must also populate the self.needed_locks

    member, as a dict with lock levels as keys, and a list of needed lock names

    as values. Rules:

      - use an empty dict if you don't need any lock

      - if you don't need any lock at a particular level omit that level

      - don't put anything for the BGL level

      - if you want all locks at a level use locking.ALL_SET as a value

    If you need to share locks (rather than acquire them exclusively) at one

    level you can modify self.share_locks, setting a true value (usually 1) for

    that level. By default locks are not shared.

    Examples::

      # Acquire all nodes and one instance

      self.needed_locks = {

        locking.LEVEL_NODE: locking.ALL_SET,

        locking.LEVEL_INSTANCE: ['instance1.example.tld'],

      }

      # Acquire just two nodes

      self.needed_locks = {

        locking.LEVEL_NODE: ['node1.example.tld', 'node2.example.tld'],

      }

      # Acquire no locks

      self.needed_locks = {} # No, you can't leave it to the default value None

    """

    # The implementation of this method is mandatory only if the new LU is

    # concurrent, so that old LUs don't need to be changed all at the same

    # time.

    if self.REQ_BGL:

      self.needed_locks = {} # Exclusive LUs don't need locks.

    else:

      raise NotImplementedError

  def DeclareLocks(self, level):

    """Declare LU locking needs for a level

    While most LUs can just declare their locking needs at ExpandNames time,

    sometimes there's the need to calculate some locks after having acquired

    the ones before. This function is called just before acquiring locks at a

    particular level, but after acquiring the ones at lower levels, and permits

    such calculations. It can be used to modify self.needed_locks, and by

    default it does nothing.

    This function is only called if you have something already set in

    self.needed_locks for the level.

    @param level: Locking level which is going to be locked

    @type level: member of ganeti.locking.LEVELS

    """

  def CheckPrereq(self):

    """Check prerequisites for this LU.

    This method should check that the prerequisites for the execution

    of this LU are fulfilled. It can do internode communication, but

    it should be idempotent - no cluster or system changes are

    allowed.

    The method should raise errors.OpPrereqError in case something is

    not fulfilled. Its return value is ignored.

    This method should also update all the parameters of the opcode to

    their canonical form if it hasn't been done by ExpandNames before.

    """

    raise NotImplementedError

  def Exec(self, feedback_fn):

    """Execute the LU.

    This method should implement the actual work. It should raise

    errors.OpExecError for failures that are somewhat dealt with in

    code, or expected.

    """

    raise NotImplementedError

  def BuildHooksEnv(self):

    """Build hooks environment for this LU.

    This method should return a three-node tuple consisting of: a dict

    containing the environment that will be used for running the

    specific hook for this LU, a list of node names on which the hook

    should run before the execution, and a list of node names on which

    the hook should run after the execution.

    The keys of the dict must not have 'GANETI_' prefixed as this will

    be handled in the hooks runner. Also note additional keys will be

    added by the hooks runner. If the LU doesn't define any

    environment, an empty dict (and not None) should be returned.

    No nodes should be returned as an empty list (and not None).

    Note that if the HPATH for a LU class is None, this function will

    not be called.

    """

    raise NotImplementedError

  def HooksCallBack(self, phase, hook_results, feedback_fn, lu_result):

    """Notify the LU about the results of its hooks.

    This method is called every time a hooks phase is executed, and notifies

    the Logical Unit about the hooks' result. The LU can then use it to alter

    its result based on the hooks.  By default the method does nothing and the

    previous result is passed back unchanged but any LU can define it if it

    wants to use the local cluster hook-scripts somehow.

    @param phase: one of L{constants.HOOKS_PHASE_POST} or

        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase

    @param hook_results: the results of the multi-node hooks rpc call

    @param feedback_fn: function used send feedback back to the caller

    @param lu_result: the previous Exec result this LU had, or None

        in the PRE phase

    @return: the new Exec result, based on the previous result

        and hook results

    """

    return lu_result

  def _ExpandAndLockInstance(self):

    """Helper function to expand and lock an instance.

    Many LUs that work on an instance take its name in self.op.instance_name

    and need to expand it and then declare the expanded name for locking. This

    function does it, and then updates self.op.instance_name to the expanded

    name. It also initializes needed_locks as a dict, if this hasn't been done

    before.

    """

    if self.needed_locks is None:

      self.needed_locks = {}

    else:

      assert locking.LEVEL_INSTANCE not in self.needed_locks, \

        "_ExpandAndLockInstance called with instance-level locks set"

    expanded_name = self.cfg.ExpandInstanceName(self.op.instance_name)

    if expanded_name is None:

      raise errors.OpPrereqError("Instance '%s' not known" %

                                  self.op.instance_name)

    self.needed_locks[locking.LEVEL_INSTANCE] = expanded_name

    self.op.instance_name = expanded_name

  def _LockInstancesNodes(self, primary_only=False):

    """Helper function to declare instances' nodes for locking.

    This function should be called after locking one or more instances to lock

    their nodes. Its effect is populating self.needed_locks[locking.LEVEL_NODE]

    with all primary or secondary nodes for instances already locked and

    present in self.needed_locks[locking.LEVEL_INSTANCE].

    It should be called from DeclareLocks, and for safety only works if

    self.recalculate_locks[locking.LEVEL_NODE] is set.

    In the future it may grow parameters to just lock some instance's nodes, or

    to just lock primaries or secondary nodes, if needed.

    If should be called in DeclareLocks in a way similar to::

      if level == locking.LEVEL_NODE:

        self._LockInstancesNodes()

    @type primary_only: boolean

    @param primary_only: only lock primary nodes of locked instances

    """

    assert locking.LEVEL_NODE in self.recalculate_locks, \

      "_LockInstancesNodes helper function called with no nodes to recalculate"

    # TODO: check if we're really been called with the instance locks held

    # For now we'll replace self.needed_locks[locking.LEVEL_NODE], but in the

    # future we might want to have different behaviors depending on the value

    # of self.recalculate_locks[locking.LEVEL_NODE]

    wanted_nodes = []

    for instance_name in self.acquired_locks[locking.LEVEL_INSTANCE]:

      instance = self.context.cfg.GetInstanceInfo(instance_name)

      wanted_nodes.append(instance.primary_node)

      if not primary_only:

        wanted_nodes.extend(instance.secondary_nodes)

    if self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_REPLACE:

      self.needed_locks[locking.LEVEL_NODE] = wanted_nodes

    elif self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_APPEND:

      self.needed_locks[locking.LEVEL_NODE].extend(wanted_nodes)

    del self.recalculate_locks[locking.LEVEL_NODE]

class NoHooksLU(LogicalUnit):

  """Simple LU which runs no hooks.

  This LU is intended as a parent for other LogicalUnits which will

  run no hooks, in order to reduce duplicate code.

  """

  HPATH = None

  HTYPE = None

def _GetWantedNodes(lu, nodes):

  """Returns list of checked and expanded node names.

  @type lu: L{LogicalUnit}

  @param lu: the logical unit on whose behalf we execute

  @type nodes: list

  @param nodes: list of node names or None for all nodes

  @rtype: list

  @return: the list of nodes, sorted

  @raise errors.OpProgrammerError: if the nodes parameter is wrong type

  """

  if not isinstance(nodes, list):

    raise errors.OpPrereqError("Invalid argument type 'nodes'")

  if not nodes:

    raise errors.ProgrammerError("_GetWantedNodes should only be called with a"

      " non-empty list of nodes whose name is to be expanded.")

  wanted = []

  for name in nodes:

    node = lu.cfg.ExpandNodeName(name)

    if node is None:

      raise errors.OpPrereqError("No such node name '%s'" % name)

    wanted.append(node)

  return utils.NiceSort(wanted)

def _GetWantedInstances(lu, instances):

  """Returns list of checked and expanded instance names.

  @type lu: L{LogicalUnit}

  @param lu: the logical unit on whose behalf we execute

  @type instances: list

  @param instances: list of instance names or None for all instances

  @rtype: list

  @return: the list of instances, sorted

  @raise errors.OpPrereqError: if the instances parameter is wrong type

  @raise errors.OpPrereqError: if any of the passed instances is not found

  """

  if not isinstance(instances, list):

    raise errors.OpPrereqError("Invalid argument type 'instances'")

  if instances:

    wanted = []

    for name in instances:

      instance = lu.cfg.ExpandInstanceName(name)

      if instance is None:

        raise errors.OpPrereqError("No such instance name '%s'" % name)

      wanted.append(instance)

  else:

    wanted = lu.cfg.GetInstanceList()

  return utils.NiceSort(wanted)

def _CheckOutputFields(static, dynamic, selected):

  """Checks whether all selected fields are valid.

  @type static: L{utils.FieldSet}

  @param static: static fields set

  @type dynamic: L{utils.FieldSet}

  @param dynamic: dynamic fields set

  """

  f = utils.FieldSet()

  f.Extend(static)

  f.Extend(dynamic)

  delta = f.NonMatching(selected)

  if delta:

    raise errors.OpPrereqError("Unknown output fields selected: %s"

                               % ",".join(delta))

def _CheckBooleanOpField(op, name):

  """Validates boolean opcode parameters.

  This will ensure that an opcode parameter is either a boolean value,

  or None (but that it always exists).

  """

  val = getattr(op, name, None)

  if not (val is None or isinstance(val, bool)):

    raise errors.OpPrereqError("Invalid boolean parameter '%s' (%s)" %

                               (name, str(val)))

  setattr(op, name, val)

def _CheckNodeOnline(lu, node):

  """Ensure that a given node is online.

  @param lu: the LU on behalf of which we make the check

  @param node: the node to check

  @raise errors.OpPrereqError: if the nodes is offline

  """

  if lu.cfg.GetNodeInfo(node).offline:

    raise errors.OpPrereqError("Can't use offline node %s" % node)

def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status,

                          memory, vcpus, nics):

  """Builds instance related env variables for hooks

  This builds the hook environment from individual variables.

  @type name: string

  @param name: the name of the instance

  @type primary_node: string

  @param primary_node: the name of the instance's primary node

  @type secondary_nodes: list

  @param secondary_nodes: list of secondary nodes as strings

  @type os_type: string

  @param os_type: the name of the instance's OS

  @type status: boolean

  @param status: the should_run status of the instance

  @type memory: string

  @param memory: the memory size of the instance

  @type vcpus: string

  @param vcpus: the count of VCPUs the instance has

  @type nics: list

  @param nics: list of tuples (ip, bridge, mac) representing

      the NICs the instance  has

  @rtype: dict

  @return: the hook environment for this instance

  """

  if status:

    str_status = "up"

  else:

    str_status = "down"

  env = {

    "OP_TARGET": name,

    "INSTANCE_NAME": name,

    "INSTANCE_PRIMARY": primary_node,

    "INSTANCE_SECONDARIES": " ".join(secondary_nodes),

    "INSTANCE_OS_TYPE": os_type,

    "INSTANCE_STATUS": str_status,

    "INSTANCE_MEMORY": memory,

    "INSTANCE_VCPUS": vcpus,

  }

  if nics:

    nic_count = len(nics)

    for idx, (ip, bridge, mac) in enumerate(nics):

      if ip is None:

        ip = ""

      env["INSTANCE_NIC%d_IP" % idx] = ip

      env["INSTANCE_NIC%d_BRIDGE" % idx] = bridge

      env["INSTANCE_NIC%d_HWADDR" % idx] = mac

  else:

    nic_count = 0

  env["INSTANCE_NIC_COUNT"] = nic_count

  return env

def _BuildInstanceHookEnvByObject(lu, instance, override=None):

  """Builds instance related env variables for hooks from an object.

  @type lu: L{LogicalUnit}

  @param lu: the logical unit on whose behalf we execute

  @type instance: L{objects.Instance}

  @param instance: the instance for which we should build the

      environment

  @type override: dict

  @param override: dictionary with key/values that will override

      our values

  @rtype: dict

  @return: the hook environment dictionary

  """

  bep = lu.cfg.GetClusterInfo().FillBE(instance)

  args = {

    'name': instance.name,

    'primary_node': instance.primary_node,

    'secondary_nodes': instance.secondary_nodes,

    'os_type': instance.os,

    'status': instance.admin_up,

    'memory': bep[constants.BE_MEMORY],

    'vcpus': bep[constants.BE_VCPUS],

    'nics': [(nic.ip, nic.bridge, nic.mac) for nic in instance.nics],

  }

  if override:

    args.update(override)

  return _BuildInstanceHookEnv(**args)

def _AdjustCandidatePool(lu):

  """Adjust the candidate pool after node operations.

  """

  mod_list = lu.cfg.MaintainCandidatePool()

  if mod_list:

    lu.LogInfo("Promoted nodes to master candidate role: %s",

               ", ".join(node.name for node in mod_list))

    for name in mod_list:

      lu.context.ReaddNode(name)

  mc_now, mc_max = lu.cfg.GetMasterCandidateStats()

  if mc_now > mc_max:

    lu.LogInfo("Note: more nodes are candidates (%d) than desired (%d)" %

               (mc_now, mc_max))

def _CheckInstanceBridgesExist(lu, instance):

  """Check that the brigdes needed by an instance exist.

  """

  # check bridges existance

  brlist = [nic.bridge for nic in instance.nics]

  result = lu.rpc.call_bridges_exist(instance.primary_node, brlist)

  result.Raise()

  if not result.data:

    raise errors.OpPrereqError("One or more target bridges %s does not"

                               " exist on destination node '%s'" %

                               (brlist, instance.primary_node))

class LUDestroyCluster(NoHooksLU):

  """Logical unit for destroying the cluster.

  """

  _OP_REQP = []

  def CheckPrereq(self):

    """Check prerequisites.

    This checks whether the cluster is empty.

    Any errors are signalled by raising errors.OpPrereqError.

    """

    master = self.cfg.GetMasterNode()

    nodelist = self.cfg.GetNodeList()

    if len(nodelist) != 1 or nodelist[0] != master:

      raise errors.OpPrereqError("There are still %d node(s) in"

                                 " this cluster." % (len(nodelist) - 1))

    instancelist = self.cfg.GetInstanceList()

    if instancelist:

      raise errors.OpPrereqError("There are still %d instance(s) in"

                                 " this cluster." % len(instancelist))

  def Exec(self, feedback_fn):

    """Destroys the cluster.

    """

    master = self.cfg.GetMasterNode()

    result = self.rpc.call_node_stop_master(master, False)

    result.Raise()

    if not result.data:

      raise errors.OpExecError("Could not disable the master role")

    priv_key, pub_key, _ = ssh.GetUserFiles(constants.GANETI_RUNAS)

    utils.CreateBackup(priv_key)

    utils.CreateBackup(pub_key)

    return master

class LUVerifyCluster(LogicalUnit):

  """Verifies the cluster status.

  """

  HPATH = "cluster-verify"

  HTYPE = constants.HTYPE_CLUSTER

  _OP_REQP = ["skip_checks"]

  REQ_BGL = False

  def ExpandNames(self):

    self.needed_locks = {

      locking.LEVEL_NODE: locking.ALL_SET,

      locking.LEVEL_INSTANCE: locking.ALL_SET,

    }

    self.share_locks = dict(((i, 1) for i in locking.LEVELS))

  def _VerifyNode(self, nodeinfo, file_list, local_cksum,

                  node_result, feedback_fn, master_files,

                  drbd_map):

    """Run multiple tests against a node.

    Test list:

      - compares ganeti version

      - checks vg existance and size > 20G

      - checks config file checksum

      - checks ssh to other nodes

    @type nodeinfo: L{objects.Node}

    @param nodeinfo: the node to check

    @param file_list: required list of files

    @param local_cksum: dictionary of local files and their checksums

    @param node_result: the results from the node

    @param feedback_fn: function used to accumulate results

    @param master_files: list of files that only masters should have

    @param drbd_map: the useddrbd minors for this node, in

        form of minor: (instance, must_exist) which correspond to instances

        and their running status

    """

    node = nodeinfo.name

    # main result, node_result should be a non-empty dict

    if not node_result or not isinstance(node_result, dict):

      feedback_fn("  - ERROR: unable to verify node %s." % (node,))

      return True

    # compares ganeti version

    local_version = constants.PROTOCOL_VERSION

    remote_version = node_result.get('version', None)

    if not remote_version:

      feedback_fn("  - ERROR: connection to %s failed" % (node))

      return True

    if local_version != remote_version:

      feedback_fn("  - ERROR: sw version mismatch: master %s, node(%s) %s" %

                      (local_version, node, remote_version))

      return True

    # checks vg existance and size > 20G

    bad = False

    vglist = node_result.get(constants.NV_VGLIST, None)

    if not vglist:

      feedback_fn("  - ERROR: unable to check volume groups on node %s." %

                      (node,))

      bad = True

    else:

      vgstatus = utils.CheckVolumeGroupSize(vglist, self.cfg.GetVGName(),

                                            constants.MIN_VG_SIZE)

      if vgstatus:

        feedback_fn("  - ERROR: %s on node %s" % (vgstatus, node))

        bad = True

    # checks config file checksum

    remote_cksum = node_result.get(constants.NV_FILELIST, None)

    if not isinstance(remote_cksum, dict):

      bad = True

      feedback_fn("  - ERROR: node hasn't returned file checksum data")

    else:

      for file_name in file_list:

        node_is_mc = nodeinfo.master_candidate

        must_have_file = file_name not in master_files

        if file_name not in remote_cksum:

          if node_is_mc or must_have_file:

            bad = True

            feedback_fn("  - ERROR: file '%s' missing" % file_name)

        elif remote_cksum[file_name] != local_cksum[file_name]:

          if node_is_mc or must_have_file:

            bad = True

            feedback_fn("  - ERROR: file '%s' has wrong checksum" % file_name)

          else:

            # not candidate and this is not a must-have file

            bad = True

            feedback_fn("  - ERROR: non master-candidate has old/wrong file"

                        " '%s'" % file_name)

        else:

          # all good, except non-master/non-must have combination

          if not node_is_mc and not must_have_file:

            feedback_fn("  - ERROR: file '%s' should not exist on non master"

                        " candidates" % file_name)

    # checks ssh to any

    if constants.NV_NODELIST not in node_result:

      bad = True

      feedback_fn("  - ERROR: node hasn't returned node ssh connectivity data")

    else:

      if node_result[constants.NV_NODELIST]:

        bad = True

        for node in node_result[constants.NV_NODELIST]:

          feedback_fn("  - ERROR: ssh communication with node '%s': %s" %

                          (node, node_result[constants.NV_NODELIST][node]))

    if constants.NV_NODENETTEST not in node_result:

      bad = True

      feedback_fn("  - ERROR: node hasn't returned node tcp connectivity data")

    else:

      if node_result[constants.NV_NODENETTEST]:

        bad = True

        nlist = utils.NiceSort(node_result[constants.NV_NODENETTEST].keys())

        for node in nlist:

          feedback_fn("  - ERROR: tcp communication with node '%s': %s" %

                          (node, node_result[constants.NV_NODENETTEST][node]))

    hyp_result = node_result.get(constants.NV_HYPERVISOR, None)

    if isinstance(hyp_result, dict):

      for hv_name, hv_result in hyp_result.iteritems():

        if hv_result is not None:

          feedback_fn("  - ERROR: hypervisor %s verify failure: '%s'" %

                      (hv_name, hv_result))