Synnefo » snf-ganeti

#

#

# Copyright (C) 2006, 2007, 2008 Google Inc.

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful, but

# WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

# General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

# 02110-1301, USA.

"""Module implementing the master-side code."""

# pylint: disable-msg=W0613,W0201

import os

import os.path

import sha

import time

import tempfile

import re

import platform

from ganeti import rpc

from ganeti import ssh

from ganeti import logger

from ganeti import utils

from ganeti import errors

from ganeti import hypervisor

from ganeti import config

from ganeti import constants

from ganeti import objects

from ganeti import opcodes

from ganeti import ssconf

from ganeti import serializer

class LogicalUnit(object):

  """Logical Unit base class.

  Subclasses must follow these rules:

    - implement CheckPrereq which also fills in the opcode instance

      with all the fields (even if as None)

    - implement Exec

    - implement BuildHooksEnv

    - redefine HPATH and HTYPE

    - optionally redefine their run requirements:

        REQ_MASTER: the LU needs to run on the master node

        REQ_WSSTORE: the LU needs a writable SimpleStore

  Note that all commands require root permissions.

  """

  HPATH = None

  HTYPE = None

  _OP_REQP = []

  REQ_MASTER = True

  REQ_WSSTORE = False

  def __init__(self, processor, op, cfg, sstore):

    """Constructor for LogicalUnit.

    This needs to be overriden in derived classes in order to check op

    validity.

    """

    self.proc = processor

    self.op = op

    self.cfg = cfg

    self.sstore = sstore

    self.__ssh = None

    for attr_name in self._OP_REQP:

      attr_val = getattr(op, attr_name, None)

      if attr_val is None:

        raise errors.OpPrereqError("Required parameter '%s' missing" %

                                   attr_name)

    if not cfg.IsCluster():

      raise errors.OpPrereqError("Cluster not initialized yet,"

                                 " use 'gnt-cluster init' first.")

    if self.REQ_MASTER:

      master = sstore.GetMasterNode()

      if master != utils.HostInfo().name:

        raise errors.OpPrereqError("Commands must be run on the master"

                                   " node %s" % master)

  def __GetSSH(self):

    """Returns the SshRunner object

    """

    if not self.__ssh:

      self.__ssh = ssh.SshRunner(self.sstore)

    return self.__ssh

  ssh = property(fget=__GetSSH)

  def CheckPrereq(self):

    """Check prerequisites for this LU.

    This method should check that the prerequisites for the execution

    of this LU are fulfilled. It can do internode communication, but

    it should be idempotent - no cluster or system changes are

    allowed.

    The method should raise errors.OpPrereqError in case something is

    not fulfilled. Its return value is ignored.

    This method should also update all the parameters of the opcode to

    their canonical form; e.g. a short node name must be fully

    expanded after this method has successfully completed (so that

    hooks, logging, etc. work correctly).

    """

    raise NotImplementedError

  def Exec(self, feedback_fn):

    """Execute the LU.

    This method should implement the actual work. It should raise

    errors.OpExecError for failures that are somewhat dealt with in

    code, or expected.

    """

    raise NotImplementedError

  def BuildHooksEnv(self):

    """Build hooks environment for this LU.

    This method should return a three-node tuple consisting of: a dict

    containing the environment that will be used for running the

    specific hook for this LU, a list of node names on which the hook

    should run before the execution, and a list of node names on which

    the hook should run after the execution.

    The keys of the dict must not have 'GANETI_' prefixed as this will

    be handled in the hooks runner. Also note additional keys will be

    added by the hooks runner. If the LU doesn't define any

    environment, an empty dict (and not None) should be returned.

    No nodes should be returned as an empty list (and not None).

    Note that if the HPATH for a LU class is None, this function will

    not be called.

    """

    raise NotImplementedError

  def HooksCallBack(self, phase, hook_results, feedback_fn, lu_result):

    """Notify the LU about the results of its hooks.

    This method is called every time a hooks phase is executed, and notifies

    the Logical Unit about the hooks' result. The LU can then use it to alter

    its result based on the hooks.  By default the method does nothing and the

    previous result is passed back unchanged but any LU can define it if it

    wants to use the local cluster hook-scripts somehow.

    Args:

      phase: the hooks phase that has just been run

      hooks_results: the results of the multi-node hooks rpc call

      feedback_fn: function to send feedback back to the caller

      lu_result: the previous result this LU had, or None in the PRE phase.

    """

    return lu_result

class NoHooksLU(LogicalUnit):

  """Simple LU which runs no hooks.

  This LU is intended as a parent for other LogicalUnits which will

  run no hooks, in order to reduce duplicate code.

  """

  HPATH = None

  HTYPE = None

def _GetWantedNodes(lu, nodes):

  """Returns list of checked and expanded node names.

  Args:

    nodes: List of nodes (strings) or None for all

  """

  if not isinstance(nodes, list):

    raise errors.OpPrereqError("Invalid argument type 'nodes'")

  if nodes:

    wanted = []

    for name in nodes:

      node = lu.cfg.ExpandNodeName(name)

      if node is None:

        raise errors.OpPrereqError("No such node name '%s'" % name)

      wanted.append(node)

  else:

    wanted = lu.cfg.GetNodeList()

  return utils.NiceSort(wanted)

def _GetWantedInstances(lu, instances):

  """Returns list of checked and expanded instance names.

  Args:

    instances: List of instances (strings) or None for all

  """

  if not isinstance(instances, list):

    raise errors.OpPrereqError("Invalid argument type 'instances'")

  if instances:

    wanted = []

    for name in instances:

      instance = lu.cfg.ExpandInstanceName(name)

      if instance is None:

        raise errors.OpPrereqError("No such instance name '%s'" % name)

      wanted.append(instance)

  else:

    wanted = lu.cfg.GetInstanceList()

  return utils.NiceSort(wanted)

def _CheckOutputFields(static, dynamic, selected):

  """Checks whether all selected fields are valid.

  Args:

    static: Static fields

    dynamic: Dynamic fields

  """

  static_fields = frozenset(static)

  dynamic_fields = frozenset(dynamic)

  all_fields = static_fields | dynamic_fields

  if not all_fields.issuperset(selected):

    raise errors.OpPrereqError("Unknown output fields selected: %s"

                               % ",".join(frozenset(selected).

                                          difference(all_fields)))

def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status,

                          memory, vcpus, nics):

  """Builds instance related env variables for hooks from single variables.

  Args:

    secondary_nodes: List of secondary nodes as strings

  """

  env = {

    "OP_TARGET": name,

    "INSTANCE_NAME": name,

    "INSTANCE_PRIMARY": primary_node,

    "INSTANCE_SECONDARIES": " ".join(secondary_nodes),

    "INSTANCE_OS_TYPE": os_type,

    "INSTANCE_STATUS": status,

    "INSTANCE_MEMORY": memory,

    "INSTANCE_VCPUS": vcpus,

  }

  if nics:

    nic_count = len(nics)

    for idx, (ip, bridge, mac) in enumerate(nics):

      if ip is None:

        ip = ""

      env["INSTANCE_NIC%d_IP" % idx] = ip

      env["INSTANCE_NIC%d_BRIDGE" % idx] = bridge

      env["INSTANCE_NIC%d_HWADDR" % idx] = mac

  else:

    nic_count = 0

  env["INSTANCE_NIC_COUNT"] = nic_count

  return env

def _BuildInstanceHookEnvByObject(instance, override=None):

  """Builds instance related env variables for hooks from an object.

  Args:

    instance: objects.Instance object of instance

    override: dict of values to override

  """

  args = {

    'name': instance.name,

    'primary_node': instance.primary_node,

    'secondary_nodes': instance.secondary_nodes,

    'os_type': instance.os,

    'status': instance.os,

    'memory': instance.memory,

    'vcpus': instance.vcpus,

    'nics': [(nic.ip, nic.bridge, nic.mac) for nic in instance.nics],

  }

  if override:

    args.update(override)

  return _BuildInstanceHookEnv(**args)

def _CheckInstanceBridgesExist(instance):

  """Check that the brigdes needed by an instance exist.

  """

  # check bridges existance

  brlist = [nic.bridge for nic in instance.nics]

  if not rpc.call_bridges_exist(instance.primary_node, brlist):

    raise errors.OpPrereqError("one or more target bridges %s does not"

                               " exist on destination node '%s'" %

                               (brlist, instance.primary_node))

class LUDestroyCluster(NoHooksLU):

  """Logical unit for destroying the cluster.

  """

  _OP_REQP = []

  def CheckPrereq(self):

    """Check prerequisites.

    This checks whether the cluster is empty.

    Any errors are signalled by raising errors.OpPrereqError.

    """

    master = self.sstore.GetMasterNode()

    nodelist = self.cfg.GetNodeList()

    if len(nodelist) != 1 or nodelist[0] != master:

      raise errors.OpPrereqError("There are still %d node(s) in"

                                 " this cluster." % (len(nodelist) - 1))

    instancelist = self.cfg.GetInstanceList()

    if instancelist:

      raise errors.OpPrereqError("There are still %d instance(s) in"

                                 " this cluster." % len(instancelist))

  def Exec(self, feedback_fn):

    """Destroys the cluster.

    """

    master = self.sstore.GetMasterNode()

    if not rpc.call_node_stop_master(master):

      raise errors.OpExecError("Could not disable the master role")

    priv_key, pub_key, _ = ssh.GetUserFiles(constants.GANETI_RUNAS)

    utils.CreateBackup(priv_key)

    utils.CreateBackup(pub_key)

    rpc.call_node_leave_cluster(master)

class LUVerifyCluster(LogicalUnit):

  """Verifies the cluster status.

  """

  HPATH = "cluster-verify"

  HTYPE = constants.HTYPE_CLUSTER

  _OP_REQP = ["skip_checks"]

  def _VerifyNode(self, node, file_list, local_cksum, vglist, node_result,

                  remote_version, feedback_fn):

    """Run multiple tests against a node.

    Test list:

      - compares ganeti version

      - checks vg existance and size > 20G

      - checks config file checksum

      - checks ssh to other nodes

    Args:

      node: name of the node to check

      file_list: required list of files

      local_cksum: dictionary of local files and their checksums

    """

    # compares ganeti version

    local_version = constants.PROTOCOL_VERSION

    if not remote_version:

      feedback_fn("  - ERROR: connection to %s failed" % (node))

      return True

    if local_version != remote_version:

      feedback_fn("  - ERROR: sw version mismatch: master %s, node(%s) %s" %

                      (local_version, node, remote_version))

      return True

    # checks vg existance and size > 20G

    bad = False

    if not vglist:

      feedback_fn("  - ERROR: unable to check volume groups on node %s." %

                      (node,))

      bad = True

    else:

      vgstatus = utils.CheckVolumeGroupSize(vglist, self.cfg.GetVGName(),

                                            constants.MIN_VG_SIZE)

      if vgstatus:

        feedback_fn("  - ERROR: %s on node %s" % (vgstatus, node))

        bad = True

    # checks config file checksum

    # checks ssh to any

    if 'filelist' not in node_result:

      bad = True

      feedback_fn("  - ERROR: node hasn't returned file checksum data")

    else:

      remote_cksum = node_result['filelist']

      for file_name in file_list:

        if file_name not in remote_cksum:

          bad = True

          feedback_fn("  - ERROR: file '%s' missing" % file_name)

        elif remote_cksum[file_name] != local_cksum[file_name]:

          bad = True

          feedback_fn("  - ERROR: file '%s' has wrong checksum" % file_name)

    if 'nodelist' not in node_result:

      bad = True

      feedback_fn("  - ERROR: node hasn't returned node ssh connectivity data")

    else:

      if node_result['nodelist']:

        bad = True

        for node in node_result['nodelist']:

          feedback_fn("  - ERROR: ssh communication with node '%s': %s" %

                          (node, node_result['nodelist'][node]))

    if 'node-net-test' not in node_result:

      bad = True

      feedback_fn("  - ERROR: node hasn't returned node tcp connectivity data")

    else:

      if node_result['node-net-test']:

        bad = True

        nlist = utils.NiceSort(node_result['node-net-test'].keys())

        for node in nlist:

          feedback_fn("  - ERROR: tcp communication with node '%s': %s" %

                          (node, node_result['node-net-test'][node]))

    hyp_result = node_result.get('hypervisor', None)

    if hyp_result is not None:

      feedback_fn("  - ERROR: hypervisor verify failure: '%s'" % hyp_result)

    return bad

  def _VerifyInstance(self, instance, instanceconfig, node_vol_is,

                      node_instance, feedback_fn):

    """Verify an instance.

    This function checks to see if the required block devices are

    available on the instance's node.

    """

    bad = False

    node_current = instanceconfig.primary_node

    node_vol_should = {}

    instanceconfig.MapLVsByNode(node_vol_should)

    for node in node_vol_should:

      for volume in node_vol_should[node]:

        if node not in node_vol_is or volume not in node_vol_is[node]:

          feedback_fn("  - ERROR: volume %s missing on node %s" %

                          (volume, node))

          bad = True

    if not instanceconfig.status == 'down':

      if (node_current not in node_instance or

          not instance in node_instance[node_current]):

        feedback_fn("  - ERROR: instance %s not running on node %s" %

                        (instance, node_current))

        bad = True

    for node in node_instance:

      if (not node == node_current):

        if instance in node_instance[node]:

          feedback_fn("  - ERROR: instance %s should not run on node %s" %

                          (instance, node))

          bad = True

    return bad

  def _VerifyOrphanVolumes(self, node_vol_should, node_vol_is, feedback_fn):

    """Verify if there are any unknown volumes in the cluster.

    The .os, .swap and backup volumes are ignored. All other volumes are

    reported as unknown.

    """

    bad = False

    for node in node_vol_is:

      for volume in node_vol_is[node]:

        if node not in node_vol_should or volume not in node_vol_should[node]:

          feedback_fn("  - ERROR: volume %s on node %s should not exist" %

                      (volume, node))

          bad = True

    return bad

  def _VerifyOrphanInstances(self, instancelist, node_instance, feedback_fn):

    """Verify the list of running instances.

    This checks what instances are running but unknown to the cluster.

    """

    bad = False

    for node in node_instance:

      for runninginstance in node_instance[node]:

        if runninginstance not in instancelist:

          feedback_fn("  - ERROR: instance %s on node %s should not exist" %

                          (runninginstance, node))

          bad = True

    return bad

  def _VerifyNPlusOneMemory(self, node_info, instance_cfg, feedback_fn):

    """Verify N+1 Memory Resilience.

    Check that if one single node dies we can still start all the instances it

    was primary for.

    """

    bad = False

    for node, nodeinfo in node_info.iteritems():

      # This code checks that every node which is now listed as secondary has

      # enough memory to host all instances it is supposed to should a single

      # other node in the cluster fail.

      # FIXME: not ready for failover to an arbitrary node

      # FIXME: does not support file-backed instances

      # WARNING: we currently take into account down instances as well as up

      # ones, considering that even if they're down someone might want to start

      # them even in the event of a node failure.

      for prinode, instances in nodeinfo['sinst-by-pnode'].iteritems():

        needed_mem = 0

        for instance in instances:

          needed_mem += instance_cfg[instance].memory

        if nodeinfo['mfree'] < needed_mem:

          feedback_fn("  - ERROR: not enough memory on node %s to accomodate"

                      " failovers should node %s fail" % (node, prinode))

          bad = True

    return bad

  def CheckPrereq(self):

    """Check prerequisites.

    Transform the list of checks we're going to skip into a set and check that

    all its members are valid.

    """

    self.skip_set = frozenset(self.op.skip_checks)

    if not constants.VERIFY_OPTIONAL_CHECKS.issuperset(self.skip_set):

      raise errors.OpPrereqError("Invalid checks to be skipped specified")

  def BuildHooksEnv(self):

    """Build hooks env.

    Cluster-Verify hooks just rone in the post phase and their failure makes

    the output be logged in the verify output and the verification to fail.

    """

    all_nodes = self.cfg.GetNodeList()

    # TODO: populate the environment with useful information for verify hooks

    env = {}

    return env, [], all_nodes

  def Exec(self, feedback_fn):

    """Verify integrity of cluster, performing various test on nodes.

    """

    bad = False

    feedback_fn("* Verifying global settings")

    for msg in self.cfg.VerifyConfig():

      feedback_fn("  - ERROR: %s" % msg)

    vg_name = self.cfg.GetVGName()

    nodelist = utils.NiceSort(self.cfg.GetNodeList())

    nodeinfo = [self.cfg.GetNodeInfo(nname) for nname in nodelist]

    instancelist = utils.NiceSort(self.cfg.GetInstanceList())

    i_non_redundant = [] # Non redundant instances

    node_volume = {}

    node_instance = {}

    node_info = {}

    instance_cfg = {}

    # FIXME: verify OS list

    # do local checksums

    file_names = list(self.sstore.GetFileList())

    file_names.append(constants.SSL_CERT_FILE)

    file_names.append(constants.CLUSTER_CONF_FILE)

    local_checksums = utils.FingerprintFiles(file_names)

    feedback_fn("* Gathering data (%d nodes)" % len(nodelist))

    all_volumeinfo = rpc.call_volume_list(nodelist, vg_name)

    all_instanceinfo = rpc.call_instance_list(nodelist)

    all_vglist = rpc.call_vg_list(nodelist)

    node_verify_param = {

      'filelist': file_names,

      'nodelist': nodelist,

      'hypervisor': None,

      'node-net-test': [(node.name, node.primary_ip, node.secondary_ip)

                        for node in nodeinfo]

      }

    all_nvinfo = rpc.call_node_verify(nodelist, node_verify_param)

    all_rversion = rpc.call_version(nodelist)

    all_ninfo = rpc.call_node_info(nodelist, self.cfg.GetVGName())

    for node in nodelist:

      feedback_fn("* Verifying node %s" % node)

      result = self._VerifyNode(node, file_names, local_checksums,

                                all_vglist[node], all_nvinfo[node],

                                all_rversion[node], feedback_fn)

      bad = bad or result

      # node_volume

      volumeinfo = all_volumeinfo[node]

      if isinstance(volumeinfo, basestring):

        feedback_fn("  - ERROR: LVM problem on node %s: %s" %

                    (node, volumeinfo[-400:].encode('string_escape')))

        bad = True

        node_volume[node] = {}

      elif not isinstance(volumeinfo, dict):

        feedback_fn("  - ERROR: connection to %s failed" % (node,))

        bad = True

        continue

      else:

        node_volume[node] = volumeinfo

      # node_instance

      nodeinstance = all_instanceinfo[node]

      if type(nodeinstance) != list:

        feedback_fn("  - ERROR: connection to %s failed" % (node,))

        bad = True

        continue

      node_instance[node] = nodeinstance

      # node_info

      nodeinfo = all_ninfo[node]

      if not isinstance(nodeinfo, dict):

        feedback_fn("  - ERROR: connection to %s failed" % (node,))

        bad = True

        continue

      try:

        node_info[node] = {

          "mfree": int(nodeinfo['memory_free']),

          "dfree": int(nodeinfo['vg_free']),

          "pinst": [],

          "sinst": [],

          # dictionary holding all instances this node is secondary for,

          # grouped by their primary node. Each key is a cluster node, and each

          # value is a list of instances which have the key as primary and the

          # current node as secondary.  this is handy to calculate N+1 memory

          # availability if you can only failover from a primary to its

          # secondary.

          "sinst-by-pnode": {},

        }

      except ValueError:

        feedback_fn("  - ERROR: invalid value returned from node %s" % (node,))

        bad = True

        continue

    node_vol_should = {}

    for instance in instancelist:

      feedback_fn("* Verifying instance %s" % instance)

      inst_config = self.cfg.GetInstanceInfo(instance)

      result =  self._VerifyInstance(instance, inst_config, node_volume,

                                     node_instance, feedback_fn)

      bad = bad or result

      inst_config.MapLVsByNode(node_vol_should)

      instance_cfg[instance] = inst_config

      pnode = inst_config.primary_node

      if pnode in node_info:

        node_info[pnode]['pinst'].append(instance)

      else:

        feedback_fn("  - ERROR: instance %s, connection to primary node"

                    " %s failed" % (instance, pnode))

        bad = True

      # If the instance is non-redundant we cannot survive losing its primary

      # node, so we are not N+1 compliant. On the other hand we have no disk

      # templates with more than one secondary so that situation is not well

      # supported either.

      # FIXME: does not support file-backed instances

      if len(inst_config.secondary_nodes) == 0:

        i_non_redundant.append(instance)

      elif len(inst_config.secondary_nodes) > 1:

        feedback_fn("  - WARNING: multiple secondaries for instance %s"

                    % instance)

      for snode in inst_config.secondary_nodes:

        if snode in node_info:

          node_info[snode]['sinst'].append(instance)

          if pnode not in node_info[snode]['sinst-by-pnode']:

            node_info[snode]['sinst-by-pnode'][pnode] = []

          node_info[snode]['sinst-by-pnode'][pnode].append(instance)

        else:

          feedback_fn("  - ERROR: instance %s, connection to secondary node"

                      " %s failed" % (instance, snode))

    feedback_fn("* Verifying orphan volumes")

    result = self._VerifyOrphanVolumes(node_vol_should, node_volume,

                                       feedback_fn)

    bad = bad or result

    feedback_fn("* Verifying remaining instances")

    result = self._VerifyOrphanInstances(instancelist, node_instance,

                                         feedback_fn)

    bad = bad or result

    if constants.VERIFY_NPLUSONE_MEM not in self.skip_set:

      feedback_fn("* Verifying N+1 Memory redundancy")

      result = self._VerifyNPlusOneMemory(node_info, instance_cfg, feedback_fn)

      bad = bad or result

    feedback_fn("* Other Notes")

    if i_non_redundant:

      feedback_fn("  - NOTICE: %d non-redundant instance(s) found."

                  % len(i_non_redundant))

    return int(bad)

  def HooksCallBack(self, phase, hooks_results, feedback_fn, lu_result):

    """Analize the post-hooks' result, handle it, and send some

    nicely-formatted feedback back to the user.

    Args:

      phase: the hooks phase that has just been run

      hooks_results: the results of the multi-node hooks rpc call

      feedback_fn: function to send feedback back to the caller

      lu_result: previous Exec result

    """

    # We only really run POST phase hooks, and are only interested in their results

    if phase == constants.HOOKS_PHASE_POST:

      # Used to change hooks' output to proper indentation

      indent_re = re.compile('^', re.M)