Synnefo » snf-ganeti » ganeti-local

from ganeti import bootstrap

  In order to account for cold-start of all nodes, we retry for up to

  a minute until we get a real answer as the top-voted one. If the

  nodes are more out-of-sync, for now manual startup of the master

  should be attempted.

  Note that for a even number of nodes cluster, we need at least half

  of the nodes (beside ourselves) to vote for us. This creates a

  problem on two-node clusters, since in this case we require the

  other node to be up too to confirm our status.

  retries = 6

  while retries > 0:

    votes = bootstrap.GatherMasterVotes(node_list)

    if not votes:

      # empty node list, this is a one node cluster

      return True

    if votes[0][0] is None:

      retries -= 1

      time.sleep(10)

    break

  if retries == 0:

      logging.critical("Cluster inconsistent, most of the nodes didn't answer"

                       " after multiple retries. Aborting startup")

      return False

  # here a real node is at the top of the list

  all_votes = sum(item[1] for item in votes)

  top_node, top_votes = votes[0]

  result = False

  if top_node != myself:

    logging.critical("It seems we are not the master (top-voted node"

                     " is %s)", top_node)

  elif top_votes < all_votes - top_votes:

                     " %d votes against)", top_votes, all_votes - top_votes)

  else:

    result = True

  return result

def GatherMasterVotes(node_list):

  """Check the agreement on who is the master.

  This function will return a list of (node, number of votes), ordered

  by the number of votes. Errors will be denoted by the key 'None'.

  Note that the sum of votes is the number of nodes this machine

  knows, whereas the number of entries in the list could be different

  (if some nodes vote for another master).

  We remove ourselves from the list since we know that (bugs aside)

  since we use the same source for configuration information for both

  backend and boostrap, we'll always vote for ourselves.

  @type node_list: list

  @param node_list: the list of nodes to query for master info; the current

      node wil be removed if it is in the list

  @rtype: list

  @return: list of (node, votes)

  """

  myself = utils.HostInfo().name

  try:

    node_list.remove(myself)

  except ValueError:

    pass

  if not node_list:

    # no nodes left (eventually after removing myself)

    return []

  results = rpc.RpcRunner.call_master_info(node_list)

  if not isinstance(results, dict):

    # this should not happen (unless internal error in rpc)

    logging.critical("Can't complete rpc call, aborting master startup")

    return [(None, len(node_list))]

  positive = negative = 0

  other_masters = {}

  votes = {}

  for node in results:

    if not isinstance(results[node], (tuple, list)) or len(results[node]) < 3:

      # here the rpc layer should have already logged errors

      if None not in votes:

        votes[None] = 0

      votes[None] += 1

      continue

    master_node = results[node][2]

    if master_node not in votes:

      votes[master_node] = 0

    votes[master_node] += 1

  vote_list = [v for v in votes.items()]

  # sort first on number of votes then on name, since we want None

  # sorted later if we have the half of the nodes not responding, and

  # half voting all for the same master

  vote_list.sort(key=lambda x: (x[1], x[0]), reverse=True)

  return vote_list