| Branch: | Tag: | Revision:

root / lib / @ f3044516

History | View | Annotate | Download (25.8 kB)

1 a0c9f010 Michael Hanselmann
2 a0c9f010 Michael Hanselmann
3 a0c9f010 Michael Hanselmann
4 600535f0 Manuel Franceschini
# Copyright (C) 2006, 2007, 2008, 2010 Google Inc.
5 a0c9f010 Michael Hanselmann
6 a0c9f010 Michael Hanselmann
# This program is free software; you can redistribute it and/or modify
7 a0c9f010 Michael Hanselmann
# it under the terms of the GNU General Public License as published by
8 a0c9f010 Michael Hanselmann
# the Free Software Foundation; either version 2 of the License, or
9 a0c9f010 Michael Hanselmann
# (at your option) any later version.
10 a0c9f010 Michael Hanselmann
11 a0c9f010 Michael Hanselmann
# This program is distributed in the hope that it will be useful, but
12 a0c9f010 Michael Hanselmann
# WITHOUT ANY WARRANTY; without even the implied warranty of
13 a0c9f010 Michael Hanselmann
14 a0c9f010 Michael Hanselmann
# General Public License for more details.
15 a0c9f010 Michael Hanselmann
16 a0c9f010 Michael Hanselmann
# You should have received a copy of the GNU General Public License
17 a0c9f010 Michael Hanselmann
# along with this program; if not, write to the Free Software
18 a0c9f010 Michael Hanselmann
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 a0c9f010 Michael Hanselmann
# 02110-1301, USA.
20 a0c9f010 Michael Hanselmann
21 a0c9f010 Michael Hanselmann
22 a0c9f010 Michael Hanselmann
"""Functions to bootstrap a new cluster.
23 a0c9f010 Michael Hanselmann

24 a0c9f010 Michael Hanselmann
25 a0c9f010 Michael Hanselmann
26 a0c9f010 Michael Hanselmann
import os
27 a0c9f010 Michael Hanselmann
import os.path
28 a0c9f010 Michael Hanselmann
import re
29 b1b6ea87 Iustin Pop
import logging
30 d693c864 Iustin Pop
import time
31 a0c9f010 Michael Hanselmann
32 a0c9f010 Michael Hanselmann
from ganeti import rpc
33 a0c9f010 Michael Hanselmann
from ganeti import ssh
34 a0c9f010 Michael Hanselmann
from ganeti import utils
35 a0c9f010 Michael Hanselmann
from ganeti import errors
36 a0c9f010 Michael Hanselmann
from ganeti import config
37 a0c9f010 Michael Hanselmann
from ganeti import constants
38 b9eeeb02 Michael Hanselmann
from ganeti import objects
39 a0c9f010 Michael Hanselmann
from ganeti import ssconf
40 a33848a5 Guido Trotter
from ganeti import serializer
41 a5728081 Guido Trotter
from ganeti import hypervisor
42 a721e23a Luca Bigliardi
from ganeti import bdev
43 a744b676 Manuel Franceschini
from ganeti import netutils
44 d367b66c Manuel Franceschini
from ganeti import backend
45 a0c9f010 Michael Hanselmann
46 e38220e4 Michael Hanselmann
47 531baf8e Iustin Pop
def _InitSSHSetup():
48 a0c9f010 Michael Hanselmann
  """Setup the SSH configuration for the cluster.
49 a0c9f010 Michael Hanselmann

50 a0c9f010 Michael Hanselmann
  This generates a dsa keypair for root, adds the pub key to the
51 a0c9f010 Michael Hanselmann
  permitted hosts and adds the hostkey to its own known hosts.
52 a0c9f010 Michael Hanselmann

53 a0c9f010 Michael Hanselmann
54 a0c9f010 Michael Hanselmann
  priv_key, pub_key, auth_keys = ssh.GetUserFiles(constants.GANETI_RUNAS)
55 a0c9f010 Michael Hanselmann
56 a0c9f010 Michael Hanselmann
  for name in priv_key, pub_key:
57 a0c9f010 Michael Hanselmann
    if os.path.exists(name):
58 a0c9f010 Michael Hanselmann
59 a0c9f010 Michael Hanselmann
60 a0c9f010 Michael Hanselmann
61 a0c9f010 Michael Hanselmann
  result = utils.RunCmd(["ssh-keygen", "-t", "dsa",
62 a0c9f010 Michael Hanselmann
                         "-f", priv_key,
63 a0c9f010 Michael Hanselmann
                         "-q", "-N", ""])
64 a0c9f010 Michael Hanselmann
  if result.failed:
65 a0c9f010 Michael Hanselmann
    raise errors.OpExecError("Could not generate ssh keypair, error %s" %
66 a0c9f010 Michael Hanselmann
67 a0c9f010 Michael Hanselmann
68 7a0156dc Luca Bigliardi
  utils.AddAuthorizedKey(auth_keys, utils.ReadFile(pub_key))
69 a0c9f010 Michael Hanselmann
70 a0c9f010 Michael Hanselmann
71 c008906b Michael Hanselmann
def GenerateHmacKey(file_name):
72 c008906b Michael Hanselmann
  """Writes a new HMAC key.
73 c008906b Michael Hanselmann

74 c008906b Michael Hanselmann
  @type file_name: str
75 c008906b Michael Hanselmann
  @param file_name: Path to output file
76 c008906b Michael Hanselmann

77 c008906b Michael Hanselmann
78 43575108 Michael Hanselmann
  utils.WriteFile(file_name, data="%s\n" % utils.GenerateSecret(), mode=0400,
79 43575108 Michael Hanselmann
80 43575108 Michael Hanselmann
81 43575108 Michael Hanselmann
82 6b7d5878 Michael Hanselmann
def GenerateClusterCrypto(new_cluster_cert, new_rapi_cert, new_confd_hmac_key,
83 af2ae1c0 Iustin Pop
                          new_cds, rapi_cert_pem=None, cds=None,
84 aeefe835 Iustin Pop
85 aeefe835 Iustin Pop
86 fc0726b9 Michael Hanselmann
87 fc0726b9 Michael Hanselmann
88 43575108 Michael Hanselmann
  """Updates the cluster certificates, keys and secrets.
89 43575108 Michael Hanselmann

90 43575108 Michael Hanselmann
  @type new_cluster_cert: bool
91 43575108 Michael Hanselmann
  @param new_cluster_cert: Whether to generate a new cluster certificate
92 43575108 Michael Hanselmann
  @type new_rapi_cert: bool
93 43575108 Michael Hanselmann
  @param new_rapi_cert: Whether to generate a new RAPI certificate
94 6b7d5878 Michael Hanselmann
  @type new_confd_hmac_key: bool
95 6b7d5878 Michael Hanselmann
  @param new_confd_hmac_key: Whether to generate a new HMAC key
96 3db3eb2a Michael Hanselmann
  @type new_cds: bool
97 3db3eb2a Michael Hanselmann
  @param new_cds: Whether to generate a new cluster domain secret
98 43575108 Michael Hanselmann
  @type rapi_cert_pem: string
99 43575108 Michael Hanselmann
  @param rapi_cert_pem: New RAPI certificate in PEM format
100 3db3eb2a Michael Hanselmann
  @type cds: string
101 3db3eb2a Michael Hanselmann
  @param cds: New cluster domain secret
102 aeefe835 Iustin Pop
  @type nodecert_file: string
103 aeefe835 Iustin Pop
  @param nodecert_file: optional override of the node cert file path
104 aeefe835 Iustin Pop
  @type rapicert_file: string
105 aeefe835 Iustin Pop
  @param rapicert_file: optional override of the rapi cert file path
106 aeefe835 Iustin Pop
  @type hmackey_file: string
107 aeefe835 Iustin Pop
  @param hmackey_file: optional override of the hmac key file path
108 43575108 Michael Hanselmann

109 43575108 Michael Hanselmann
110 168c1de2 Michael Hanselmann
  # noded SSL certificate
111 aeefe835 Iustin Pop
  cluster_cert_exists = os.path.exists(nodecert_file)
112 43575108 Michael Hanselmann
  if new_cluster_cert or not cluster_cert_exists:
113 43575108 Michael Hanselmann
    if cluster_cert_exists:
114 aeefe835 Iustin Pop
115 43575108 Michael Hanselmann
116 aeefe835 Iustin Pop
    logging.debug("Generating new cluster certificate at %s", nodecert_file)
117 af2ae1c0 Iustin Pop
118 43575108 Michael Hanselmann
119 6b7d5878 Michael Hanselmann
  # confd HMAC key
120 aeefe835 Iustin Pop
  if new_confd_hmac_key or not os.path.exists(hmackey_file):
121 aeefe835 Iustin Pop
    logging.debug("Writing new confd HMAC key to %s", hmackey_file)
122 aeefe835 Iustin Pop
123 43575108 Michael Hanselmann
124 43575108 Michael Hanselmann
  # RAPI
125 aeefe835 Iustin Pop
  rapi_cert_exists = os.path.exists(rapicert_file)
126 43575108 Michael Hanselmann
127 43575108 Michael Hanselmann
  if rapi_cert_pem:
128 43575108 Michael Hanselmann
    # Assume rapi_pem contains a valid PEM-formatted certificate and key
129 aeefe835 Iustin Pop
    logging.debug("Writing RAPI certificate at %s", rapicert_file)
130 aeefe835 Iustin Pop
    utils.WriteFile(rapicert_file, data=rapi_cert_pem, backup=True)
131 43575108 Michael Hanselmann
132 43575108 Michael Hanselmann
  elif new_rapi_cert or not rapi_cert_exists:
133 43575108 Michael Hanselmann
    if rapi_cert_exists:
134 aeefe835 Iustin Pop
135 43575108 Michael Hanselmann
136 aeefe835 Iustin Pop
    logging.debug("Generating new RAPI certificate at %s", rapicert_file)
137 af2ae1c0 Iustin Pop
138 c008906b Michael Hanselmann
139 3db3eb2a Michael Hanselmann
  # Cluster domain secret
140 3db3eb2a Michael Hanselmann
  if cds:
141 fc0726b9 Michael Hanselmann
    logging.debug("Writing cluster domain secret to %s", cds_file)
142 fc0726b9 Michael Hanselmann
    utils.WriteFile(cds_file, data=cds, backup=True)
143 fc0726b9 Michael Hanselmann
144 fc0726b9 Michael Hanselmann
  elif new_cds or not os.path.exists(cds_file):
145 fc0726b9 Michael Hanselmann
    logging.debug("Generating new cluster domain secret at %s", cds_file)
146 fc0726b9 Michael Hanselmann
147 3db3eb2a Michael Hanselmann
148 c008906b Michael Hanselmann
149 8f215968 Michael Hanselmann
def _InitGanetiServerSetup(master_name):
150 40a97d80 Michael Hanselmann
  """Setup the necessary configuration for the initial node daemon.
151 40a97d80 Michael Hanselmann

152 40a97d80 Michael Hanselmann
  This creates the nodepass file containing the shared password for
153 600535f0 Manuel Franceschini
  the cluster, generates the SSL certificate and starts the node daemon.
154 600535f0 Manuel Franceschini

155 600535f0 Manuel Franceschini
  @type master_name: str
156 600535f0 Manuel Franceschini
  @param master_name: Name of the master node
157 40a97d80 Michael Hanselmann

158 40a97d80 Michael Hanselmann
159 43575108 Michael Hanselmann
  # Generate cluster secrets
160 3db3eb2a Michael Hanselmann
  GenerateClusterCrypto(True, False, False, False)
161 4a34c5cf Guido Trotter
162 f154a7a3 Michael Hanselmann
  result = utils.RunCmd([constants.DAEMON_UTIL, "start", constants.NODED])
163 a0c9f010 Michael Hanselmann
  if result.failed:
164 a0c9f010 Michael Hanselmann
    raise errors.OpExecError("Could not start the node daemon, command %s"
165 a0c9f010 Michael Hanselmann
                             " had exitcode %s and error %s" %
166 a0c9f010 Michael Hanselmann
                             (result.cmd, result.exit_code, result.output))
167 a0c9f010 Michael Hanselmann
168 5627f375 Michael Hanselmann
169 5627f375 Michael Hanselmann
170 5627f375 Michael Hanselmann
171 5627f375 Michael Hanselmann
def _WaitForNodeDaemon(node_name):
172 5627f375 Michael Hanselmann
  """Wait for node daemon to become responsive.
173 5627f375 Michael Hanselmann

174 5627f375 Michael Hanselmann
175 d3833ebd Michael Hanselmann
  def _CheckNodeDaemon():
176 5627f375 Michael Hanselmann
    result = rpc.RpcRunner.call_version([node_name])[node_name]
177 d3833ebd Michael Hanselmann
    if result.fail_msg:
178 d3833ebd Michael Hanselmann
      raise utils.RetryAgain()
179 8f215968 Michael Hanselmann
180 d3833ebd Michael Hanselmann
181 d3833ebd Michael Hanselmann
    utils.Retry(_CheckNodeDaemon, 1.0, 10.0)
182 d3833ebd Michael Hanselmann
  except utils.RetryTimeout:
183 5627f375 Michael Hanselmann
    raise errors.OpExecError("Node daemon on %s didn't answer queries within"
184 5627f375 Michael Hanselmann
                             " 10 seconds" % node_name)
185 5627f375 Michael Hanselmann
186 a0c9f010 Michael Hanselmann
187 0e3baaf3 Iustin Pop
def _InitFileStorage(file_storage_dir):
188 0e3baaf3 Iustin Pop
  """Initialize if needed the file storage.
189 0e3baaf3 Iustin Pop

190 0e3baaf3 Iustin Pop
  @param file_storage_dir: the user-supplied value
191 0e3baaf3 Iustin Pop
  @return: either empty string (if file storage was disabled at build
192 0e3baaf3 Iustin Pop
      time) or the normalized path to the storage directory
193 0e3baaf3 Iustin Pop

194 0e3baaf3 Iustin Pop
195 0e3baaf3 Iustin Pop
  if not constants.ENABLE_FILE_STORAGE:
196 0e3baaf3 Iustin Pop
    return ""
197 0e3baaf3 Iustin Pop
198 0e3baaf3 Iustin Pop
  file_storage_dir = os.path.normpath(file_storage_dir)
199 0e3baaf3 Iustin Pop
200 0e3baaf3 Iustin Pop
  if not os.path.isabs(file_storage_dir):
201 0e3baaf3 Iustin Pop
    raise errors.OpPrereqError("The file storage directory you passed is"
202 0e3baaf3 Iustin Pop
                               " not an absolute path.", errors.ECODE_INVAL)
203 0e3baaf3 Iustin Pop
204 0e3baaf3 Iustin Pop
  if not os.path.exists(file_storage_dir):
205 0e3baaf3 Iustin Pop
206 0e3baaf3 Iustin Pop
      os.makedirs(file_storage_dir, 0750)
207 0e3baaf3 Iustin Pop
    except OSError, err:
208 0e3baaf3 Iustin Pop
      raise errors.OpPrereqError("Cannot create file storage directory"
209 0e3baaf3 Iustin Pop
                                 " '%s': %s" % (file_storage_dir, err),
210 0e3baaf3 Iustin Pop
211 0e3baaf3 Iustin Pop
212 0e3baaf3 Iustin Pop
  if not os.path.isdir(file_storage_dir):
213 0e3baaf3 Iustin Pop
    raise errors.OpPrereqError("The file storage directory '%s' is not"
214 0e3baaf3 Iustin Pop
                               " a directory." % file_storage_dir,
215 0e3baaf3 Iustin Pop
216 0e3baaf3 Iustin Pop
  return file_storage_dir
217 0e3baaf3 Iustin Pop
218 0e3baaf3 Iustin Pop
219 a721e23a Luca Bigliardi
#pylint: disable-msg=R0913
220 ec0652ad Guido Trotter
def InitCluster(cluster_name, mac_prefix,
221 ce735215 Guido Trotter
                master_netdev, file_storage_dir, candidate_pool_size,
222 b6a30b0d Guido Trotter
                secondary_ip=None, vg_name=None, beparams=None,
223 b6a30b0d Guido Trotter
                nicparams=None, hvparams=None, enabled_hypervisors=None,
224 3953242f Iustin Pop
                modify_etc_hosts=True, modify_ssh_setup=True,
225 a721e23a Luca Bigliardi
                maintain_node_health=False, drbd_helper=None,
226 2f20d07b Manuel Franceschini
                uid_pool=None, default_iallocator=None,
227 2f20d07b Manuel Franceschini
228 a0c9f010 Michael Hanselmann
  """Initialise the cluster.
229 a0c9f010 Michael Hanselmann

230 ce735215 Guido Trotter
  @type candidate_pool_size: int
231 ce735215 Guido Trotter
  @param candidate_pool_size: master candidate pool size
232 ce735215 Guido Trotter

233 a0c9f010 Michael Hanselmann
234 ce735215 Guido Trotter
  # TODO: complete the docstring
235 a0c9f010 Michael Hanselmann
  if config.ConfigWriter.IsCluster():
236 debac808 Iustin Pop
    raise errors.OpPrereqError("Cluster is already initialised",
237 debac808 Iustin Pop
238 a0c9f010 Michael Hanselmann
239 b119bccb Guido Trotter
  if not enabled_hypervisors:
240 b119bccb Guido Trotter
    raise errors.OpPrereqError("Enabled hypervisors list must contain at"
241 debac808 Iustin Pop
                               " least one member", errors.ECODE_INVAL)
242 b119bccb Guido Trotter
  invalid_hvs = set(enabled_hypervisors) - constants.HYPER_TYPES
243 b119bccb Guido Trotter
  if invalid_hvs:
244 b119bccb Guido Trotter
    raise errors.OpPrereqError("Enabled hypervisors contains invalid"
245 debac808 Iustin Pop
                               " entries: %s" % invalid_hvs,
246 debac808 Iustin Pop
247 b119bccb Guido Trotter
248 a0c9f010 Michael Hanselmann
249 2f20d07b Manuel Franceschini
  ipcls = None
250 2f20d07b Manuel Franceschini
  if primary_ip_version == constants.IP4_VERSION:
251 2f20d07b Manuel Franceschini
    ipcls = netutils.IP4Address
252 2f20d07b Manuel Franceschini
  elif primary_ip_version == constants.IP6_VERSION:
253 2f20d07b Manuel Franceschini
    ipcls = netutils.IP6Address
254 2f20d07b Manuel Franceschini
255 2f20d07b Manuel Franceschini
    raise errors.OpPrereqError("Invalid primary ip version: %d." %
256 2f20d07b Manuel Franceschini
257 2f20d07b Manuel Franceschini
258 2f20d07b Manuel Franceschini
  hostname = netutils.GetHostname(
259 2f20d07b Manuel Franceschini
  if not ipcls.IsValid(hostname.ip):
260 2f20d07b Manuel Franceschini
    raise errors.OpPrereqError("This host's IP (%s) is not a valid IPv%d"
261 2f20d07b Manuel Franceschini
                               " address." % (hostname.ip, primary_ip_version))
262 2f20d07b Manuel Franceschini
263 2f20d07b Manuel Franceschini
  if ipcls.IsLoopback(hostname.ip):
264 8b312c1d Manuel Franceschini
    raise errors.OpPrereqError("This host's IP (%s) resolves to a loopback"
265 8b312c1d Manuel Franceschini
                               " address. Please fix DNS or %s." %
266 debac808 Iustin Pop
                               (hostname.ip, constants.ETC_HOSTS),
267 debac808 Iustin Pop
268 a0c9f010 Michael Hanselmann
269 2f20d07b Manuel Franceschini
  if not ipcls.Own(hostname.ip):
270 a0c9f010 Michael Hanselmann
    raise errors.OpPrereqError("Inconsistency: this host's name resolves"
271 a0c9f010 Michael Hanselmann
                               " to %s,\nbut this ip address does not"
272 debac808 Iustin Pop
                               " belong to this host. Aborting." %
273 debac808 Iustin Pop
                               hostname.ip, errors.ECODE_ENVIRON)
274 a0c9f010 Michael Hanselmann
275 2f20d07b Manuel Franceschini
  clustername = netutils.GetHostname(name=cluster_name,
276 a0c9f010 Michael Hanselmann
277 2f20d07b Manuel Franceschini
  if netutils.TcpPing(clustername.ip, constants.DEFAULT_NODED_PORT, timeout=5):
278 debac808 Iustin Pop
    raise errors.OpPrereqError("Cluster IP already active. Aborting.",
279 debac808 Iustin Pop
280 a0c9f010 Michael Hanselmann
281 2f20d07b Manuel Franceschini
  if not secondary_ip:
282 2f20d07b Manuel Franceschini
    if primary_ip_version == constants.IP6_VERSION:
283 2f20d07b Manuel Franceschini
      raise errors.OpPrereqError("When using a IPv6 primary address, a valid"
284 2f20d07b Manuel Franceschini
                                 " IPv4 address must be given as secondary."
285 2f20d07b Manuel Franceschini
                                 " Aborting.", errors.ECODE_INVAL)
286 b9eeeb02 Michael Hanselmann
    secondary_ip = hostname.ip
287 a0c9f010 Michael Hanselmann
288 2f20d07b Manuel Franceschini
  if not netutils.IP4Address.IsValid(secondary_ip):
289 2f20d07b Manuel Franceschini
    raise errors.OpPrereqError("Secondary IP address (%s) has to be a valid"
290 2f20d07b Manuel Franceschini
                               " IPv4 address." % secondary_ip,
291 2f20d07b Manuel Franceschini
292 2f20d07b Manuel Franceschini
293 2f20d07b Manuel Franceschini
  if not netutils.IP4Address.Own(secondary_ip):
294 2f20d07b Manuel Franceschini
    raise errors.OpPrereqError("You gave %s as secondary IP,"
295 2f20d07b Manuel Franceschini
                               " but it does not belong to this host." %
296 2f20d07b Manuel Franceschini
                               secondary_ip, errors.ECODE_ENVIRON)
297 2f20d07b Manuel Franceschini
298 a0c9f010 Michael Hanselmann
  if vg_name is not None:
299 a0c9f010 Michael Hanselmann
    # Check if volume group is valid
300 a0c9f010 Michael Hanselmann
    vgstatus = utils.CheckVolumeGroupSize(utils.ListVolumeGroups(), vg_name,
301 a0c9f010 Michael Hanselmann
302 a0c9f010 Michael Hanselmann
    if vgstatus:
303 a0c9f010 Michael Hanselmann
      raise errors.OpPrereqError("Error: %s\nspecify --no-lvm-storage if"
304 debac808 Iustin Pop
                                 " you are not using lvm" % vgstatus,
305 debac808 Iustin Pop
306 a0c9f010 Michael Hanselmann
307 a721e23a Luca Bigliardi
  if drbd_helper is not None:
308 a721e23a Luca Bigliardi
309 a721e23a Luca Bigliardi
      curr_helper = bdev.BaseDRBD.GetUsermodeHelper()
310 a721e23a Luca Bigliardi
    except errors.BlockDeviceError, err:
311 a721e23a Luca Bigliardi
      raise errors.OpPrereqError("Error while checking drbd helper"
312 a721e23a Luca Bigliardi
                                 " (specify --no-drbd-storage if you are not"
313 a721e23a Luca Bigliardi
                                 " using drbd): %s" % str(err),
314 a721e23a Luca Bigliardi
315 a721e23a Luca Bigliardi
    if drbd_helper != curr_helper:
316 a721e23a Luca Bigliardi
      raise errors.OpPrereqError("Error: requiring %s as drbd helper but %s"
317 a721e23a Luca Bigliardi
                                 " is the current helper" % (drbd_helper,
318 a721e23a Luca Bigliardi
319 a721e23a Luca Bigliardi
320 a721e23a Luca Bigliardi
321 0e3baaf3 Iustin Pop
  file_storage_dir = _InitFileStorage(file_storage_dir)
322 a0c9f010 Michael Hanselmann
323 a0c9f010 Michael Hanselmann
  if not re.match("^[0-9a-z]{2}:[0-9a-z]{2}:[0-9a-z]{2}$", mac_prefix):
324 debac808 Iustin Pop
    raise errors.OpPrereqError("Invalid mac prefix given '%s'" % mac_prefix,
325 debac808 Iustin Pop
326 a0c9f010 Michael Hanselmann
327 a0c9f010 Michael Hanselmann
  result = utils.RunCmd(["ip", "link", "show", "dev", master_netdev])
328 a0c9f010 Michael Hanselmann
  if result.failed:
329 a0c9f010 Michael Hanselmann
    raise errors.OpPrereqError("Invalid master netdev given (%s): '%s'" %
330 a0c9f010 Michael Hanselmann
331 debac808 Iustin Pop
                                result.output.strip()), errors.ECODE_INVAL)
332 a0c9f010 Michael Hanselmann
333 9dae41ad Guido Trotter
  dirs = [(constants.RUN_GANETI_DIR, constants.RUN_DIRS_MODE)]
334 9dae41ad Guido Trotter
335 9dae41ad Guido Trotter
336 a5728081 Guido Trotter
  utils.ForceDictType(beparams, constants.BES_PARAMETER_TYPES)
337 b6a30b0d Guido Trotter
  utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
338 b6a30b0d Guido Trotter
339 b6a30b0d Guido Trotter
340 a5728081 Guido Trotter
  # hvparams is a mapping of hypervisor->hvparams dict
341 a5728081 Guido Trotter
  for hv_name, hv_params in hvparams.iteritems():
342 a5728081 Guido Trotter
    utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
343 a5728081 Guido Trotter
    hv_class = hypervisor.GetHypervisor(hv_name)
344 a5728081 Guido Trotter
345 d4b72030 Guido Trotter
346 a0c9f010 Michael Hanselmann
  # set up ssh config and /etc/hosts
347 13998ef2 Michael Hanselmann
  sshline = utils.ReadFile(constants.SSH_HOST_RSA_PUB)
348 a0c9f010 Michael Hanselmann
  sshkey = sshline.split(" ")[1]
349 a0c9f010 Michael Hanselmann
350 b86a6bcd Guido Trotter
  if modify_etc_hosts:
351 b43dcc5a Manuel Franceschini
352 b86a6bcd Guido Trotter
353 b989b9d9 Ken Wehr
  if modify_ssh_setup:
354 b989b9d9 Ken Wehr
355 a0c9f010 Michael Hanselmann
356 bf4af505 Apollon Oikonomopoulos
  if default_iallocator is not None:
357 bf4af505 Apollon Oikonomopoulos
    alloc_script = utils.FindFile(default_iallocator,
358 bf4af505 Apollon Oikonomopoulos
359 bf4af505 Apollon Oikonomopoulos
360 bf4af505 Apollon Oikonomopoulos
    if alloc_script is None:
361 bf4af505 Apollon Oikonomopoulos
      raise errors.OpPrereqError("Invalid default iallocator script '%s'"
362 bf4af505 Apollon Oikonomopoulos
                                 " specified" % default_iallocator,
363 bf4af505 Apollon Oikonomopoulos
364 bf4af505 Apollon Oikonomopoulos
365 430b923c Iustin Pop
  now = time.time()
366 430b923c Iustin Pop
367 a0c9f010 Michael Hanselmann
  # init of cluster config file
368 b9eeeb02 Michael Hanselmann
  cluster_config = objects.Cluster(
369 b9eeeb02 Michael Hanselmann
370 b9eeeb02 Michael Hanselmann
371 b9eeeb02 Michael Hanselmann
    highest_used_port=(constants.FIRST_DRBD_PORT - 1),
372 b9eeeb02 Michael Hanselmann
373 b9eeeb02 Michael Hanselmann
374 b9eeeb02 Michael Hanselmann
375 f6bd6e98 Michael Hanselmann,
376 f6bd6e98 Michael Hanselmann
377 f6bd6e98 Michael Hanselmann
378 f6bd6e98 Michael Hanselmann,
379 f6bd6e98 Michael Hanselmann
380 ea3a925f Alexander Schreiber
381 4ef7f423 Guido Trotter
    beparams={constants.PP_DEFAULT: beparams},
382 b6a30b0d Guido Trotter
    nicparams={constants.PP_DEFAULT: nicparams},
383 ea3a925f Alexander Schreiber
384 ce735215 Guido Trotter
385 022c3a0b Guido Trotter
386 b989b9d9 Ken Wehr
387 39b0f0c2 Balazs Lecz
388 430b923c Iustin Pop
389 430b923c Iustin Pop
390 430b923c Iustin Pop
391 3953242f Iustin Pop
392 a721e23a Luca Bigliardi
393 bf4af505 Apollon Oikonomopoulos
394 2f20d07b Manuel Franceschini,
395 b9eeeb02 Michael Hanselmann
396 b9eeeb02 Michael Hanselmann
  master_node_config = objects.Node(,
397 b9eeeb02 Michael Hanselmann
398 b9222f32 Guido Trotter
399 c044f32c Guido Trotter
400 c044f32c Guido Trotter
401 af64c0ea Iustin Pop
                                    offline=False, drained=False,
402 c044f32c Guido Trotter
403 9e1333b9 Guido Trotter
  InitConfig(constants.CONFIG_VERSION, cluster_config, master_node_config)
404 d367b66c Manuel Franceschini
  cfg = config.ConfigWriter(offline=True)
405 9e1333b9 Guido Trotter
  ssh.WriteKnownHostsFile(cfg, constants.SSH_KNOWN_HOSTS_FILE)
406 a4eae71f Michael Hanselmann
  cfg.Update(cfg.GetClusterInfo(), logging.error)
407 d367b66c Manuel Franceschini
408 d367b66c Manuel Franceschini
409 d367b66c Manuel Franceschini
  # set up the inter-node password and certificate
410 d367b66c Manuel Franceschini
411 827f753e Guido Trotter
412 b3f1cf6f Iustin Pop
  # start the master ip
413 b3f1cf6f Iustin Pop
  # TODO: Review rpc call from bootstrap
414 b726aff0 Iustin Pop
  # TODO: Warn on failed start master
415 3583908a Guido Trotter
  rpc.RpcRunner.call_node_start_master(, True, False)
416 b3f1cf6f Iustin Pop
417 b1b6ea87 Iustin Pop
418 02f99608 Oleksiy Mishchenko
def InitConfig(version, cluster_config, master_node_config,
419 02f99608 Oleksiy Mishchenko
420 7b3a8fb5 Iustin Pop
  """Create the initial cluster configuration.
421 7b3a8fb5 Iustin Pop

422 7b3a8fb5 Iustin Pop
  It will contain the current node, which will also be the master
423 7b3a8fb5 Iustin Pop
  node, and no instances.
424 7b3a8fb5 Iustin Pop

425 7b3a8fb5 Iustin Pop
  @type version: int
426 c41eea6e Iustin Pop
  @param version: configuration version
427 c41eea6e Iustin Pop
  @type cluster_config: L{objects.Cluster}
428 c41eea6e Iustin Pop
  @param cluster_config: cluster configuration
429 c41eea6e Iustin Pop
  @type master_node_config: L{objects.Node}
430 c41eea6e Iustin Pop
  @param master_node_config: master node configuration
431 c41eea6e Iustin Pop
  @type cfg_file: string
432 c41eea6e Iustin Pop
  @param cfg_file: configuration file path
433 c41eea6e Iustin Pop

434 7b3a8fb5 Iustin Pop
435 7b3a8fb5 Iustin Pop
  nodes = {
436 7b3a8fb5 Iustin Pop master_node_config,
437 7b3a8fb5 Iustin Pop
438 7b3a8fb5 Iustin Pop
439 d693c864 Iustin Pop
  now = time.time()
440 7b3a8fb5 Iustin Pop
  config_data = objects.ConfigData(version=version,
441 7b3a8fb5 Iustin Pop
442 7b3a8fb5 Iustin Pop
443 7b3a8fb5 Iustin Pop
444 d693c864 Iustin Pop
445 d693c864 Iustin Pop
                                   ctime=now, mtime=now)
446 a33848a5 Guido Trotter
447 a33848a5 Guido Trotter
448 a33848a5 Guido Trotter
449 02f99608 Oleksiy Mishchenko
450 02f99608 Oleksiy Mishchenko
451 140aa4a8 Iustin Pop
def FinalizeClusterDestroy(master):
452 140aa4a8 Iustin Pop
  """Execute the last steps of cluster destroy
453 140aa4a8 Iustin Pop

454 140aa4a8 Iustin Pop
  This function shuts down all the daemons, completing the destroy
455 140aa4a8 Iustin Pop
  begun in cmdlib.LUDestroyOpcode.
456 140aa4a8 Iustin Pop

457 140aa4a8 Iustin Pop
458 b989b9d9 Ken Wehr
  cfg = config.ConfigWriter()
459 b989b9d9 Ken Wehr
  modify_ssh_setup = cfg.GetClusterInfo().modify_ssh_setup
460 781de953 Iustin Pop
  result = rpc.RpcRunner.call_node_stop_master(master, True)
461 3cebe102 Michael Hanselmann
  msg = result.fail_msg
462 6c00d19a Iustin Pop
  if msg:
463 099c52ad Iustin Pop
    logging.warning("Could not disable the master role: %s", msg)
464 b989b9d9 Ken Wehr
  result = rpc.RpcRunner.call_node_leave_cluster(master, modify_ssh_setup)
465 3cebe102 Michael Hanselmann
  msg = result.fail_msg
466 0623d351 Iustin Pop
  if msg:
467 0623d351 Iustin Pop
    logging.warning("Could not shutdown the node daemon and cleanup"
468 0623d351 Iustin Pop
                    " the node: %s", msg)
469 140aa4a8 Iustin Pop
470 140aa4a8 Iustin Pop
471 87622829 Iustin Pop
def SetupNodeDaemon(cluster_name, node, ssh_key_check):
472 827f753e Guido Trotter
  """Add a node to the cluster.
473 827f753e Guido Trotter

474 b1b6ea87 Iustin Pop
  This function must be called before the actual opcode, and will ssh
475 b1b6ea87 Iustin Pop
  to the remote node, copy the needed files, and start ganeti-noded,
476 b1b6ea87 Iustin Pop
  allowing the master to do the rest via normal rpc calls.
477 827f753e Guido Trotter

478 87622829 Iustin Pop
  @param cluster_name: the cluster name
479 87622829 Iustin Pop
  @param node: the name of the new node
480 87622829 Iustin Pop
  @param ssh_key_check: whether to do a strict key check
481 827f753e Guido Trotter

482 827f753e Guido Trotter
483 b43dcc5a Manuel Franceschini
  family = ssconf.SimpleStore().GetPrimaryIPFamily()
484 b43dcc5a Manuel Franceschini
  sshrunner = ssh.SshRunner(cluster_name,
485 b43dcc5a Manuel Franceschini
486 5557b04c Michael Hanselmann
487 168c1de2 Michael Hanselmann
  noded_cert = utils.ReadFile(constants.NODED_CERT_FILE)
488 2438c157 Michael Hanselmann
  rapi_cert = utils.ReadFile(constants.RAPI_CERT_FILE)
489 6b7d5878 Michael Hanselmann
  confd_hmac_key = utils.ReadFile(constants.CONFD_HMAC_KEY)
490 5557b04c Michael Hanselmann
491 827f753e Guido Trotter
  # in the base64 pem encoding, neither '!' nor '.' are valid chars,
492 827f753e Guido Trotter
  # so we use this to detect an invalid certificate; as long as the
493 827f753e Guido Trotter
  # cert doesn't contain this, the here-document will be correctly
494 77b076ca Guido Trotter
  # parsed by the shell sequence below. HMAC keys are hexadecimal strings,
495 77b076ca Guido Trotter
  # so the same restrictions apply.
496 6b7d5878 Michael Hanselmann
  for content in (noded_cert, rapi_cert, confd_hmac_key):
497 77b076ca Guido Trotter
    if'^!EOF\.', content, re.MULTILINE):
498 77b076ca Guido Trotter
      raise errors.OpExecError("invalid SSL certificate or HMAC key")
499 5557b04c Michael Hanselmann
500 5557b04c Michael Hanselmann
  if not noded_cert.endswith("\n"):
501 5557b04c Michael Hanselmann
    noded_cert += "\n"
502 2438c157 Michael Hanselmann
  if not rapi_cert.endswith("\n"):
503 2438c157 Michael Hanselmann
    rapi_cert += "\n"
504 6b7d5878 Michael Hanselmann
  if not confd_hmac_key.endswith("\n"):
505 6b7d5878 Michael Hanselmann
    confd_hmac_key += "\n"
506 827f753e Guido Trotter
507 b43dcc5a Manuel Franceschini
  bind_address = constants.IP4_ADDRESS_ANY
508 b43dcc5a Manuel Franceschini
  if family ==
509 b43dcc5a Manuel Franceschini
    bind_address = constants.IP6_ADDRESS_ANY
510 b43dcc5a Manuel Franceschini
511 827f753e Guido Trotter
  # set up inter-node password and certificate and restarts the node daemon
512 827f753e Guido Trotter
  # and then connect with ssh to set password and start ganeti-noded
513 827f753e Guido Trotter
  # note that all the below variables are sanitized at this point,
514 827f753e Guido Trotter
  # either by being constants or by the checks above
515 3db3eb2a Michael Hanselmann
  # TODO: Could this command exceed a shell's maximum command length?
516 827f753e Guido Trotter
  mycommand = ("umask 077 && "
517 827f753e Guido Trotter
               "cat > '%s' << '!EOF.' && \n"
518 2438c157 Michael Hanselmann
519 2438c157 Michael Hanselmann
               "cat > '%s' << '!EOF.' && \n"
520 2438c157 Michael Hanselmann
521 77b076ca Guido Trotter
               "cat > '%s' << '!EOF.' && \n"
522 77b076ca Guido Trotter
523 77b076ca Guido Trotter
               "chmod 0400 %s %s %s && "
524 b43dcc5a Manuel Franceschini
               "%s start %s -b '%s'" %
525 168c1de2 Michael Hanselmann
               (constants.NODED_CERT_FILE, noded_cert,
526 2438c157 Michael Hanselmann
                constants.RAPI_CERT_FILE, rapi_cert,
527 6b7d5878 Michael Hanselmann
                constants.CONFD_HMAC_KEY, confd_hmac_key,
528 168c1de2 Michael Hanselmann
                constants.NODED_CERT_FILE, constants.RAPI_CERT_FILE,
529 6b7d5878 Michael Hanselmann
530 b43dcc5a Manuel Franceschini
                constants.DAEMON_UTIL, constants.NODED, bind_address))
531 827f753e Guido Trotter
532 c4b6c29c Michael Hanselmann
  result = sshrunner.Run(node, 'root', mycommand, batch=False,
533 c4b6c29c Michael Hanselmann
534 c4b6c29c Michael Hanselmann
535 c4b6c29c Michael Hanselmann
536 827f753e Guido Trotter
  if result.failed:
537 827f753e Guido Trotter
    raise errors.OpExecError("Remote command on node %s, error: %s,"
538 827f753e Guido Trotter
                             " output: %s" %
539 827f753e Guido Trotter
                             (node, result.fail_reason, result.output))
540 827f753e Guido Trotter
541 5627f375 Michael Hanselmann
542 5627f375 Michael Hanselmann
543 b1b6ea87 Iustin Pop
544 8e2524c3 Guido Trotter
def MasterFailover(no_voting=False):
545 b1b6ea87 Iustin Pop
  """Failover the master node.
546 b1b6ea87 Iustin Pop

547 b1b6ea87 Iustin Pop
  This checks that we are not already the master, and will cause the
548 b1b6ea87 Iustin Pop
  current master to cease being master, and the non-master to become
549 b1b6ea87 Iustin Pop
  new master.
550 b1b6ea87 Iustin Pop

551 8e2524c3 Guido Trotter
  @type no_voting: boolean
552 8e2524c3 Guido Trotter
  @param no_voting: force the operation without remote nodes agreement
553 8e2524c3 Guido Trotter
554 8e2524c3 Guido Trotter

555 b1b6ea87 Iustin Pop
556 8135a2db Iustin Pop
  sstore = ssconf.SimpleStore()
557 b1b6ea87 Iustin Pop
558 8135a2db Iustin Pop
  old_master, new_master = ssconf.GetMasterAndMyself(sstore)
559 8135a2db Iustin Pop
  node_list = sstore.GetNodeList()
560 8135a2db Iustin Pop
  mc_list = sstore.GetMasterCandidates()
561 b1b6ea87 Iustin Pop
562 b1b6ea87 Iustin Pop
  if old_master == new_master:
563 b1b6ea87 Iustin Pop
    raise errors.OpPrereqError("This commands must be run on the node"
564 b1b6ea87 Iustin Pop
                               " where you want the new master to be."
565 b1b6ea87 Iustin Pop
                               " %s is already the master" %
566 debac808 Iustin Pop
                               old_master, errors.ECODE_INVAL)
567 d5927e48 Iustin Pop
568 8135a2db Iustin Pop
  if new_master not in mc_list:
569 8135a2db Iustin Pop
    mc_no_master = [name for name in mc_list if name != old_master]
570 8135a2db Iustin Pop
    raise errors.OpPrereqError("This node is not among the nodes marked"
571 8135a2db Iustin Pop
                               " as master candidates. Only these nodes"
572 8135a2db Iustin Pop
                               " can become masters. Current list of"
573 8135a2db Iustin Pop
                               " master candidates is:\n"
574 debac808 Iustin Pop
                               "%s" % ('\n'.join(mc_no_master)),
575 debac808 Iustin Pop
576 8135a2db Iustin Pop
577 8e2524c3 Guido Trotter
  if not no_voting:
578 8e2524c3 Guido Trotter
    vote_list = GatherMasterVotes(node_list)
579 8e2524c3 Guido Trotter
580 8e2524c3 Guido Trotter
    if vote_list:
581 8e2524c3 Guido Trotter
      voted_master = vote_list[0][0]
582 8e2524c3 Guido Trotter
      if voted_master is None:
583 8e2524c3 Guido Trotter
        raise errors.OpPrereqError("Cluster is inconsistent, most nodes did"
584 debac808 Iustin Pop
                                   " not respond.", errors.ECODE_ENVIRON)
585 8e2524c3 Guido Trotter
      elif voted_master != old_master:
586 8e2524c3 Guido Trotter
        raise errors.OpPrereqError("I have a wrong configuration, I believe"
587 8e2524c3 Guido Trotter
                                   " the master is %s but the other nodes"
588 8e2524c3 Guido Trotter
                                   " voted %s. Please resync the configuration"
589 8e2524c3 Guido Trotter
                                   " of this node." %
590 debac808 Iustin Pop
                                   (old_master, voted_master),
591 debac808 Iustin Pop
592 b1b6ea87 Iustin Pop
  # end checks
593 b1b6ea87 Iustin Pop
594 b1b6ea87 Iustin Pop
  rcode = 0
595 b1b6ea87 Iustin Pop
596 d5927e48 Iustin Pop"Setting master to %s, old master: %s", new_master, old_master)
597 b1b6ea87 Iustin Pop
598 781de953 Iustin Pop
  result = rpc.RpcRunner.call_node_stop_master(old_master, True)
599 3cebe102 Michael Hanselmann
  msg = result.fail_msg
600 6c00d19a Iustin Pop
  if msg:
601 d5927e48 Iustin Pop
    logging.error("Could not disable the master role on the old master"
602 6c00d19a Iustin Pop
                 " %s, please disable manually: %s", old_master, msg)
603 b1b6ea87 Iustin Pop
604 425f0f54 Iustin Pop
  master_ip = sstore.GetMasterIP()
605 425f0f54 Iustin Pop
  total_timeout = 30
606 d23ef431 Michael Hanselmann
  # Here we have a phase where no master should be running
607 425f0f54 Iustin Pop
  def _check_ip():
608 a744b676 Manuel Franceschini
    if netutils.TcpPing(master_ip, constants.DEFAULT_NODED_PORT):
609 425f0f54 Iustin Pop
      raise utils.RetryAgain()
610 425f0f54 Iustin Pop
611 425f0f54 Iustin Pop
612 425f0f54 Iustin Pop
    utils.Retry(_check_ip, (1, 1.5, 5), total_timeout)
613 425f0f54 Iustin Pop
  except utils.RetryTimeout:
614 425f0f54 Iustin Pop
    logging.warning("The master IP is still reachable after %s seconds,"
615 425f0f54 Iustin Pop
                    " continuing but activating the master on the current"
616 425f0f54 Iustin Pop
                    " node will probably fail", total_timeout)
617 b1b6ea87 Iustin Pop
618 bbe19c17 Iustin Pop
  # instantiate a real config writer, as we now know we have the
619 bbe19c17 Iustin Pop
  # configuration data
620 bbe19c17 Iustin Pop
  cfg = config.ConfigWriter()
621 b1b6ea87 Iustin Pop
622 bbe19c17 Iustin Pop
  cluster_info = cfg.GetClusterInfo()
623 bbe19c17 Iustin Pop
  cluster_info.master_node = new_master
624 bbe19c17 Iustin Pop
  # this will also regenerate the ssconf files, since we updated the
625 bbe19c17 Iustin Pop
  # cluster info
626 a4eae71f Michael Hanselmann
  cfg.Update(cluster_info, logging.error)
627 d5927e48 Iustin Pop
628 3583908a Guido Trotter
  result = rpc.RpcRunner.call_node_start_master(new_master, True, no_voting)
629 3cebe102 Michael Hanselmann
  msg = result.fail_msg
630 b726aff0 Iustin Pop
  if msg:
631 d5927e48 Iustin Pop
    logging.error("Could not start the master role on the new master"
632 b726aff0 Iustin Pop
                  " %s, please check: %s", new_master, msg)
633 b1b6ea87 Iustin Pop
    rcode = 1
634 b1b6ea87 Iustin Pop
635 b1b6ea87 Iustin Pop
  return rcode
636 d7cdb55d Iustin Pop
637 d7cdb55d Iustin Pop
638 8eb148ae Iustin Pop
def GetMaster():
639 8eb148ae Iustin Pop
  """Returns the current master node.
640 8eb148ae Iustin Pop

641 8eb148ae Iustin Pop
  This is a separate function in bootstrap since it's needed by
642 8eb148ae Iustin Pop
  gnt-cluster, and instead of importing directly ssconf, it's better
643 8eb148ae Iustin Pop
  to abstract it in bootstrap, where we do use ssconf in other
644 8eb148ae Iustin Pop
  functions too.
645 8eb148ae Iustin Pop

646 8eb148ae Iustin Pop
647 8eb148ae Iustin Pop
  sstore = ssconf.SimpleStore()
648 8eb148ae Iustin Pop
649 8eb148ae Iustin Pop
  old_master, _ = ssconf.GetMasterAndMyself(sstore)
650 8eb148ae Iustin Pop
651 8eb148ae Iustin Pop
  return old_master
652 8eb148ae Iustin Pop
653 8eb148ae Iustin Pop
654 d7cdb55d Iustin Pop
def GatherMasterVotes(node_list):
655 d7cdb55d Iustin Pop
  """Check the agreement on who is the master.
656 d7cdb55d Iustin Pop

657 d7cdb55d Iustin Pop
  This function will return a list of (node, number of votes), ordered
658 d7cdb55d Iustin Pop
  by the number of votes. Errors will be denoted by the key 'None'.
659 d7cdb55d Iustin Pop

660 d7cdb55d Iustin Pop
  Note that the sum of votes is the number of nodes this machine
661 d7cdb55d Iustin Pop
  knows, whereas the number of entries in the list could be different
662 d7cdb55d Iustin Pop
  (if some nodes vote for another master).
663 d7cdb55d Iustin Pop

664 d7cdb55d Iustin Pop
  We remove ourselves from the list since we know that (bugs aside)
665 d7cdb55d Iustin Pop
  since we use the same source for configuration information for both
666 d7cdb55d Iustin Pop
  backend and boostrap, we'll always vote for ourselves.
667 d7cdb55d Iustin Pop

668 d7cdb55d Iustin Pop
  @type node_list: list
669 d7cdb55d Iustin Pop
  @param node_list: the list of nodes to query for master info; the current
670 5bbd3f7f Michael Hanselmann
      node will be removed if it is in the list
671 d7cdb55d Iustin Pop
  @rtype: list
672 d7cdb55d Iustin Pop
  @return: list of (node, votes)
673 d7cdb55d Iustin Pop

674 d7cdb55d Iustin Pop
675 b705c7a6 Manuel Franceschini
  myself = netutils.Hostname.GetSysName()
676 d7cdb55d Iustin Pop
677 d7cdb55d Iustin Pop
678 d7cdb55d Iustin Pop
  except ValueError:
679 d7cdb55d Iustin Pop
680 d7cdb55d Iustin Pop
  if not node_list:
681 d7cdb55d Iustin Pop
    # no nodes left (eventually after removing myself)
682 d7cdb55d Iustin Pop
    return []
683 d7cdb55d Iustin Pop
  results = rpc.RpcRunner.call_master_info(node_list)
684 d7cdb55d Iustin Pop
  if not isinstance(results, dict):
685 d7cdb55d Iustin Pop
    # this should not happen (unless internal error in rpc)
686 d7cdb55d Iustin Pop
    logging.critical("Can't complete rpc call, aborting master startup")
687 d7cdb55d Iustin Pop
    return [(None, len(node_list))]
688 d7cdb55d Iustin Pop
  votes = {}
689 d7cdb55d Iustin Pop
  for node in results:
690 781de953 Iustin Pop
    nres = results[node]
691 2a52a064 Iustin Pop
    data = nres.payload
692 3cebe102 Michael Hanselmann
    msg = nres.fail_msg
693 2a52a064 Iustin Pop
    fail = False
694 2a52a064 Iustin Pop
    if msg:
695 2a52a064 Iustin Pop
      logging.warning("Error contacting node %s: %s", node, msg)
696 2a52a064 Iustin Pop
      fail = True
697 2a52a064 Iustin Pop
    elif not isinstance(data, (tuple, list)) or len(data) < 3:
698 2a52a064 Iustin Pop
      logging.warning("Invalid data received from node %s: %s", node, data)
699 2a52a064 Iustin Pop
      fail = True
700 2a52a064 Iustin Pop
    if fail:
701 d7cdb55d Iustin Pop
      if None not in votes:
702 d7cdb55d Iustin Pop
        votes[None] = 0
703 d7cdb55d Iustin Pop
      votes[None] += 1
704 d7cdb55d Iustin Pop
705 781de953 Iustin Pop
    master_node = data[2]
706 d7cdb55d Iustin Pop
    if master_node not in votes:
707 d7cdb55d Iustin Pop
      votes[master_node] = 0
708 d7cdb55d Iustin Pop
    votes[master_node] += 1
709 d7cdb55d Iustin Pop
710 d7cdb55d Iustin Pop
  vote_list = [v for v in votes.items()]
711 d7cdb55d Iustin Pop
  # sort first on number of votes then on name, since we want None
712 d7cdb55d Iustin Pop
  # sorted later if we have the half of the nodes not responding, and
713 d7cdb55d Iustin Pop
  # half voting all for the same master
714 d7cdb55d Iustin Pop
  vote_list.sort(key=lambda x: (x[1], x[0]), reverse=True)
715 d7cdb55d Iustin Pop
716 d7cdb55d Iustin Pop
  return vote_list