Statistics
| Branch: | Tag: | Revision:

root / lib / bootstrap.py @ c1912a48

History | View | Annotate | Download (34.5 kB)

1 a0c9f010 Michael Hanselmann
#
2 a0c9f010 Michael Hanselmann
#
3 a0c9f010 Michael Hanselmann
4 57dc299a Iustin Pop
# Copyright (C) 2006, 2007, 2008, 2010, 2011, 2012 Google Inc.
5 a0c9f010 Michael Hanselmann
#
6 a0c9f010 Michael Hanselmann
# This program is free software; you can redistribute it and/or modify
7 a0c9f010 Michael Hanselmann
# it under the terms of the GNU General Public License as published by
8 a0c9f010 Michael Hanselmann
# the Free Software Foundation; either version 2 of the License, or
9 a0c9f010 Michael Hanselmann
# (at your option) any later version.
10 a0c9f010 Michael Hanselmann
#
11 a0c9f010 Michael Hanselmann
# This program is distributed in the hope that it will be useful, but
12 a0c9f010 Michael Hanselmann
# WITHOUT ANY WARRANTY; without even the implied warranty of
13 a0c9f010 Michael Hanselmann
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 a0c9f010 Michael Hanselmann
# General Public License for more details.
15 a0c9f010 Michael Hanselmann
#
16 a0c9f010 Michael Hanselmann
# You should have received a copy of the GNU General Public License
17 a0c9f010 Michael Hanselmann
# along with this program; if not, write to the Free Software
18 a0c9f010 Michael Hanselmann
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 a0c9f010 Michael Hanselmann
# 02110-1301, USA.
20 a0c9f010 Michael Hanselmann
21 a0c9f010 Michael Hanselmann
22 a0c9f010 Michael Hanselmann
"""Functions to bootstrap a new cluster.
23 a0c9f010 Michael Hanselmann

24 a0c9f010 Michael Hanselmann
"""
25 a0c9f010 Michael Hanselmann
26 a0c9f010 Michael Hanselmann
import os
27 a0c9f010 Michael Hanselmann
import os.path
28 a0c9f010 Michael Hanselmann
import re
29 b1b6ea87 Iustin Pop
import logging
30 d693c864 Iustin Pop
import time
31 a698cdbb Michael Hanselmann
import tempfile
32 a0c9f010 Michael Hanselmann
33 a0c9f010 Michael Hanselmann
from ganeti import rpc
34 a0c9f010 Michael Hanselmann
from ganeti import ssh
35 a0c9f010 Michael Hanselmann
from ganeti import utils
36 a0c9f010 Michael Hanselmann
from ganeti import errors
37 a0c9f010 Michael Hanselmann
from ganeti import config
38 a0c9f010 Michael Hanselmann
from ganeti import constants
39 b9eeeb02 Michael Hanselmann
from ganeti import objects
40 a0c9f010 Michael Hanselmann
from ganeti import ssconf
41 a33848a5 Guido Trotter
from ganeti import serializer
42 a5728081 Guido Trotter
from ganeti import hypervisor
43 a721e23a Luca Bigliardi
from ganeti import bdev
44 a744b676 Manuel Franceschini
from ganeti import netutils
45 3b6b6129 Michael Hanselmann
from ganeti import luxi
46 ff699aa9 Michael Hanselmann
from ganeti import jstore
47 7ede9c6a Michael Hanselmann
from ganeti import pathutils
48 3b6b6129 Michael Hanselmann
49 a0c9f010 Michael Hanselmann
50 88b92fe3 Guido Trotter
# ec_id for InitConfig's temporary reservation manager
51 88b92fe3 Guido Trotter
_INITCONF_ECID = "initconfig-ecid"
52 88b92fe3 Guido Trotter
53 3b6b6129 Michael Hanselmann
#: After how many seconds daemon must be responsive
54 3b6b6129 Michael Hanselmann
_DAEMON_READY_TIMEOUT = 10.0
55 3b6b6129 Michael Hanselmann
56 e38220e4 Michael Hanselmann
57 531baf8e Iustin Pop
def _InitSSHSetup():
58 a0c9f010 Michael Hanselmann
  """Setup the SSH configuration for the cluster.
59 a0c9f010 Michael Hanselmann

60 a0c9f010 Michael Hanselmann
  This generates a dsa keypair for root, adds the pub key to the
61 a0c9f010 Michael Hanselmann
  permitted hosts and adds the hostkey to its own known hosts.
62 a0c9f010 Michael Hanselmann

63 a0c9f010 Michael Hanselmann
  """
64 052783ff Michael Hanselmann
  priv_key, pub_key, auth_keys = ssh.GetUserFiles(constants.SSH_LOGIN_USER)
65 a0c9f010 Michael Hanselmann
66 a0c9f010 Michael Hanselmann
  for name in priv_key, pub_key:
67 a0c9f010 Michael Hanselmann
    if os.path.exists(name):
68 a0c9f010 Michael Hanselmann
      utils.CreateBackup(name)
69 a0c9f010 Michael Hanselmann
    utils.RemoveFile(name)
70 a0c9f010 Michael Hanselmann
71 a0c9f010 Michael Hanselmann
  result = utils.RunCmd(["ssh-keygen", "-t", "dsa",
72 a0c9f010 Michael Hanselmann
                         "-f", priv_key,
73 a0c9f010 Michael Hanselmann
                         "-q", "-N", ""])
74 a0c9f010 Michael Hanselmann
  if result.failed:
75 a0c9f010 Michael Hanselmann
    raise errors.OpExecError("Could not generate ssh keypair, error %s" %
76 a0c9f010 Michael Hanselmann
                             result.output)
77 a0c9f010 Michael Hanselmann
78 7a0156dc Luca Bigliardi
  utils.AddAuthorizedKey(auth_keys, utils.ReadFile(pub_key))
79 a0c9f010 Michael Hanselmann
80 a0c9f010 Michael Hanselmann
81 c008906b Michael Hanselmann
def GenerateHmacKey(file_name):
82 c008906b Michael Hanselmann
  """Writes a new HMAC key.
83 c008906b Michael Hanselmann

84 c008906b Michael Hanselmann
  @type file_name: str
85 c008906b Michael Hanselmann
  @param file_name: Path to output file
86 c008906b Michael Hanselmann

87 c008906b Michael Hanselmann
  """
88 43575108 Michael Hanselmann
  utils.WriteFile(file_name, data="%s\n" % utils.GenerateSecret(), mode=0400,
89 43575108 Michael Hanselmann
                  backup=True)
90 43575108 Michael Hanselmann
91 43575108 Michael Hanselmann
92 b6267745 Andrea Spadaccini
def GenerateClusterCrypto(new_cluster_cert, new_rapi_cert, new_spice_cert,
93 b6267745 Andrea Spadaccini
                          new_confd_hmac_key, new_cds,
94 b6267745 Andrea Spadaccini
                          rapi_cert_pem=None, spice_cert_pem=None,
95 b6267745 Andrea Spadaccini
                          spice_cacert_pem=None, cds=None,
96 7ede9c6a Michael Hanselmann
                          nodecert_file=pathutils.NODED_CERT_FILE,
97 7ede9c6a Michael Hanselmann
                          rapicert_file=pathutils.RAPI_CERT_FILE,
98 7ede9c6a Michael Hanselmann
                          spicecert_file=pathutils.SPICE_CERT_FILE,
99 7ede9c6a Michael Hanselmann
                          spicecacert_file=pathutils.SPICE_CACERT_FILE,
100 7ede9c6a Michael Hanselmann
                          hmackey_file=pathutils.CONFD_HMAC_KEY,
101 7ede9c6a Michael Hanselmann
                          cds_file=pathutils.CLUSTER_DOMAIN_SECRET_FILE):
102 43575108 Michael Hanselmann
  """Updates the cluster certificates, keys and secrets.
103 43575108 Michael Hanselmann

104 43575108 Michael Hanselmann
  @type new_cluster_cert: bool
105 43575108 Michael Hanselmann
  @param new_cluster_cert: Whether to generate a new cluster certificate
106 43575108 Michael Hanselmann
  @type new_rapi_cert: bool
107 43575108 Michael Hanselmann
  @param new_rapi_cert: Whether to generate a new RAPI certificate
108 b6267745 Andrea Spadaccini
  @type new_spice_cert: bool
109 b6267745 Andrea Spadaccini
  @param new_spice_cert: Whether to generate a new SPICE certificate
110 6b7d5878 Michael Hanselmann
  @type new_confd_hmac_key: bool
111 6b7d5878 Michael Hanselmann
  @param new_confd_hmac_key: Whether to generate a new HMAC key
112 3db3eb2a Michael Hanselmann
  @type new_cds: bool
113 3db3eb2a Michael Hanselmann
  @param new_cds: Whether to generate a new cluster domain secret
114 43575108 Michael Hanselmann
  @type rapi_cert_pem: string
115 43575108 Michael Hanselmann
  @param rapi_cert_pem: New RAPI certificate in PEM format
116 b6267745 Andrea Spadaccini
  @type spice_cert_pem: string
117 b6267745 Andrea Spadaccini
  @param spice_cert_pem: New SPICE certificate in PEM format
118 b6267745 Andrea Spadaccini
  @type spice_cacert_pem: string
119 b6267745 Andrea Spadaccini
  @param spice_cacert_pem: Certificate of the CA that signed the SPICE
120 b6267745 Andrea Spadaccini
                           certificate, in PEM format
121 3db3eb2a Michael Hanselmann
  @type cds: string
122 3db3eb2a Michael Hanselmann
  @param cds: New cluster domain secret
123 aeefe835 Iustin Pop
  @type nodecert_file: string
124 aeefe835 Iustin Pop
  @param nodecert_file: optional override of the node cert file path
125 aeefe835 Iustin Pop
  @type rapicert_file: string
126 aeefe835 Iustin Pop
  @param rapicert_file: optional override of the rapi cert file path
127 b6267745 Andrea Spadaccini
  @type spicecert_file: string
128 b6267745 Andrea Spadaccini
  @param spicecert_file: optional override of the spice cert file path
129 b6267745 Andrea Spadaccini
  @type spicecacert_file: string
130 b6267745 Andrea Spadaccini
  @param spicecacert_file: optional override of the spice CA cert file path
131 aeefe835 Iustin Pop
  @type hmackey_file: string
132 aeefe835 Iustin Pop
  @param hmackey_file: optional override of the hmac key file path
133 43575108 Michael Hanselmann

134 43575108 Michael Hanselmann
  """
135 168c1de2 Michael Hanselmann
  # noded SSL certificate
136 aeefe835 Iustin Pop
  cluster_cert_exists = os.path.exists(nodecert_file)
137 43575108 Michael Hanselmann
  if new_cluster_cert or not cluster_cert_exists:
138 43575108 Michael Hanselmann
    if cluster_cert_exists:
139 aeefe835 Iustin Pop
      utils.CreateBackup(nodecert_file)
140 43575108 Michael Hanselmann
141 aeefe835 Iustin Pop
    logging.debug("Generating new cluster certificate at %s", nodecert_file)
142 af2ae1c0 Iustin Pop
    utils.GenerateSelfSignedSslCert(nodecert_file)
143 43575108 Michael Hanselmann
144 6b7d5878 Michael Hanselmann
  # confd HMAC key
145 aeefe835 Iustin Pop
  if new_confd_hmac_key or not os.path.exists(hmackey_file):
146 aeefe835 Iustin Pop
    logging.debug("Writing new confd HMAC key to %s", hmackey_file)
147 aeefe835 Iustin Pop
    GenerateHmacKey(hmackey_file)
148 43575108 Michael Hanselmann
149 43575108 Michael Hanselmann
  # RAPI
150 aeefe835 Iustin Pop
  rapi_cert_exists = os.path.exists(rapicert_file)
151 43575108 Michael Hanselmann
152 43575108 Michael Hanselmann
  if rapi_cert_pem:
153 43575108 Michael Hanselmann
    # Assume rapi_pem contains a valid PEM-formatted certificate and key
154 aeefe835 Iustin Pop
    logging.debug("Writing RAPI certificate at %s", rapicert_file)
155 aeefe835 Iustin Pop
    utils.WriteFile(rapicert_file, data=rapi_cert_pem, backup=True)
156 43575108 Michael Hanselmann
157 43575108 Michael Hanselmann
  elif new_rapi_cert or not rapi_cert_exists:
158 43575108 Michael Hanselmann
    if rapi_cert_exists:
159 aeefe835 Iustin Pop
      utils.CreateBackup(rapicert_file)
160 43575108 Michael Hanselmann
161 aeefe835 Iustin Pop
    logging.debug("Generating new RAPI certificate at %s", rapicert_file)
162 af2ae1c0 Iustin Pop
    utils.GenerateSelfSignedSslCert(rapicert_file)
163 c008906b Michael Hanselmann
164 b6267745 Andrea Spadaccini
  # SPICE
165 b6267745 Andrea Spadaccini
  spice_cert_exists = os.path.exists(spicecert_file)
166 b6267745 Andrea Spadaccini
  spice_cacert_exists = os.path.exists(spicecacert_file)
167 b6267745 Andrea Spadaccini
  if spice_cert_pem:
168 b6267745 Andrea Spadaccini
    # spice_cert_pem implies also spice_cacert_pem
169 b6267745 Andrea Spadaccini
    logging.debug("Writing SPICE certificate at %s", spicecert_file)
170 b6267745 Andrea Spadaccini
    utils.WriteFile(spicecert_file, data=spice_cert_pem, backup=True)
171 b6267745 Andrea Spadaccini
    logging.debug("Writing SPICE CA certificate at %s", spicecacert_file)
172 b6267745 Andrea Spadaccini
    utils.WriteFile(spicecacert_file, data=spice_cacert_pem, backup=True)
173 b6267745 Andrea Spadaccini
  elif new_spice_cert or not spice_cert_exists:
174 b6267745 Andrea Spadaccini
    if spice_cert_exists:
175 b6267745 Andrea Spadaccini
      utils.CreateBackup(spicecert_file)
176 b6267745 Andrea Spadaccini
    if spice_cacert_exists:
177 b6267745 Andrea Spadaccini
      utils.CreateBackup(spicecacert_file)
178 b6267745 Andrea Spadaccini
179 b6267745 Andrea Spadaccini
    logging.debug("Generating new self-signed SPICE certificate at %s",
180 b6267745 Andrea Spadaccini
                  spicecert_file)
181 b6267745 Andrea Spadaccini
    (_, cert_pem) = utils.GenerateSelfSignedSslCert(spicecert_file)
182 b6267745 Andrea Spadaccini
183 b6267745 Andrea Spadaccini
    # Self-signed certificate -> the public certificate is also the CA public
184 b6267745 Andrea Spadaccini
    # certificate
185 b6267745 Andrea Spadaccini
    logging.debug("Writing the public certificate to %s",
186 b6267745 Andrea Spadaccini
                  spicecert_file)
187 b6267745 Andrea Spadaccini
    utils.io.WriteFile(spicecacert_file, mode=0400, data=cert_pem)
188 b6267745 Andrea Spadaccini
189 3db3eb2a Michael Hanselmann
  # Cluster domain secret
190 3db3eb2a Michael Hanselmann
  if cds:
191 fc0726b9 Michael Hanselmann
    logging.debug("Writing cluster domain secret to %s", cds_file)
192 fc0726b9 Michael Hanselmann
    utils.WriteFile(cds_file, data=cds, backup=True)
193 fc0726b9 Michael Hanselmann
194 fc0726b9 Michael Hanselmann
  elif new_cds or not os.path.exists(cds_file):
195 fc0726b9 Michael Hanselmann
    logging.debug("Generating new cluster domain secret at %s", cds_file)
196 fc0726b9 Michael Hanselmann
    GenerateHmacKey(cds_file)
197 3db3eb2a Michael Hanselmann
198 c008906b Michael Hanselmann
199 8f215968 Michael Hanselmann
def _InitGanetiServerSetup(master_name):
200 40a97d80 Michael Hanselmann
  """Setup the necessary configuration for the initial node daemon.
201 40a97d80 Michael Hanselmann

202 40a97d80 Michael Hanselmann
  This creates the nodepass file containing the shared password for
203 600535f0 Manuel Franceschini
  the cluster, generates the SSL certificate and starts the node daemon.
204 600535f0 Manuel Franceschini

205 600535f0 Manuel Franceschini
  @type master_name: str
206 600535f0 Manuel Franceschini
  @param master_name: Name of the master node
207 40a97d80 Michael Hanselmann

208 40a97d80 Michael Hanselmann
  """
209 43575108 Michael Hanselmann
  # Generate cluster secrets
210 b6267745 Andrea Spadaccini
  GenerateClusterCrypto(True, False, False, False, False)
211 4a34c5cf Guido Trotter
212 7ede9c6a Michael Hanselmann
  result = utils.RunCmd([pathutils.DAEMON_UTIL, "start", constants.NODED])
213 a0c9f010 Michael Hanselmann
  if result.failed:
214 a0c9f010 Michael Hanselmann
    raise errors.OpExecError("Could not start the node daemon, command %s"
215 a0c9f010 Michael Hanselmann
                             " had exitcode %s and error %s" %
216 a0c9f010 Michael Hanselmann
                             (result.cmd, result.exit_code, result.output))
217 a0c9f010 Michael Hanselmann
218 5627f375 Michael Hanselmann
  _WaitForNodeDaemon(master_name)
219 5627f375 Michael Hanselmann
220 5627f375 Michael Hanselmann
221 5627f375 Michael Hanselmann
def _WaitForNodeDaemon(node_name):
222 5627f375 Michael Hanselmann
  """Wait for node daemon to become responsive.
223 5627f375 Michael Hanselmann

224 5627f375 Michael Hanselmann
  """
225 d3833ebd Michael Hanselmann
  def _CheckNodeDaemon():
226 bd6d1202 Renรฉ Nussbaumer
    # Pylint bug <http://www.logilab.org/ticket/35642>
227 bd6d1202 Renรฉ Nussbaumer
    # pylint: disable=E1101
228 db04ce5d Michael Hanselmann
    result = rpc.BootstrapRunner().call_version([node_name])[node_name]
229 d3833ebd Michael Hanselmann
    if result.fail_msg:
230 d3833ebd Michael Hanselmann
      raise utils.RetryAgain()
231 8f215968 Michael Hanselmann
232 d3833ebd Michael Hanselmann
  try:
233 3b6b6129 Michael Hanselmann
    utils.Retry(_CheckNodeDaemon, 1.0, _DAEMON_READY_TIMEOUT)
234 d3833ebd Michael Hanselmann
  except utils.RetryTimeout:
235 5627f375 Michael Hanselmann
    raise errors.OpExecError("Node daemon on %s didn't answer queries within"
236 3b6b6129 Michael Hanselmann
                             " %s seconds" % (node_name, _DAEMON_READY_TIMEOUT))
237 3b6b6129 Michael Hanselmann
238 3b6b6129 Michael Hanselmann
239 3b6b6129 Michael Hanselmann
def _WaitForMasterDaemon():
240 3b6b6129 Michael Hanselmann
  """Wait for master daemon to become responsive.
241 3b6b6129 Michael Hanselmann

242 3b6b6129 Michael Hanselmann
  """
243 3b6b6129 Michael Hanselmann
  def _CheckMasterDaemon():
244 3b6b6129 Michael Hanselmann
    try:
245 3b6b6129 Michael Hanselmann
      cl = luxi.Client()
246 3b6b6129 Michael Hanselmann
      (cluster_name, ) = cl.QueryConfigValues(["cluster_name"])
247 3b6b6129 Michael Hanselmann
    except Exception:
248 3b6b6129 Michael Hanselmann
      raise utils.RetryAgain()
249 3b6b6129 Michael Hanselmann
250 3b6b6129 Michael Hanselmann
    logging.debug("Received cluster name %s from master", cluster_name)
251 3b6b6129 Michael Hanselmann
252 3b6b6129 Michael Hanselmann
  try:
253 3b6b6129 Michael Hanselmann
    utils.Retry(_CheckMasterDaemon, 1.0, _DAEMON_READY_TIMEOUT)
254 3b6b6129 Michael Hanselmann
  except utils.RetryTimeout:
255 3b6b6129 Michael Hanselmann
    raise errors.OpExecError("Master daemon didn't answer queries within"
256 3b6b6129 Michael Hanselmann
                             " %s seconds" % _DAEMON_READY_TIMEOUT)
257 5627f375 Michael Hanselmann
258 a0c9f010 Michael Hanselmann
259 a698cdbb Michael Hanselmann
def RunNodeSetupCmd(cluster_name, node, basecmd, debug, verbose,
260 a698cdbb Michael Hanselmann
                    use_cluster_key, ask_key, strict_host_check, data):
261 a698cdbb Michael Hanselmann
  """Runs a command to configure something on a remote machine.
262 a698cdbb Michael Hanselmann

263 a698cdbb Michael Hanselmann
  @type cluster_name: string
264 a698cdbb Michael Hanselmann
  @param cluster_name: Cluster name
265 a698cdbb Michael Hanselmann
  @type node: string
266 a698cdbb Michael Hanselmann
  @param node: Node name
267 a698cdbb Michael Hanselmann
  @type basecmd: string
268 a698cdbb Michael Hanselmann
  @param basecmd: Base command (path on the remote machine)
269 a698cdbb Michael Hanselmann
  @type debug: bool
270 a698cdbb Michael Hanselmann
  @param debug: Enable debug output
271 a698cdbb Michael Hanselmann
  @type verbose: bool
272 a698cdbb Michael Hanselmann
  @param verbose: Enable verbose output
273 a698cdbb Michael Hanselmann
  @type use_cluster_key: bool
274 a698cdbb Michael Hanselmann
  @param use_cluster_key: See L{ssh.SshRunner.BuildCmd}
275 a698cdbb Michael Hanselmann
  @type ask_key: bool
276 a698cdbb Michael Hanselmann
  @param ask_key: See L{ssh.SshRunner.BuildCmd}
277 a698cdbb Michael Hanselmann
  @type strict_host_check: bool
278 a698cdbb Michael Hanselmann
  @param strict_host_check: See L{ssh.SshRunner.BuildCmd}
279 a698cdbb Michael Hanselmann
  @param data: JSON-serializable input data for script (passed to stdin)
280 a698cdbb Michael Hanselmann

281 a698cdbb Michael Hanselmann
  """
282 a698cdbb Michael Hanselmann
  cmd = [basecmd]
283 a698cdbb Michael Hanselmann
284 a698cdbb Michael Hanselmann
  # Pass --debug/--verbose to the external script if set on our invocation
285 a698cdbb Michael Hanselmann
  if debug:
286 a698cdbb Michael Hanselmann
    cmd.append("--debug")
287 a698cdbb Michael Hanselmann
288 a698cdbb Michael Hanselmann
  if verbose:
289 a698cdbb Michael Hanselmann
    cmd.append("--verbose")
290 a698cdbb Michael Hanselmann
291 e1874aa7 Michael Hanselmann
  family = ssconf.SimpleStore().GetPrimaryIPFamily()
292 e1874aa7 Michael Hanselmann
  srun = ssh.SshRunner(cluster_name,
293 e1874aa7 Michael Hanselmann
                       ipv6=(family == netutils.IP6Address.family))
294 a698cdbb Michael Hanselmann
  scmd = srun.BuildCmd(node, constants.SSH_LOGIN_USER,
295 a698cdbb Michael Hanselmann
                       utils.ShellQuoteArgs(cmd),
296 a698cdbb Michael Hanselmann
                       batch=False, ask_key=ask_key, quiet=False,
297 a698cdbb Michael Hanselmann
                       strict_host_check=strict_host_check,
298 a698cdbb Michael Hanselmann
                       use_cluster_key=use_cluster_key)
299 a698cdbb Michael Hanselmann
300 a698cdbb Michael Hanselmann
  tempfh = tempfile.TemporaryFile()
301 a698cdbb Michael Hanselmann
  try:
302 a698cdbb Michael Hanselmann
    tempfh.write(serializer.DumpJson(data))
303 a698cdbb Michael Hanselmann
    tempfh.seek(0)
304 a698cdbb Michael Hanselmann
305 a698cdbb Michael Hanselmann
    result = utils.RunCmd(scmd, interactive=True, input_fd=tempfh)
306 a698cdbb Michael Hanselmann
  finally:
307 a698cdbb Michael Hanselmann
    tempfh.close()
308 a698cdbb Michael Hanselmann
309 a698cdbb Michael Hanselmann
  if result.failed:
310 a698cdbb Michael Hanselmann
    raise errors.OpExecError("Command '%s' failed: %s" %
311 a698cdbb Michael Hanselmann
                             (result.cmd, result.fail_reason))
312 a698cdbb Michael Hanselmann
313 a698cdbb Michael Hanselmann
314 0e3baaf3 Iustin Pop
def _InitFileStorage(file_storage_dir):
315 0e3baaf3 Iustin Pop
  """Initialize if needed the file storage.
316 0e3baaf3 Iustin Pop

317 0e3baaf3 Iustin Pop
  @param file_storage_dir: the user-supplied value
318 0e3baaf3 Iustin Pop
  @return: either empty string (if file storage was disabled at build
319 0e3baaf3 Iustin Pop
      time) or the normalized path to the storage directory
320 0e3baaf3 Iustin Pop

321 0e3baaf3 Iustin Pop
  """
322 0e3baaf3 Iustin Pop
  file_storage_dir = os.path.normpath(file_storage_dir)
323 0e3baaf3 Iustin Pop
324 0e3baaf3 Iustin Pop
  if not os.path.isabs(file_storage_dir):
325 0376655e Guido Trotter
    raise errors.OpPrereqError("File storage directory '%s' is not an absolute"
326 0376655e Guido Trotter
                               " path" % file_storage_dir, errors.ECODE_INVAL)
327 0e3baaf3 Iustin Pop
328 0e3baaf3 Iustin Pop
  if not os.path.exists(file_storage_dir):
329 0e3baaf3 Iustin Pop
    try:
330 0e3baaf3 Iustin Pop
      os.makedirs(file_storage_dir, 0750)
331 0e3baaf3 Iustin Pop
    except OSError, err:
332 0e3baaf3 Iustin Pop
      raise errors.OpPrereqError("Cannot create file storage directory"
333 0e3baaf3 Iustin Pop
                                 " '%s': %s" % (file_storage_dir, err),
334 0e3baaf3 Iustin Pop
                                 errors.ECODE_ENVIRON)
335 0e3baaf3 Iustin Pop
336 0e3baaf3 Iustin Pop
  if not os.path.isdir(file_storage_dir):
337 0e3baaf3 Iustin Pop
    raise errors.OpPrereqError("The file storage directory '%s' is not"
338 0e3baaf3 Iustin Pop
                               " a directory." % file_storage_dir,
339 0e3baaf3 Iustin Pop
                               errors.ECODE_ENVIRON)
340 0e3baaf3 Iustin Pop
  return file_storage_dir
341 0e3baaf3 Iustin Pop
342 0e3baaf3 Iustin Pop
343 18bb6d28 Agata Murawska
def InitCluster(cluster_name, mac_prefix, # pylint: disable=R0913, R0914
344 5a8648eb Andrea Spadaccini
                master_netmask, master_netdev, file_storage_dir,
345 5a8648eb Andrea Spadaccini
                shared_file_storage_dir, candidate_pool_size, secondary_ip=None,
346 5a8648eb Andrea Spadaccini
                vg_name=None, beparams=None, nicparams=None, ndparams=None,
347 bc5d0215 Andrea Spadaccini
                hvparams=None, diskparams=None, enabled_hypervisors=None,
348 bc5d0215 Andrea Spadaccini
                modify_etc_hosts=True, modify_ssh_setup=True,
349 bc5d0215 Andrea Spadaccini
                maintain_node_health=False, drbd_helper=None, uid_pool=None,
350 18bb6d28 Agata Murawska
                default_iallocator=None, primary_ip_version=None, ipolicy=None,
351 c4929a8b Renรฉ Nussbaumer
                prealloc_wipe_disks=False, use_external_mip_script=False,
352 c4929a8b Renรฉ Nussbaumer
                hv_state=None, disk_state=None):
353 a0c9f010 Michael Hanselmann
  """Initialise the cluster.
354 a0c9f010 Michael Hanselmann

355 ce735215 Guido Trotter
  @type candidate_pool_size: int
356 ce735215 Guido Trotter
  @param candidate_pool_size: master candidate pool size
357 ce735215 Guido Trotter

358 a0c9f010 Michael Hanselmann
  """
359 ce735215 Guido Trotter
  # TODO: complete the docstring
360 a0c9f010 Michael Hanselmann
  if config.ConfigWriter.IsCluster():
361 debac808 Iustin Pop
    raise errors.OpPrereqError("Cluster is already initialised",
362 debac808 Iustin Pop
                               errors.ECODE_STATE)
363 a0c9f010 Michael Hanselmann
364 b119bccb Guido Trotter
  if not enabled_hypervisors:
365 b119bccb Guido Trotter
    raise errors.OpPrereqError("Enabled hypervisors list must contain at"
366 debac808 Iustin Pop
                               " least one member", errors.ECODE_INVAL)
367 b119bccb Guido Trotter
  invalid_hvs = set(enabled_hypervisors) - constants.HYPER_TYPES
368 b119bccb Guido Trotter
  if invalid_hvs:
369 b119bccb Guido Trotter
    raise errors.OpPrereqError("Enabled hypervisors contains invalid"
370 debac808 Iustin Pop
                               " entries: %s" % invalid_hvs,
371 debac808 Iustin Pop
                               errors.ECODE_INVAL)
372 b119bccb Guido Trotter
373 5a8648eb Andrea Spadaccini
  try:
374 5a8648eb Andrea Spadaccini
    ipcls = netutils.IPAddress.GetClassFromIpVersion(primary_ip_version)
375 5a8648eb Andrea Spadaccini
  except errors.ProgrammerError:
376 2f20d07b Manuel Franceschini
    raise errors.OpPrereqError("Invalid primary ip version: %d." %
377 2cfbc784 Iustin Pop
                               primary_ip_version, errors.ECODE_INVAL)
378 2f20d07b Manuel Franceschini
379 2f20d07b Manuel Franceschini
  hostname = netutils.GetHostname(family=ipcls.family)
380 2f20d07b Manuel Franceschini
  if not ipcls.IsValid(hostname.ip):
381 2f20d07b Manuel Franceschini
    raise errors.OpPrereqError("This host's IP (%s) is not a valid IPv%d"
382 2cfbc784 Iustin Pop
                               " address." % (hostname.ip, primary_ip_version),
383 2cfbc784 Iustin Pop
                               errors.ECODE_INVAL)
384 2f20d07b Manuel Franceschini
385 2f20d07b Manuel Franceschini
  if ipcls.IsLoopback(hostname.ip):
386 8b312c1d Manuel Franceschini
    raise errors.OpPrereqError("This host's IP (%s) resolves to a loopback"
387 8b312c1d Manuel Franceschini
                               " address. Please fix DNS or %s." %
388 ee045466 Michael Hanselmann
                               (hostname.ip, pathutils.ETC_HOSTS),
389 debac808 Iustin Pop
                               errors.ECODE_ENVIRON)
390 a0c9f010 Michael Hanselmann
391 2f20d07b Manuel Franceschini
  if not ipcls.Own(hostname.ip):
392 a0c9f010 Michael Hanselmann
    raise errors.OpPrereqError("Inconsistency: this host's name resolves"
393 a0c9f010 Michael Hanselmann
                               " to %s,\nbut this ip address does not"
394 7c4c22f5 Manuel Franceschini
                               " belong to this host" %
395 debac808 Iustin Pop
                               hostname.ip, errors.ECODE_ENVIRON)
396 a0c9f010 Michael Hanselmann
397 2f20d07b Manuel Franceschini
  clustername = netutils.GetHostname(name=cluster_name, family=ipcls.family)
398 a0c9f010 Michael Hanselmann
399 2f20d07b Manuel Franceschini
  if netutils.TcpPing(clustername.ip, constants.DEFAULT_NODED_PORT, timeout=5):
400 7c4c22f5 Manuel Franceschini
    raise errors.OpPrereqError("Cluster IP already active",
401 debac808 Iustin Pop
                               errors.ECODE_NOTUNIQUE)
402 a0c9f010 Michael Hanselmann
403 2f20d07b Manuel Franceschini
  if not secondary_ip:
404 2f20d07b Manuel Franceschini
    if primary_ip_version == constants.IP6_VERSION:
405 2f20d07b Manuel Franceschini
      raise errors.OpPrereqError("When using a IPv6 primary address, a valid"
406 7c4c22f5 Manuel Franceschini
                                 " IPv4 address must be given as secondary",
407 7c4c22f5 Manuel Franceschini
                                 errors.ECODE_INVAL)
408 b9eeeb02 Michael Hanselmann
    secondary_ip = hostname.ip
409 a0c9f010 Michael Hanselmann
410 2f20d07b Manuel Franceschini
  if not netutils.IP4Address.IsValid(secondary_ip):
411 2f20d07b Manuel Franceschini
    raise errors.OpPrereqError("Secondary IP address (%s) has to be a valid"
412 2f20d07b Manuel Franceschini
                               " IPv4 address." % secondary_ip,
413 2f20d07b Manuel Franceschini
                               errors.ECODE_INVAL)
414 2f20d07b Manuel Franceschini
415 2f20d07b Manuel Franceschini
  if not netutils.IP4Address.Own(secondary_ip):
416 2f20d07b Manuel Franceschini
    raise errors.OpPrereqError("You gave %s as secondary IP,"
417 2f20d07b Manuel Franceschini
                               " but it does not belong to this host." %
418 2f20d07b Manuel Franceschini
                               secondary_ip, errors.ECODE_ENVIRON)
419 2f20d07b Manuel Franceschini
420 5a8648eb Andrea Spadaccini
  if master_netmask is not None:
421 5a8648eb Andrea Spadaccini
    if not ipcls.ValidateNetmask(master_netmask):
422 5a8648eb Andrea Spadaccini
      raise errors.OpPrereqError("CIDR netmask (%s) not valid for IPv%s " %
423 2cfbc784 Iustin Pop
                                  (master_netmask, primary_ip_version),
424 2cfbc784 Iustin Pop
                                 errors.ECODE_INVAL)
425 5a8648eb Andrea Spadaccini
  else:
426 5a8648eb Andrea Spadaccini
    master_netmask = ipcls.iplen
427 5a8648eb Andrea Spadaccini
428 a0c9f010 Michael Hanselmann
  if vg_name is not None:
429 a0c9f010 Michael Hanselmann
    # Check if volume group is valid
430 a0c9f010 Michael Hanselmann
    vgstatus = utils.CheckVolumeGroupSize(utils.ListVolumeGroups(), vg_name,
431 a0c9f010 Michael Hanselmann
                                          constants.MIN_VG_SIZE)
432 a0c9f010 Michael Hanselmann
    if vgstatus:
433 a0c9f010 Michael Hanselmann
      raise errors.OpPrereqError("Error: %s\nspecify --no-lvm-storage if"
434 debac808 Iustin Pop
                                 " you are not using lvm" % vgstatus,
435 debac808 Iustin Pop
                                 errors.ECODE_INVAL)
436 a0c9f010 Michael Hanselmann
437 a721e23a Luca Bigliardi
  if drbd_helper is not None:
438 a721e23a Luca Bigliardi
    try:
439 a721e23a Luca Bigliardi
      curr_helper = bdev.BaseDRBD.GetUsermodeHelper()
440 a721e23a Luca Bigliardi
    except errors.BlockDeviceError, err:
441 a721e23a Luca Bigliardi
      raise errors.OpPrereqError("Error while checking drbd helper"
442 a721e23a Luca Bigliardi
                                 " (specify --no-drbd-storage if you are not"
443 a721e23a Luca Bigliardi
                                 " using drbd): %s" % str(err),
444 a721e23a Luca Bigliardi
                                 errors.ECODE_ENVIRON)
445 a721e23a Luca Bigliardi
    if drbd_helper != curr_helper:
446 a721e23a Luca Bigliardi
      raise errors.OpPrereqError("Error: requiring %s as drbd helper but %s"
447 a721e23a Luca Bigliardi
                                 " is the current helper" % (drbd_helper,
448 a721e23a Luca Bigliardi
                                                             curr_helper),
449 a721e23a Luca Bigliardi
                                 errors.ECODE_INVAL)
450 a721e23a Luca Bigliardi
451 32ba31be Michael Hanselmann
  logging.debug("Stopping daemons (if any are running)")
452 32ba31be Michael Hanselmann
  result = utils.RunCmd([pathutils.DAEMON_UTIL, "stop-all"])
453 32ba31be Michael Hanselmann
  if result.failed:
454 32ba31be Michael Hanselmann
    raise errors.OpExecError("Could not stop daemons, command %s"
455 32ba31be Michael Hanselmann
                             " had exitcode %s and error '%s'" %
456 32ba31be Michael Hanselmann
                             (result.cmd, result.exit_code, result.output))
457 32ba31be Michael Hanselmann
458 0376655e Guido Trotter
  if constants.ENABLE_FILE_STORAGE:
459 0376655e Guido Trotter
    file_storage_dir = _InitFileStorage(file_storage_dir)
460 0376655e Guido Trotter
  else:
461 0376655e Guido Trotter
    file_storage_dir = ""
462 0376655e Guido Trotter
463 0376655e Guido Trotter
  if constants.ENABLE_SHARED_FILE_STORAGE:
464 0376655e Guido Trotter
    shared_file_storage_dir = _InitFileStorage(shared_file_storage_dir)
465 0376655e Guido Trotter
  else:
466 0376655e Guido Trotter
    shared_file_storage_dir = ""
467 a0c9f010 Michael Hanselmann
468 a0c9f010 Michael Hanselmann
  if not re.match("^[0-9a-z]{2}:[0-9a-z]{2}:[0-9a-z]{2}$", mac_prefix):
469 debac808 Iustin Pop
    raise errors.OpPrereqError("Invalid mac prefix given '%s'" % mac_prefix,
470 debac808 Iustin Pop
                               errors.ECODE_INVAL)
471 a0c9f010 Michael Hanselmann
472 a0c9f010 Michael Hanselmann
  result = utils.RunCmd(["ip", "link", "show", "dev", master_netdev])
473 a0c9f010 Michael Hanselmann
  if result.failed:
474 a0c9f010 Michael Hanselmann
    raise errors.OpPrereqError("Invalid master netdev given (%s): '%s'" %
475 a0c9f010 Michael Hanselmann
                               (master_netdev,
476 debac808 Iustin Pop
                                result.output.strip()), errors.ECODE_INVAL)
477 a0c9f010 Michael Hanselmann
478 7ede9c6a Michael Hanselmann
  dirs = [(pathutils.RUN_DIR, constants.RUN_DIRS_MODE)]
479 9dae41ad Guido Trotter
  utils.EnsureDirs(dirs)
480 9dae41ad Guido Trotter
481 b2e233a5 Guido Trotter
  objects.UpgradeBeParams(beparams)
482 a5728081 Guido Trotter
  utils.ForceDictType(beparams, constants.BES_PARAMETER_TYPES)
483 b6a30b0d Guido Trotter
  utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
484 18bb6d28 Agata Murawska
485 b6a30b0d Guido Trotter
  objects.NIC.CheckParameterSyntax(nicparams)
486 57dc299a Iustin Pop
487 2cc673a3 Iustin Pop
  full_ipolicy = objects.FillIPolicy(constants.IPOLICY_DEFAULTS, ipolicy)
488 b6a30b0d Guido Trotter
489 6204ee71 Renรฉ Nussbaumer
  if ndparams is not None:
490 6204ee71 Renรฉ Nussbaumer
    utils.ForceDictType(ndparams, constants.NDS_PARAMETER_TYPES)
491 6204ee71 Renรฉ Nussbaumer
  else:
492 6204ee71 Renรฉ Nussbaumer
    ndparams = dict(constants.NDC_DEFAULTS)
493 6204ee71 Renรฉ Nussbaumer
494 c4929a8b Renรฉ Nussbaumer
  # This is ugly, as we modify the dict itself
495 57dc299a Iustin Pop
  # FIXME: Make utils.ForceDictType pure functional or write a wrapper
496 57dc299a Iustin Pop
  # around it
497 c4929a8b Renรฉ Nussbaumer
  if hv_state:
498 c4929a8b Renรฉ Nussbaumer
    for hvname, hvs_data in hv_state.items():
499 c4929a8b Renรฉ Nussbaumer
      utils.ForceDictType(hvs_data, constants.HVSTS_PARAMETER_TYPES)
500 c4929a8b Renรฉ Nussbaumer
      hv_state[hvname] = objects.Cluster.SimpleFillHvState(hvs_data)
501 c4929a8b Renรฉ Nussbaumer
  else:
502 c4929a8b Renรฉ Nussbaumer
    hv_state = dict((hvname, constants.HVST_DEFAULTS)
503 c4929a8b Renรฉ Nussbaumer
                    for hvname in enabled_hypervisors)
504 c4929a8b Renรฉ Nussbaumer
505 c4929a8b Renรฉ Nussbaumer
  # FIXME: disk_state has no default values yet
506 c4929a8b Renรฉ Nussbaumer
  if disk_state:
507 c4929a8b Renรฉ Nussbaumer
    for storage, ds_data in disk_state.items():
508 c4929a8b Renรฉ Nussbaumer
      if storage not in constants.DS_VALID_TYPES:
509 c4929a8b Renรฉ Nussbaumer
        raise errors.OpPrereqError("Invalid storage type in disk state: %s" %
510 c4929a8b Renรฉ Nussbaumer
                                   storage, errors.ECODE_INVAL)
511 c4929a8b Renรฉ Nussbaumer
      for ds_name, state in ds_data.items():
512 c4929a8b Renรฉ Nussbaumer
        utils.ForceDictType(state, constants.DSS_PARAMETER_TYPES)
513 c4929a8b Renรฉ Nussbaumer
        ds_data[ds_name] = objects.Cluster.SimpleFillDiskState(state)
514 c4929a8b Renรฉ Nussbaumer
515 a5728081 Guido Trotter
  # hvparams is a mapping of hypervisor->hvparams dict
516 a5728081 Guido Trotter
  for hv_name, hv_params in hvparams.iteritems():
517 a5728081 Guido Trotter
    utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
518 a5728081 Guido Trotter
    hv_class = hypervisor.GetHypervisor(hv_name)
519 a5728081 Guido Trotter
    hv_class.CheckParameterSyntax(hv_params)
520 d4b72030 Guido Trotter
521 bc5d0215 Andrea Spadaccini
  # diskparams is a mapping of disk-template->diskparams dict
522 bc5d0215 Andrea Spadaccini
  for template, dt_params in diskparams.items():
523 bc5d0215 Andrea Spadaccini
    param_keys = set(dt_params.keys())
524 bc5d0215 Andrea Spadaccini
    default_param_keys = set(constants.DISK_DT_DEFAULTS[template].keys())
525 bc5d0215 Andrea Spadaccini
    if not (param_keys <= default_param_keys):
526 bc5d0215 Andrea Spadaccini
      unknown_params = param_keys - default_param_keys
527 bc5d0215 Andrea Spadaccini
      raise errors.OpPrereqError("Invalid parameters for disk template %s:"
528 bc5d0215 Andrea Spadaccini
                                 " %s" % (template,
529 2cfbc784 Iustin Pop
                                          utils.CommaJoin(unknown_params)),
530 2cfbc784 Iustin Pop
                                 errors.ECODE_INVAL)
531 bc5d0215 Andrea Spadaccini
    utils.ForceDictType(dt_params, constants.DISK_DT_TYPES)
532 e4a4391d Renรฉ Nussbaumer
  try:
533 e4a4391d Renรฉ Nussbaumer
    utils.VerifyDictOptions(diskparams, constants.DISK_DT_DEFAULTS)
534 e4a4391d Renรฉ Nussbaumer
  except errors.OpPrereqError, err:
535 e4a4391d Renรฉ Nussbaumer
    raise errors.OpPrereqError("While verify diskparam options: %s" % err,
536 e4a4391d Renรฉ Nussbaumer
                               errors.ECODE_INVAL)
537 bc5d0215 Andrea Spadaccini
538 a0c9f010 Michael Hanselmann
  # set up ssh config and /etc/hosts
539 7ede9c6a Michael Hanselmann
  sshline = utils.ReadFile(pathutils.SSH_HOST_RSA_PUB)
540 a0c9f010 Michael Hanselmann
  sshkey = sshline.split(" ")[1]
541 a0c9f010 Michael Hanselmann
542 b86a6bcd Guido Trotter
  if modify_etc_hosts:
543 ea8ac9c9 Renรฉ Nussbaumer
    utils.AddHostToEtcHosts(hostname.name, hostname.ip)
544 b86a6bcd Guido Trotter
545 b989b9d9 Ken Wehr
  if modify_ssh_setup:
546 b989b9d9 Ken Wehr
    _InitSSHSetup()
547 a0c9f010 Michael Hanselmann
548 bf4af505 Apollon Oikonomopoulos
  if default_iallocator is not None:
549 bf4af505 Apollon Oikonomopoulos
    alloc_script = utils.FindFile(default_iallocator,
550 bf4af505 Apollon Oikonomopoulos
                                  constants.IALLOCATOR_SEARCH_PATH,
551 bf4af505 Apollon Oikonomopoulos
                                  os.path.isfile)
552 bf4af505 Apollon Oikonomopoulos
    if alloc_script is None:
553 bf4af505 Apollon Oikonomopoulos
      raise errors.OpPrereqError("Invalid default iallocator script '%s'"
554 bf4af505 Apollon Oikonomopoulos
                                 " specified" % default_iallocator,
555 bf4af505 Apollon Oikonomopoulos
                                 errors.ECODE_INVAL)
556 d1e9c98d Iustin Pop
  elif constants.HTOOLS:
557 d1e9c98d Iustin Pop
    # htools was enabled at build-time, we default to it
558 d1e9c98d Iustin Pop
    if utils.FindFile(constants.IALLOC_HAIL,
559 d1e9c98d Iustin Pop
                      constants.IALLOCATOR_SEARCH_PATH,
560 d1e9c98d Iustin Pop
                      os.path.isfile):
561 d1e9c98d Iustin Pop
      default_iallocator = constants.IALLOC_HAIL
562 bf4af505 Apollon Oikonomopoulos
563 430b923c Iustin Pop
  now = time.time()
564 430b923c Iustin Pop
565 a0c9f010 Michael Hanselmann
  # init of cluster config file
566 b9eeeb02 Michael Hanselmann
  cluster_config = objects.Cluster(
567 b9eeeb02 Michael Hanselmann
    serial_no=1,
568 b9eeeb02 Michael Hanselmann
    rsahostkeypub=sshkey,
569 b9eeeb02 Michael Hanselmann
    highest_used_port=(constants.FIRST_DRBD_PORT - 1),
570 b9eeeb02 Michael Hanselmann
    mac_prefix=mac_prefix,
571 b9eeeb02 Michael Hanselmann
    volume_group_name=vg_name,
572 b9eeeb02 Michael Hanselmann
    tcpudp_port_pool=set(),
573 f6bd6e98 Michael Hanselmann
    master_node=hostname.name,
574 f6bd6e98 Michael Hanselmann
    master_ip=clustername.ip,
575 5a8648eb Andrea Spadaccini
    master_netmask=master_netmask,
576 f6bd6e98 Michael Hanselmann
    master_netdev=master_netdev,
577 f6bd6e98 Michael Hanselmann
    cluster_name=clustername.name,
578 f6bd6e98 Michael Hanselmann
    file_storage_dir=file_storage_dir,
579 4b97f902 Apollon Oikonomopoulos
    shared_file_storage_dir=shared_file_storage_dir,
580 ea3a925f Alexander Schreiber
    enabled_hypervisors=enabled_hypervisors,
581 4ef7f423 Guido Trotter
    beparams={constants.PP_DEFAULT: beparams},
582 b6a30b0d Guido Trotter
    nicparams={constants.PP_DEFAULT: nicparams},
583 6204ee71 Renรฉ Nussbaumer
    ndparams=ndparams,
584 ea3a925f Alexander Schreiber
    hvparams=hvparams,
585 bc5d0215 Andrea Spadaccini
    diskparams=diskparams,
586 ce735215 Guido Trotter
    candidate_pool_size=candidate_pool_size,
587 022c3a0b Guido Trotter
    modify_etc_hosts=modify_etc_hosts,
588 b989b9d9 Ken Wehr
    modify_ssh_setup=modify_ssh_setup,
589 39b0f0c2 Balazs Lecz
    uid_pool=uid_pool,
590 430b923c Iustin Pop
    ctime=now,
591 430b923c Iustin Pop
    mtime=now,
592 3953242f Iustin Pop
    maintain_node_health=maintain_node_health,
593 a721e23a Luca Bigliardi
    drbd_usermode_helper=drbd_helper,
594 bf4af505 Apollon Oikonomopoulos
    default_iallocator=default_iallocator,
595 2f20d07b Manuel Franceschini
    primary_ip_family=ipcls.family,
596 3d914585 Renรฉ Nussbaumer
    prealloc_wipe_disks=prealloc_wipe_disks,
597 bf689b7a Andrea Spadaccini
    use_external_mip_script=use_external_mip_script,
598 57dc299a Iustin Pop
    ipolicy=full_ipolicy,
599 c4929a8b Renรฉ Nussbaumer
    hv_state_static=hv_state,
600 c4929a8b Renรฉ Nussbaumer
    disk_state_static=disk_state,
601 b9eeeb02 Michael Hanselmann
    )
602 b9eeeb02 Michael Hanselmann
  master_node_config = objects.Node(name=hostname.name,
603 b9eeeb02 Michael Hanselmann
                                    primary_ip=hostname.ip,
604 b9222f32 Guido Trotter
                                    secondary_ip=secondary_ip,
605 c044f32c Guido Trotter
                                    serial_no=1,
606 c044f32c Guido Trotter
                                    master_candidate=True,
607 af64c0ea Iustin Pop
                                    offline=False, drained=False,
608 435e4bd6 Michael Hanselmann
                                    ctime=now, mtime=now,
609 c044f32c Guido Trotter
                                    )
610 9e1333b9 Guido Trotter
  InitConfig(constants.CONFIG_VERSION, cluster_config, master_node_config)
611 d367b66c Manuel Franceschini
  cfg = config.ConfigWriter(offline=True)
612 7ede9c6a Michael Hanselmann
  ssh.WriteKnownHostsFile(cfg, pathutils.SSH_KNOWN_HOSTS_FILE)
613 a4eae71f Michael Hanselmann
  cfg.Update(cfg.GetClusterInfo(), logging.error)
614 ee501db1 Michael Hanselmann
  ssconf.WriteSsconfFiles(cfg.GetSsconfValues())
615 d367b66c Manuel Franceschini
616 d367b66c Manuel Franceschini
  # set up the inter-node password and certificate
617 d367b66c Manuel Franceschini
  _InitGanetiServerSetup(hostname.name)
618 827f753e Guido Trotter
619 952d7515 Michael Hanselmann
  logging.debug("Starting daemons")
620 7ede9c6a Michael Hanselmann
  result = utils.RunCmd([pathutils.DAEMON_UTIL, "start-all"])
621 952d7515 Michael Hanselmann
  if result.failed:
622 952d7515 Michael Hanselmann
    raise errors.OpExecError("Could not start daemons, command %s"
623 952d7515 Michael Hanselmann
                             " had exitcode %s and error %s" %
624 952d7515 Michael Hanselmann
                             (result.cmd, result.exit_code, result.output))
625 b3f1cf6f Iustin Pop
626 3b6b6129 Michael Hanselmann
  _WaitForMasterDaemon()
627 b3f1cf6f Iustin Pop
628 b1b6ea87 Iustin Pop
629 02f99608 Oleksiy Mishchenko
def InitConfig(version, cluster_config, master_node_config,
630 7ede9c6a Michael Hanselmann
               cfg_file=pathutils.CLUSTER_CONF_FILE):
631 7b3a8fb5 Iustin Pop
  """Create the initial cluster configuration.
632 7b3a8fb5 Iustin Pop

633 7b3a8fb5 Iustin Pop
  It will contain the current node, which will also be the master
634 7b3a8fb5 Iustin Pop
  node, and no instances.
635 7b3a8fb5 Iustin Pop

636 7b3a8fb5 Iustin Pop
  @type version: int
637 c41eea6e Iustin Pop
  @param version: configuration version
638 c41eea6e Iustin Pop
  @type cluster_config: L{objects.Cluster}
639 c41eea6e Iustin Pop
  @param cluster_config: cluster configuration
640 c41eea6e Iustin Pop
  @type master_node_config: L{objects.Node}
641 c41eea6e Iustin Pop
  @param master_node_config: master node configuration
642 c41eea6e Iustin Pop
  @type cfg_file: string
643 c41eea6e Iustin Pop
  @param cfg_file: configuration file path
644 c41eea6e Iustin Pop

645 7b3a8fb5 Iustin Pop
  """
646 88b92fe3 Guido Trotter
  uuid_generator = config.TemporaryReservationManager()
647 88b92fe3 Guido Trotter
  cluster_config.uuid = uuid_generator.Generate([], utils.NewUUID,
648 88b92fe3 Guido Trotter
                                                _INITCONF_ECID)
649 88b92fe3 Guido Trotter
  master_node_config.uuid = uuid_generator.Generate([], utils.NewUUID,
650 88b92fe3 Guido Trotter
                                                    _INITCONF_ECID)
651 7b3a8fb5 Iustin Pop
  nodes = {
652 7b3a8fb5 Iustin Pop
    master_node_config.name: master_node_config,
653 7b3a8fb5 Iustin Pop
    }
654 88b92fe3 Guido Trotter
  default_nodegroup = objects.NodeGroup(
655 88b92fe3 Guido Trotter
    uuid=uuid_generator.Generate([], utils.NewUUID, _INITCONF_ECID),
656 75cf411a Adeodato Simo
    name=constants.INITIAL_NODE_GROUP_NAME,
657 88b92fe3 Guido Trotter
    members=[master_node_config.name],
658 99ccf8b9 Renรฉ Nussbaumer
    diskparams={},
659 88b92fe3 Guido Trotter
    )
660 88b92fe3 Guido Trotter
  nodegroups = {
661 88b92fe3 Guido Trotter
    default_nodegroup.uuid: default_nodegroup,
662 88b92fe3 Guido Trotter
    }
663 d693c864 Iustin Pop
  now = time.time()
664 7b3a8fb5 Iustin Pop
  config_data = objects.ConfigData(version=version,
665 7b3a8fb5 Iustin Pop
                                   cluster=cluster_config,
666 88b92fe3 Guido Trotter
                                   nodegroups=nodegroups,
667 7b3a8fb5 Iustin Pop
                                   nodes=nodes,
668 7b3a8fb5 Iustin Pop
                                   instances={},
669 eaa4c57c Dimitris Aragiorgis
                                   networks={},
670 d693c864 Iustin Pop
                                   serial_no=1,
671 d693c864 Iustin Pop
                                   ctime=now, mtime=now)
672 a33848a5 Guido Trotter
  utils.WriteFile(cfg_file,
673 a33848a5 Guido Trotter
                  data=serializer.Dump(config_data.ToDict()),
674 a33848a5 Guido Trotter
                  mode=0600)
675 02f99608 Oleksiy Mishchenko
676 02f99608 Oleksiy Mishchenko
677 140aa4a8 Iustin Pop
def FinalizeClusterDestroy(master):
678 140aa4a8 Iustin Pop
  """Execute the last steps of cluster destroy
679 140aa4a8 Iustin Pop

680 140aa4a8 Iustin Pop
  This function shuts down all the daemons, completing the destroy
681 140aa4a8 Iustin Pop
  begun in cmdlib.LUDestroyOpcode.
682 140aa4a8 Iustin Pop

683 140aa4a8 Iustin Pop
  """
684 b989b9d9 Ken Wehr
  cfg = config.ConfigWriter()
685 b989b9d9 Ken Wehr
  modify_ssh_setup = cfg.GetClusterInfo().modify_ssh_setup
686 7c74bbe0 Andrea Spadaccini
  runner = rpc.BootstrapRunner()
687 7c74bbe0 Andrea Spadaccini
688 f9d20654 Andrea Spadaccini
  master_params = cfg.GetMasterNetworkParameters()
689 c79198a0 Andrea Spadaccini
  master_params.name = master
690 57c7bc57 Andrea Spadaccini
  ems = cfg.GetUseExternalMipScript()
691 c79198a0 Andrea Spadaccini
  result = runner.call_node_deactivate_master_ip(master_params.name,
692 57c7bc57 Andrea Spadaccini
                                                 master_params, ems)
693 c79198a0 Andrea Spadaccini
694 7c74bbe0 Andrea Spadaccini
  msg = result.fail_msg
695 7c74bbe0 Andrea Spadaccini
  if msg:
696 7c74bbe0 Andrea Spadaccini
    logging.warning("Could not disable the master IP: %s", msg)
697 7c74bbe0 Andrea Spadaccini
698 7c74bbe0 Andrea Spadaccini
  result = runner.call_node_stop_master(master)
699 3cebe102 Michael Hanselmann
  msg = result.fail_msg
700 6c00d19a Iustin Pop
  if msg:
701 099c52ad Iustin Pop
    logging.warning("Could not disable the master role: %s", msg)
702 7c74bbe0 Andrea Spadaccini
703 7c74bbe0 Andrea Spadaccini
  result = runner.call_node_leave_cluster(master, modify_ssh_setup)
704 3cebe102 Michael Hanselmann
  msg = result.fail_msg
705 0623d351 Iustin Pop
  if msg:
706 0623d351 Iustin Pop
    logging.warning("Could not shutdown the node daemon and cleanup"
707 0623d351 Iustin Pop
                    " the node: %s", msg)
708 140aa4a8 Iustin Pop
709 140aa4a8 Iustin Pop
710 7b8ba235 Michael Hanselmann
def SetupNodeDaemon(opts, cluster_name, node):
711 827f753e Guido Trotter
  """Add a node to the cluster.
712 827f753e Guido Trotter

713 b1b6ea87 Iustin Pop
  This function must be called before the actual opcode, and will ssh
714 b1b6ea87 Iustin Pop
  to the remote node, copy the needed files, and start ganeti-noded,
715 b1b6ea87 Iustin Pop
  allowing the master to do the rest via normal rpc calls.
716 827f753e Guido Trotter

717 87622829 Iustin Pop
  @param cluster_name: the cluster name
718 87622829 Iustin Pop
  @param node: the name of the new node
719 827f753e Guido Trotter

720 827f753e Guido Trotter
  """
721 7b8ba235 Michael Hanselmann
  data = {
722 7b8ba235 Michael Hanselmann
    constants.NDS_CLUSTER_NAME: cluster_name,
723 7b8ba235 Michael Hanselmann
    constants.NDS_NODE_DAEMON_CERTIFICATE:
724 7b8ba235 Michael Hanselmann
      utils.ReadFile(pathutils.NODED_CERT_FILE),
725 7b8ba235 Michael Hanselmann
    constants.NDS_SSCONF: ssconf.SimpleStore().ReadAll(),
726 7b8ba235 Michael Hanselmann
    constants.NDS_START_NODE_DAEMON: True,
727 7b8ba235 Michael Hanselmann
    }
728 7b8ba235 Michael Hanselmann
729 7b8ba235 Michael Hanselmann
  RunNodeSetupCmd(cluster_name, node, pathutils.NODE_DAEMON_SETUP,
730 7b8ba235 Michael Hanselmann
                  opts.debug, opts.verbose,
731 7b8ba235 Michael Hanselmann
                  True, opts.ssh_key_check, opts.ssh_key_check, data)
732 827f753e Guido Trotter
733 5627f375 Michael Hanselmann
  _WaitForNodeDaemon(node)
734 5627f375 Michael Hanselmann
735 b1b6ea87 Iustin Pop
736 8e2524c3 Guido Trotter
def MasterFailover(no_voting=False):
737 b1b6ea87 Iustin Pop
  """Failover the master node.
738 b1b6ea87 Iustin Pop

739 b1b6ea87 Iustin Pop
  This checks that we are not already the master, and will cause the
740 b1b6ea87 Iustin Pop
  current master to cease being master, and the non-master to become
741 b1b6ea87 Iustin Pop
  new master.
742 b1b6ea87 Iustin Pop

743 8e2524c3 Guido Trotter
  @type no_voting: boolean
744 8e2524c3 Guido Trotter
  @param no_voting: force the operation without remote nodes agreement
745 8e2524c3 Guido Trotter
                      (dangerous)
746 8e2524c3 Guido Trotter

747 b1b6ea87 Iustin Pop
  """
748 8135a2db Iustin Pop
  sstore = ssconf.SimpleStore()
749 b1b6ea87 Iustin Pop
750 8135a2db Iustin Pop
  old_master, new_master = ssconf.GetMasterAndMyself(sstore)
751 8135a2db Iustin Pop
  node_list = sstore.GetNodeList()
752 8135a2db Iustin Pop
  mc_list = sstore.GetMasterCandidates()
753 b1b6ea87 Iustin Pop
754 b1b6ea87 Iustin Pop
  if old_master == new_master:
755 b1b6ea87 Iustin Pop
    raise errors.OpPrereqError("This commands must be run on the node"
756 b1b6ea87 Iustin Pop
                               " where you want the new master to be."
757 b1b6ea87 Iustin Pop
                               " %s is already the master" %
758 debac808 Iustin Pop
                               old_master, errors.ECODE_INVAL)
759 d5927e48 Iustin Pop
760 8135a2db Iustin Pop
  if new_master not in mc_list:
761 8135a2db Iustin Pop
    mc_no_master = [name for name in mc_list if name != old_master]
762 8135a2db Iustin Pop
    raise errors.OpPrereqError("This node is not among the nodes marked"
763 8135a2db Iustin Pop
                               " as master candidates. Only these nodes"
764 8135a2db Iustin Pop
                               " can become masters. Current list of"
765 8135a2db Iustin Pop
                               " master candidates is:\n"
766 3ccb3a64 Michael Hanselmann
                               "%s" % ("\n".join(mc_no_master)),
767 debac808 Iustin Pop
                               errors.ECODE_STATE)
768 8135a2db Iustin Pop
769 8e2524c3 Guido Trotter
  if not no_voting:
770 8e2524c3 Guido Trotter
    vote_list = GatherMasterVotes(node_list)
771 8e2524c3 Guido Trotter
772 8e2524c3 Guido Trotter
    if vote_list:
773 8e2524c3 Guido Trotter
      voted_master = vote_list[0][0]
774 8e2524c3 Guido Trotter
      if voted_master is None:
775 8e2524c3 Guido Trotter
        raise errors.OpPrereqError("Cluster is inconsistent, most nodes did"
776 debac808 Iustin Pop
                                   " not respond.", errors.ECODE_ENVIRON)
777 8e2524c3 Guido Trotter
      elif voted_master != old_master:
778 8e2524c3 Guido Trotter
        raise errors.OpPrereqError("I have a wrong configuration, I believe"
779 8e2524c3 Guido Trotter
                                   " the master is %s but the other nodes"
780 8e2524c3 Guido Trotter
                                   " voted %s. Please resync the configuration"
781 8e2524c3 Guido Trotter
                                   " of this node." %
782 debac808 Iustin Pop
                                   (old_master, voted_master),
783 debac808 Iustin Pop
                                   errors.ECODE_STATE)
784 b1b6ea87 Iustin Pop
  # end checks
785 b1b6ea87 Iustin Pop
786 b1b6ea87 Iustin Pop
  rcode = 0
787 b1b6ea87 Iustin Pop
788 d5927e48 Iustin Pop
  logging.info("Setting master to %s, old master: %s", new_master, old_master)
789 b1b6ea87 Iustin Pop
790 21004460 Iustin Pop
  try:
791 21004460 Iustin Pop
    # instantiate a real config writer, as we now know we have the
792 21004460 Iustin Pop
    # configuration data
793 eb180fe2 Iustin Pop
    cfg = config.ConfigWriter(accept_foreign=True)
794 21004460 Iustin Pop
795 21004460 Iustin Pop
    cluster_info = cfg.GetClusterInfo()
796 21004460 Iustin Pop
    cluster_info.master_node = new_master
797 21004460 Iustin Pop
    # this will also regenerate the ssconf files, since we updated the
798 21004460 Iustin Pop
    # cluster info
799 21004460 Iustin Pop
    cfg.Update(cluster_info, logging.error)
800 21004460 Iustin Pop
  except errors.ConfigurationError, err:
801 21004460 Iustin Pop
    logging.error("Error while trying to set the new master: %s",
802 21004460 Iustin Pop
                  str(err))
803 21004460 Iustin Pop
    return 1
804 21004460 Iustin Pop
805 21004460 Iustin Pop
  # if cfg.Update worked, then it means the old master daemon won't be
806 21004460 Iustin Pop
  # able now to write its own config file (we rely on locking in both
807 21004460 Iustin Pop
  # backend.UploadFile() and ConfigWriter._Write(); hence the next
808 21004460 Iustin Pop
  # step is to kill the old master
809 21004460 Iustin Pop
810 21004460 Iustin Pop
  logging.info("Stopping the master daemon on node %s", old_master)
811 21004460 Iustin Pop
812 7c74bbe0 Andrea Spadaccini
  runner = rpc.BootstrapRunner()
813 f9d20654 Andrea Spadaccini
  master_params = cfg.GetMasterNetworkParameters()
814 c79198a0 Andrea Spadaccini
  master_params.name = old_master
815 57c7bc57 Andrea Spadaccini
  ems = cfg.GetUseExternalMipScript()
816 c79198a0 Andrea Spadaccini
  result = runner.call_node_deactivate_master_ip(master_params.name,
817 57c7bc57 Andrea Spadaccini
                                                 master_params, ems)
818 c79198a0 Andrea Spadaccini
819 7c74bbe0 Andrea Spadaccini
  msg = result.fail_msg
820 7c74bbe0 Andrea Spadaccini
  if msg:
821 7c74bbe0 Andrea Spadaccini
    logging.warning("Could not disable the master IP: %s", msg)
822 7c74bbe0 Andrea Spadaccini
823 7c74bbe0 Andrea Spadaccini
  result = runner.call_node_stop_master(old_master)
824 3cebe102 Michael Hanselmann
  msg = result.fail_msg
825 6c00d19a Iustin Pop
  if msg:
826 d5927e48 Iustin Pop
    logging.error("Could not disable the master role on the old master"
827 5ae4945a Iustin Pop
                  " %s, please disable manually: %s", old_master, msg)
828 b1b6ea87 Iustin Pop
829 21004460 Iustin Pop
  logging.info("Checking master IP non-reachability...")
830 21004460 Iustin Pop
831 425f0f54 Iustin Pop
  master_ip = sstore.GetMasterIP()
832 425f0f54 Iustin Pop
  total_timeout = 30
833 e687ec01 Michael Hanselmann
834 d23ef431 Michael Hanselmann
  # Here we have a phase where no master should be running
835 425f0f54 Iustin Pop
  def _check_ip():
836 a744b676 Manuel Franceschini
    if netutils.TcpPing(master_ip, constants.DEFAULT_NODED_PORT):
837 425f0f54 Iustin Pop
      raise utils.RetryAgain()
838 425f0f54 Iustin Pop
839 425f0f54 Iustin Pop
  try:
840 425f0f54 Iustin Pop
    utils.Retry(_check_ip, (1, 1.5, 5), total_timeout)
841 425f0f54 Iustin Pop
  except utils.RetryTimeout:
842 425f0f54 Iustin Pop
    logging.warning("The master IP is still reachable after %s seconds,"
843 425f0f54 Iustin Pop
                    " continuing but activating the master on the current"
844 425f0f54 Iustin Pop
                    " node will probably fail", total_timeout)
845 b1b6ea87 Iustin Pop
846 ff699aa9 Michael Hanselmann
  if jstore.CheckDrainFlag():
847 ff699aa9 Michael Hanselmann
    logging.info("Undraining job queue")
848 ff699aa9 Michael Hanselmann
    jstore.SetDrainFlag(False)
849 ff699aa9 Michael Hanselmann
850 21004460 Iustin Pop
  logging.info("Starting the master daemons on the new master")
851 d5927e48 Iustin Pop
852 db04ce5d Michael Hanselmann
  result = rpc.BootstrapRunner().call_node_start_master_daemons(new_master,
853 db04ce5d Michael Hanselmann
                                                                no_voting)
854 3cebe102 Michael Hanselmann
  msg = result.fail_msg
855 b726aff0 Iustin Pop
  if msg:
856 d5927e48 Iustin Pop
    logging.error("Could not start the master role on the new master"
857 b726aff0 Iustin Pop
                  " %s, please check: %s", new_master, msg)
858 b1b6ea87 Iustin Pop
    rcode = 1
859 b1b6ea87 Iustin Pop
860 21004460 Iustin Pop
  logging.info("Master failed over from %s to %s", old_master, new_master)
861 b1b6ea87 Iustin Pop
  return rcode
862 d7cdb55d Iustin Pop
863 d7cdb55d Iustin Pop
864 8eb148ae Iustin Pop
def GetMaster():
865 8eb148ae Iustin Pop
  """Returns the current master node.
866 8eb148ae Iustin Pop

867 8eb148ae Iustin Pop
  This is a separate function in bootstrap since it's needed by
868 8eb148ae Iustin Pop
  gnt-cluster, and instead of importing directly ssconf, it's better
869 8eb148ae Iustin Pop
  to abstract it in bootstrap, where we do use ssconf in other
870 8eb148ae Iustin Pop
  functions too.
871 8eb148ae Iustin Pop

872 8eb148ae Iustin Pop
  """
873 8eb148ae Iustin Pop
  sstore = ssconf.SimpleStore()
874 8eb148ae Iustin Pop
875 8eb148ae Iustin Pop
  old_master, _ = ssconf.GetMasterAndMyself(sstore)
876 8eb148ae Iustin Pop
877 8eb148ae Iustin Pop
  return old_master
878 8eb148ae Iustin Pop
879 8eb148ae Iustin Pop
880 d7cdb55d Iustin Pop
def GatherMasterVotes(node_list):
881 d7cdb55d Iustin Pop
  """Check the agreement on who is the master.
882 d7cdb55d Iustin Pop

883 d7cdb55d Iustin Pop
  This function will return a list of (node, number of votes), ordered
884 d7cdb55d Iustin Pop
  by the number of votes. Errors will be denoted by the key 'None'.
885 d7cdb55d Iustin Pop

886 d7cdb55d Iustin Pop
  Note that the sum of votes is the number of nodes this machine
887 d7cdb55d Iustin Pop
  knows, whereas the number of entries in the list could be different
888 d7cdb55d Iustin Pop
  (if some nodes vote for another master).
889 d7cdb55d Iustin Pop

890 d7cdb55d Iustin Pop
  We remove ourselves from the list since we know that (bugs aside)
891 d7cdb55d Iustin Pop
  since we use the same source for configuration information for both
892 d7cdb55d Iustin Pop
  backend and boostrap, we'll always vote for ourselves.
893 d7cdb55d Iustin Pop

894 d7cdb55d Iustin Pop
  @type node_list: list
895 d7cdb55d Iustin Pop
  @param node_list: the list of nodes to query for master info; the current
896 5bbd3f7f Michael Hanselmann
      node will be removed if it is in the list
897 d7cdb55d Iustin Pop
  @rtype: list
898 d7cdb55d Iustin Pop
  @return: list of (node, votes)
899 d7cdb55d Iustin Pop

900 d7cdb55d Iustin Pop
  """
901 b705c7a6 Manuel Franceschini
  myself = netutils.Hostname.GetSysName()
902 d7cdb55d Iustin Pop
  try:
903 d7cdb55d Iustin Pop
    node_list.remove(myself)
904 d7cdb55d Iustin Pop
  except ValueError:
905 d7cdb55d Iustin Pop
    pass
906 d7cdb55d Iustin Pop
  if not node_list:
907 d7cdb55d Iustin Pop
    # no nodes left (eventually after removing myself)
908 d7cdb55d Iustin Pop
    return []
909 db04ce5d Michael Hanselmann
  results = rpc.BootstrapRunner().call_master_info(node_list)
910 d7cdb55d Iustin Pop
  if not isinstance(results, dict):
911 d7cdb55d Iustin Pop
    # this should not happen (unless internal error in rpc)
912 d7cdb55d Iustin Pop
    logging.critical("Can't complete rpc call, aborting master startup")
913 d7cdb55d Iustin Pop
    return [(None, len(node_list))]
914 d7cdb55d Iustin Pop
  votes = {}
915 d7cdb55d Iustin Pop
  for node in results:
916 781de953 Iustin Pop
    nres = results[node]
917 2a52a064 Iustin Pop
    data = nres.payload
918 3cebe102 Michael Hanselmann
    msg = nres.fail_msg
919 2a52a064 Iustin Pop
    fail = False
920 2a52a064 Iustin Pop
    if msg:
921 2a52a064 Iustin Pop
      logging.warning("Error contacting node %s: %s", node, msg)
922 2a52a064 Iustin Pop
      fail = True
923 909b3a0e Andrea Spadaccini
    # for now we accept both length 3, 4 and 5 (data[3] is primary ip version
924 909b3a0e Andrea Spadaccini
    # and data[4] is the master netmask)
925 2a52a064 Iustin Pop
    elif not isinstance(data, (tuple, list)) or len(data) < 3:
926 2a52a064 Iustin Pop
      logging.warning("Invalid data received from node %s: %s", node, data)
927 2a52a064 Iustin Pop
      fail = True
928 2a52a064 Iustin Pop
    if fail:
929 d7cdb55d Iustin Pop
      if None not in votes:
930 d7cdb55d Iustin Pop
        votes[None] = 0
931 d7cdb55d Iustin Pop
      votes[None] += 1
932 d7cdb55d Iustin Pop
      continue
933 781de953 Iustin Pop
    master_node = data[2]
934 d7cdb55d Iustin Pop
    if master_node not in votes:
935 d7cdb55d Iustin Pop
      votes[master_node] = 0
936 d7cdb55d Iustin Pop
    votes[master_node] += 1
937 d7cdb55d Iustin Pop
938 d7cdb55d Iustin Pop
  vote_list = [v for v in votes.items()]
939 d7cdb55d Iustin Pop
  # sort first on number of votes then on name, since we want None
940 d7cdb55d Iustin Pop
  # sorted later if we have the half of the nodes not responding, and
941 d7cdb55d Iustin Pop
  # half voting all for the same master
942 d7cdb55d Iustin Pop
  vote_list.sort(key=lambda x: (x[1], x[0]), reverse=True)
943 d7cdb55d Iustin Pop
944 d7cdb55d Iustin Pop
  return vote_list