Statistics
| Branch: | Tag: | Revision:

root / lib / bootstrap.py @ 7a0156dc

History | View | Annotate | Download (19.1 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2008 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Functions to bootstrap a new cluster.
23

24
"""
25

    
26
import os
27
import os.path
28
import re
29
import logging
30
import tempfile
31
import time
32

    
33
from ganeti import rpc
34
from ganeti import ssh
35
from ganeti import utils
36
from ganeti import errors
37
from ganeti import config
38
from ganeti import constants
39
from ganeti import objects
40
from ganeti import ssconf
41
from ganeti import serializer
42
from ganeti import hypervisor
43

    
44

    
45
def _InitSSHSetup():
46
  """Setup the SSH configuration for the cluster.
47

48
  This generates a dsa keypair for root, adds the pub key to the
49
  permitted hosts and adds the hostkey to its own known hosts.
50

51
  """
52
  priv_key, pub_key, auth_keys = ssh.GetUserFiles(constants.GANETI_RUNAS)
53

    
54
  for name in priv_key, pub_key:
55
    if os.path.exists(name):
56
      utils.CreateBackup(name)
57
    utils.RemoveFile(name)
58

    
59
  result = utils.RunCmd(["ssh-keygen", "-t", "dsa",
60
                         "-f", priv_key,
61
                         "-q", "-N", ""])
62
  if result.failed:
63
    raise errors.OpExecError("Could not generate ssh keypair, error %s" %
64
                             result.output)
65

    
66
  utils.AddAuthorizedKey(auth_keys, utils.ReadFile(pub_key))
67

    
68

    
69
def _GenerateSelfSignedSslCert(file_name, validity=(365 * 5)):
70
  """Generates a self-signed SSL certificate.
71

72
  @type file_name: str
73
  @param file_name: Path to output file
74
  @type validity: int
75
  @param validity: Validity for certificate in days
76

77
  """
78
  (fd, tmp_file_name) = tempfile.mkstemp(dir=os.path.dirname(file_name))
79
  try:
80
    try:
81
      # Set permissions before writing key
82
      os.chmod(tmp_file_name, 0600)
83

    
84
      result = utils.RunCmd(["openssl", "req", "-new", "-newkey", "rsa:1024",
85
                             "-days", str(validity), "-nodes", "-x509",
86
                             "-keyout", tmp_file_name, "-out", tmp_file_name,
87
                             "-batch"])
88
      if result.failed:
89
        raise errors.OpExecError("Could not generate SSL certificate, command"
90
                                 " %s had exitcode %s and error message %s" %
91
                                 (result.cmd, result.exit_code, result.output))
92

    
93
      # Make read-only
94
      os.chmod(tmp_file_name, 0400)
95

    
96
      os.rename(tmp_file_name, file_name)
97
    finally:
98
      utils.RemoveFile(tmp_file_name)
99
  finally:
100
    os.close(fd)
101

    
102

    
103
def _InitGanetiServerSetup():
104
  """Setup the necessary configuration for the initial node daemon.
105

106
  This creates the nodepass file containing the shared password for
107
  the cluster and also generates the SSL certificate.
108

109
  """
110
  _GenerateSelfSignedSslCert(constants.SSL_CERT_FILE)
111

    
112
  # Don't overwrite existing file
113
  if not os.path.exists(constants.RAPI_CERT_FILE):
114
    _GenerateSelfSignedSslCert(constants.RAPI_CERT_FILE)
115

    
116
  if not os.path.exists(constants.HMAC_CLUSTER_KEY):
117
    utils.WriteFile(constants.HMAC_CLUSTER_KEY,
118
                    data=utils.GenerateSecret(),
119
                    mode=0400)
120

    
121
  result = utils.RunCmd([constants.NODE_INITD_SCRIPT, "restart"])
122

    
123
  if result.failed:
124
    raise errors.OpExecError("Could not start the node daemon, command %s"
125
                             " had exitcode %s and error %s" %
126
                             (result.cmd, result.exit_code, result.output))
127

    
128

    
129
def InitCluster(cluster_name, mac_prefix,
130
                master_netdev, file_storage_dir, candidate_pool_size,
131
                secondary_ip=None, vg_name=None, beparams=None,
132
                nicparams=None, hvparams=None, enabled_hypervisors=None,
133
                modify_etc_hosts=True):
134
  """Initialise the cluster.
135

136
  @type candidate_pool_size: int
137
  @param candidate_pool_size: master candidate pool size
138

139
  """
140
  # TODO: complete the docstring
141
  if config.ConfigWriter.IsCluster():
142
    raise errors.OpPrereqError("Cluster is already initialised")
143

    
144
  if not enabled_hypervisors:
145
    raise errors.OpPrereqError("Enabled hypervisors list must contain at"
146
                               " least one member")
147
  invalid_hvs = set(enabled_hypervisors) - constants.HYPER_TYPES
148
  if invalid_hvs:
149
    raise errors.OpPrereqError("Enabled hypervisors contains invalid"
150
                               " entries: %s" % invalid_hvs)
151

    
152
  hostname = utils.HostInfo()
153

    
154
  if hostname.ip.startswith("127."):
155
    raise errors.OpPrereqError("This host's IP resolves to the private"
156
                               " range (%s). Please fix DNS or %s." %
157
                               (hostname.ip, constants.ETC_HOSTS))
158

    
159
  if not utils.OwnIpAddress(hostname.ip):
160
    raise errors.OpPrereqError("Inconsistency: this host's name resolves"
161
                               " to %s,\nbut this ip address does not"
162
                               " belong to this host."
163
                               " Aborting." % hostname.ip)
164

    
165
  clustername = utils.HostInfo(cluster_name)
166

    
167
  if utils.TcpPing(clustername.ip, constants.DEFAULT_NODED_PORT,
168
                   timeout=5):
169
    raise errors.OpPrereqError("Cluster IP already active. Aborting.")
170

    
171
  if secondary_ip:
172
    if not utils.IsValidIP(secondary_ip):
173
      raise errors.OpPrereqError("Invalid secondary ip given")
174
    if (secondary_ip != hostname.ip and
175
        not utils.OwnIpAddress(secondary_ip)):
176
      raise errors.OpPrereqError("You gave %s as secondary IP,"
177
                                 " but it does not belong to this host." %
178
                                 secondary_ip)
179
  else:
180
    secondary_ip = hostname.ip
181

    
182
  if vg_name is not None:
183
    # Check if volume group is valid
184
    vgstatus = utils.CheckVolumeGroupSize(utils.ListVolumeGroups(), vg_name,
185
                                          constants.MIN_VG_SIZE)
186
    if vgstatus:
187
      raise errors.OpPrereqError("Error: %s\nspecify --no-lvm-storage if"
188
                                 " you are not using lvm" % vgstatus)
189

    
190
  file_storage_dir = os.path.normpath(file_storage_dir)
191

    
192
  if not os.path.isabs(file_storage_dir):
193
    raise errors.OpPrereqError("The file storage directory you passed is"
194
                               " not an absolute path.")
195

    
196
  if not os.path.exists(file_storage_dir):
197
    try:
198
      os.makedirs(file_storage_dir, 0750)
199
    except OSError, err:
200
      raise errors.OpPrereqError("Cannot create file storage directory"
201
                                 " '%s': %s" %
202
                                 (file_storage_dir, err))
203

    
204
  if not os.path.isdir(file_storage_dir):
205
    raise errors.OpPrereqError("The file storage directory '%s' is not"
206
                               " a directory." % file_storage_dir)
207

    
208
  if not re.match("^[0-9a-z]{2}:[0-9a-z]{2}:[0-9a-z]{2}$", mac_prefix):
209
    raise errors.OpPrereqError("Invalid mac prefix given '%s'" % mac_prefix)
210

    
211
  result = utils.RunCmd(["ip", "link", "show", "dev", master_netdev])
212
  if result.failed:
213
    raise errors.OpPrereqError("Invalid master netdev given (%s): '%s'" %
214
                               (master_netdev,
215
                                result.output.strip()))
216

    
217
  if not (os.path.isfile(constants.NODE_INITD_SCRIPT) and
218
          os.access(constants.NODE_INITD_SCRIPT, os.X_OK)):
219
    raise errors.OpPrereqError("Init.d script '%s' missing or not"
220
                               " executable." % constants.NODE_INITD_SCRIPT)
221

    
222
  dirs = [(constants.RUN_GANETI_DIR, constants.RUN_DIRS_MODE)]
223
  utils.EnsureDirs(dirs)
224

    
225
  utils.ForceDictType(beparams, constants.BES_PARAMETER_TYPES)
226
  utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
227
  objects.NIC.CheckParameterSyntax(nicparams)
228

    
229
  # hvparams is a mapping of hypervisor->hvparams dict
230
  for hv_name, hv_params in hvparams.iteritems():
231
    utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
232
    hv_class = hypervisor.GetHypervisor(hv_name)
233
    hv_class.CheckParameterSyntax(hv_params)
234

    
235
  # set up the inter-node password and certificate
236
  _InitGanetiServerSetup()
237

    
238
  # set up ssh config and /etc/hosts
239
  sshline = utils.ReadFile(constants.SSH_HOST_RSA_PUB)
240
  sshkey = sshline.split(" ")[1]
241

    
242
  if modify_etc_hosts:
243
    utils.AddHostToEtcHosts(hostname.name)
244

    
245
  _InitSSHSetup()
246

    
247
  # init of cluster config file
248
  cluster_config = objects.Cluster(
249
    serial_no=1,
250
    rsahostkeypub=sshkey,
251
    highest_used_port=(constants.FIRST_DRBD_PORT - 1),
252
    mac_prefix=mac_prefix,
253
    volume_group_name=vg_name,
254
    tcpudp_port_pool=set(),
255
    master_node=hostname.name,
256
    master_ip=clustername.ip,
257
    master_netdev=master_netdev,
258
    cluster_name=clustername.name,
259
    file_storage_dir=file_storage_dir,
260
    enabled_hypervisors=enabled_hypervisors,
261
    beparams={constants.PP_DEFAULT: beparams},
262
    nicparams={constants.PP_DEFAULT: nicparams},
263
    hvparams=hvparams,
264
    candidate_pool_size=candidate_pool_size,
265
    modify_etc_hosts=modify_etc_hosts,
266
    )
267
  master_node_config = objects.Node(name=hostname.name,
268
                                    primary_ip=hostname.ip,
269
                                    secondary_ip=secondary_ip,
270
                                    serial_no=1,
271
                                    master_candidate=True,
272
                                    offline=False, drained=False,
273
                                    )
274
  InitConfig(constants.CONFIG_VERSION, cluster_config, master_node_config)
275
  cfg = config.ConfigWriter()
276
  ssh.WriteKnownHostsFile(cfg, constants.SSH_KNOWN_HOSTS_FILE)
277
  cfg.Update(cfg.GetClusterInfo())
278

    
279
  # start the master ip
280
  # TODO: Review rpc call from bootstrap
281
  # TODO: Warn on failed start master
282
  rpc.RpcRunner.call_node_start_master(hostname.name, True, False)
283

    
284

    
285
def InitConfig(version, cluster_config, master_node_config,
286
               cfg_file=constants.CLUSTER_CONF_FILE):
287
  """Create the initial cluster configuration.
288

289
  It will contain the current node, which will also be the master
290
  node, and no instances.
291

292
  @type version: int
293
  @param version: configuration version
294
  @type cluster_config: L{objects.Cluster}
295
  @param cluster_config: cluster configuration
296
  @type master_node_config: L{objects.Node}
297
  @param master_node_config: master node configuration
298
  @type cfg_file: string
299
  @param cfg_file: configuration file path
300

301
  """
302
  nodes = {
303
    master_node_config.name: master_node_config,
304
    }
305

    
306
  now = time.time()
307
  config_data = objects.ConfigData(version=version,
308
                                   cluster=cluster_config,
309
                                   nodes=nodes,
310
                                   instances={},
311
                                   serial_no=1,
312
                                   ctime=now, mtime=now)
313
  utils.WriteFile(cfg_file,
314
                  data=serializer.Dump(config_data.ToDict()),
315
                  mode=0600)
316

    
317

    
318
def FinalizeClusterDestroy(master):
319
  """Execute the last steps of cluster destroy
320

321
  This function shuts down all the daemons, completing the destroy
322
  begun in cmdlib.LUDestroyOpcode.
323

324
  """
325
  result = rpc.RpcRunner.call_node_stop_master(master, True)
326
  msg = result.RemoteFailMsg()
327
  if msg:
328
    logging.warning("Could not disable the master role: %s" % msg)
329
  result = rpc.RpcRunner.call_node_leave_cluster(master)
330
  msg = result.RemoteFailMsg()
331
  if msg:
332
    logging.warning("Could not shutdown the node daemon and cleanup"
333
                    " the node: %s", msg)
334

    
335

    
336
def SetupNodeDaemon(cluster_name, node, ssh_key_check):
337
  """Add a node to the cluster.
338

339
  This function must be called before the actual opcode, and will ssh
340
  to the remote node, copy the needed files, and start ganeti-noded,
341
  allowing the master to do the rest via normal rpc calls.
342

343
  @param cluster_name: the cluster name
344
  @param node: the name of the new node
345
  @param ssh_key_check: whether to do a strict key check
346

347
  """
348
  sshrunner = ssh.SshRunner(cluster_name)
349

    
350
  noded_cert = utils.ReadFile(constants.SSL_CERT_FILE)
351
  rapi_cert = utils.ReadFile(constants.RAPI_CERT_FILE)
352

    
353
  # in the base64 pem encoding, neither '!' nor '.' are valid chars,
354
  # so we use this to detect an invalid certificate; as long as the
355
  # cert doesn't contain this, the here-document will be correctly
356
  # parsed by the shell sequence below
357
  if (re.search('^!EOF\.', noded_cert, re.MULTILINE) or
358
      re.search('^!EOF\.', rapi_cert, re.MULTILINE)):
359
    raise errors.OpExecError("invalid PEM encoding in the SSL certificate")
360

    
361
  if not noded_cert.endswith("\n"):
362
    noded_cert += "\n"
363
  if not rapi_cert.endswith("\n"):
364
    rapi_cert += "\n"
365

    
366
  # set up inter-node password and certificate and restarts the node daemon
367
  # and then connect with ssh to set password and start ganeti-noded
368
  # note that all the below variables are sanitized at this point,
369
  # either by being constants or by the checks above
370
  mycommand = ("umask 077 && "
371
               "cat > '%s' << '!EOF.' && \n"
372
               "%s!EOF.\n"
373
               "cat > '%s' << '!EOF.' && \n"
374
               "%s!EOF.\n"
375
               "chmod 0400 %s %s && "
376
               "%s restart" %
377
               (constants.SSL_CERT_FILE, noded_cert,
378
                constants.RAPI_CERT_FILE, rapi_cert,
379
                constants.SSL_CERT_FILE, constants.RAPI_CERT_FILE,
380
                constants.NODE_INITD_SCRIPT))
381

    
382
  result = sshrunner.Run(node, 'root', mycommand, batch=False,
383
                         ask_key=ssh_key_check,
384
                         use_cluster_key=False,
385
                         strict_host_check=ssh_key_check)
386
  if result.failed:
387
    raise errors.OpExecError("Remote command on node %s, error: %s,"
388
                             " output: %s" %
389
                             (node, result.fail_reason, result.output))
390

    
391

    
392
def MasterFailover(no_voting=False):
393
  """Failover the master node.
394

395
  This checks that we are not already the master, and will cause the
396
  current master to cease being master, and the non-master to become
397
  new master.
398

399
  @type no_voting: boolean
400
  @param no_voting: force the operation without remote nodes agreement
401
                      (dangerous)
402

403
  """
404
  sstore = ssconf.SimpleStore()
405

    
406
  old_master, new_master = ssconf.GetMasterAndMyself(sstore)
407
  node_list = sstore.GetNodeList()
408
  mc_list = sstore.GetMasterCandidates()
409

    
410
  if old_master == new_master:
411
    raise errors.OpPrereqError("This commands must be run on the node"
412
                               " where you want the new master to be."
413
                               " %s is already the master" %
414
                               old_master)
415

    
416
  if new_master not in mc_list:
417
    mc_no_master = [name for name in mc_list if name != old_master]
418
    raise errors.OpPrereqError("This node is not among the nodes marked"
419
                               " as master candidates. Only these nodes"
420
                               " can become masters. Current list of"
421
                               " master candidates is:\n"
422
                               "%s" % ('\n'.join(mc_no_master)))
423

    
424
  if not no_voting:
425
    vote_list = GatherMasterVotes(node_list)
426

    
427
    if vote_list:
428
      voted_master = vote_list[0][0]
429
      if voted_master is None:
430
        raise errors.OpPrereqError("Cluster is inconsistent, most nodes did"
431
                                   " not respond.")
432
      elif voted_master != old_master:
433
        raise errors.OpPrereqError("I have a wrong configuration, I believe"
434
                                   " the master is %s but the other nodes"
435
                                   " voted %s. Please resync the configuration"
436
                                   " of this node." %
437
                                   (old_master, voted_master))
438
  # end checks
439

    
440
  rcode = 0
441

    
442
  logging.info("Setting master to %s, old master: %s", new_master, old_master)
443

    
444
  result = rpc.RpcRunner.call_node_stop_master(old_master, True)
445
  msg = result.RemoteFailMsg()
446
  if msg:
447
    logging.error("Could not disable the master role on the old master"
448
                 " %s, please disable manually: %s", old_master, msg)
449

    
450
  # Here we have a phase where no master should be running
451

    
452
  # instantiate a real config writer, as we now know we have the
453
  # configuration data
454
  cfg = config.ConfigWriter()
455

    
456
  cluster_info = cfg.GetClusterInfo()
457
  cluster_info.master_node = new_master
458
  # this will also regenerate the ssconf files, since we updated the
459
  # cluster info
460
  cfg.Update(cluster_info)
461

    
462
  result = rpc.RpcRunner.call_node_start_master(new_master, True, no_voting)
463
  msg = result.RemoteFailMsg()
464
  if msg:
465
    logging.error("Could not start the master role on the new master"
466
                  " %s, please check: %s", new_master, msg)
467
    rcode = 1
468

    
469
  return rcode
470

    
471

    
472
def GetMaster():
473
  """Returns the current master node.
474

475
  This is a separate function in bootstrap since it's needed by
476
  gnt-cluster, and instead of importing directly ssconf, it's better
477
  to abstract it in bootstrap, where we do use ssconf in other
478
  functions too.
479

480
  """
481
  sstore = ssconf.SimpleStore()
482

    
483
  old_master, _ = ssconf.GetMasterAndMyself(sstore)
484

    
485
  return old_master
486

    
487

    
488
def GatherMasterVotes(node_list):
489
  """Check the agreement on who is the master.
490

491
  This function will return a list of (node, number of votes), ordered
492
  by the number of votes. Errors will be denoted by the key 'None'.
493

494
  Note that the sum of votes is the number of nodes this machine
495
  knows, whereas the number of entries in the list could be different
496
  (if some nodes vote for another master).
497

498
  We remove ourselves from the list since we know that (bugs aside)
499
  since we use the same source for configuration information for both
500
  backend and boostrap, we'll always vote for ourselves.
501

502
  @type node_list: list
503
  @param node_list: the list of nodes to query for master info; the current
504
      node will be removed if it is in the list
505
  @rtype: list
506
  @return: list of (node, votes)
507

508
  """
509
  myself = utils.HostInfo().name
510
  try:
511
    node_list.remove(myself)
512
  except ValueError:
513
    pass
514
  if not node_list:
515
    # no nodes left (eventually after removing myself)
516
    return []
517
  results = rpc.RpcRunner.call_master_info(node_list)
518
  if not isinstance(results, dict):
519
    # this should not happen (unless internal error in rpc)
520
    logging.critical("Can't complete rpc call, aborting master startup")
521
    return [(None, len(node_list))]
522
  votes = {}
523
  for node in results:
524
    nres = results[node]
525
    data = nres.payload
526
    msg = nres.RemoteFailMsg()
527
    fail = False
528
    if msg:
529
      logging.warning("Error contacting node %s: %s", node, msg)
530
      fail = True
531
    elif not isinstance(data, (tuple, list)) or len(data) < 3:
532
      logging.warning("Invalid data received from node %s: %s", node, data)
533
      fail = True
534
    if fail:
535
      if None not in votes:
536
        votes[None] = 0
537
      votes[None] += 1
538
      continue
539
    master_node = data[2]
540
    if master_node not in votes:
541
      votes[master_node] = 0
542
    votes[master_node] += 1
543

    
544
  vote_list = [v for v in votes.items()]
545
  # sort first on number of votes then on name, since we want None
546
  # sorted later if we have the half of the nodes not responding, and
547
  # half voting all for the same master
548
  vote_list.sort(key=lambda x: (x[1], x[0]), reverse=True)
549

    
550
  return vote_list