Statistics
| Branch: | Tag: | Revision:

root / lib / client / gnt_cluster.py @ ff6c5e55

History | View | Annotate | Download (51.6 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21
"""Cluster related commands"""
22

    
23
# pylint: disable=W0401,W0613,W0614,C0103
24
# W0401: Wildcard import ganeti.cli
25
# W0613: Unused argument, since all functions follow the same API
26
# W0614: Unused import %s from wildcard import (since we need cli)
27
# C0103: Invalid name gnt-cluster
28

    
29
import os.path
30
import time
31
import OpenSSL
32
import itertools
33

    
34
from ganeti.cli import *
35
from ganeti import opcodes
36
from ganeti import constants
37
from ganeti import errors
38
from ganeti import utils
39
from ganeti import bootstrap
40
from ganeti import ssh
41
from ganeti import objects
42
from ganeti import uidpool
43
from ganeti import compat
44
from ganeti import netutils
45

    
46

    
47
ON_OPT = cli_option("--on", default=False,
48
                    action="store_true", dest="on",
49
                    help="Recover from an EPO")
50

    
51
GROUPS_OPT = cli_option("--groups", default=False,
52
                    action="store_true", dest="groups",
53
                    help="Arguments are node groups instead of nodes")
54

    
55
SHOW_MACHINE_OPT = cli_option("-M", "--show-machine-names", default=False,
56
                              action="store_true",
57
                              help="Show machine name for every line in output")
58

    
59
_EPO_PING_INTERVAL = 30 # 30 seconds between pings
60
_EPO_PING_TIMEOUT = 1 # 1 second
61
_EPO_REACHABLE_TIMEOUT = 15 * 60 # 15 minutes
62

    
63

    
64
@UsesRPC
65
def InitCluster(opts, args):
66
  """Initialize the cluster.
67

68
  @param opts: the command line options selected by the user
69
  @type args: list
70
  @param args: should contain only one element, the desired
71
      cluster name
72
  @rtype: int
73
  @return: the desired exit code
74

75
  """
76
  if not opts.lvm_storage and opts.vg_name:
77
    ToStderr("Options --no-lvm-storage and --vg-name conflict.")
78
    return 1
79

    
80
  vg_name = opts.vg_name
81
  if opts.lvm_storage and not opts.vg_name:
82
    vg_name = constants.DEFAULT_VG
83

    
84
  if not opts.drbd_storage and opts.drbd_helper:
85
    ToStderr("Options --no-drbd-storage and --drbd-usermode-helper conflict.")
86
    return 1
87

    
88
  drbd_helper = opts.drbd_helper
89
  if opts.drbd_storage and not opts.drbd_helper:
90
    drbd_helper = constants.DEFAULT_DRBD_HELPER
91

    
92
  master_netdev = opts.master_netdev
93
  if master_netdev is None:
94
    master_netdev = constants.DEFAULT_BRIDGE
95

    
96
  hvlist = opts.enabled_hypervisors
97
  if hvlist is None:
98
    hvlist = constants.DEFAULT_ENABLED_HYPERVISOR
99
  hvlist = hvlist.split(",")
100

    
101
  hvparams = dict(opts.hvparams)
102
  beparams = opts.beparams
103
  nicparams = opts.nicparams
104

    
105
  diskparams = dict(opts.diskparams)
106

    
107
  # check the disk template types here, as we cannot rely on the type check done
108
  # by the opcode parameter types
109
  diskparams_keys = set(diskparams.keys())
110
  if not (diskparams_keys <= constants.DISK_TEMPLATES):
111
    unknown = utils.NiceSort(diskparams_keys - constants.DISK_TEMPLATES)
112
    ToStderr("Disk templates unknown: %s" % utils.CommaJoin(unknown))
113
    return 1
114

    
115
  # prepare beparams dict
116
  beparams = objects.FillDict(constants.BEC_DEFAULTS, beparams)
117
  utils.ForceDictType(beparams, constants.BES_PARAMETER_COMPAT)
118

    
119
  # prepare nicparams dict
120
  nicparams = objects.FillDict(constants.NICC_DEFAULTS, nicparams)
121
  utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
122

    
123
  # prepare ndparams dict
124
  if opts.ndparams is None:
125
    ndparams = dict(constants.NDC_DEFAULTS)
126
  else:
127
    ndparams = objects.FillDict(constants.NDC_DEFAULTS, opts.ndparams)
128
    utils.ForceDictType(ndparams, constants.NDS_PARAMETER_TYPES)
129

    
130
  # prepare hvparams dict
131
  for hv in constants.HYPER_TYPES:
132
    if hv not in hvparams:
133
      hvparams[hv] = {}
134
    hvparams[hv] = objects.FillDict(constants.HVC_DEFAULTS[hv], hvparams[hv])
135
    utils.ForceDictType(hvparams[hv], constants.HVS_PARAMETER_TYPES)
136

    
137
  # prepare diskparams dict
138
  for templ in constants.DISK_TEMPLATES:
139
    if templ not in diskparams:
140
      diskparams[templ] = {}
141
    diskparams[templ] = objects.FillDict(constants.DISK_DT_DEFAULTS[templ],
142
                                         diskparams[templ])
143
    utils.ForceDictType(diskparams[templ], constants.DISK_DT_TYPES)
144

    
145
  # prepare ipolicy dict
146
  ispecs_dts = opts.ipolicy_disk_templates # hate long var names
147
  ipolicy_raw = \
148
    objects.CreateIPolicyFromOpts(ispecs_mem_size=opts.ispecs_mem_size,
149
                                  ispecs_cpu_count=opts.ispecs_cpu_count,
150
                                  ispecs_disk_count=opts.ispecs_disk_count,
151
                                  ispecs_disk_size=opts.ispecs_disk_size,
152
                                  ispecs_nic_count=opts.ispecs_nic_count,
153
                                  ipolicy_disk_templates=ispecs_dts,
154
                                  fill_all=True)
155
  ipolicy = objects.FillIPolicy(constants.IPOLICY_DEFAULTS, ipolicy_raw)
156

    
157
  if opts.candidate_pool_size is None:
158
    opts.candidate_pool_size = constants.MASTER_POOL_SIZE_DEFAULT
159

    
160
  if opts.mac_prefix is None:
161
    opts.mac_prefix = constants.DEFAULT_MAC_PREFIX
162

    
163
  uid_pool = opts.uid_pool
164
  if uid_pool is not None:
165
    uid_pool = uidpool.ParseUidPool(uid_pool)
166

    
167
  if opts.prealloc_wipe_disks is None:
168
    opts.prealloc_wipe_disks = False
169

    
170
  external_ip_setup_script = opts.use_external_mip_script
171
  if external_ip_setup_script is None:
172
    external_ip_setup_script = False
173

    
174
  try:
175
    primary_ip_version = int(opts.primary_ip_version)
176
  except (ValueError, TypeError), err:
177
    ToStderr("Invalid primary ip version value: %s" % str(err))
178
    return 1
179

    
180
  master_netmask = opts.master_netmask
181
  try:
182
    if master_netmask is not None:
183
      master_netmask = int(master_netmask)
184
  except (ValueError, TypeError), err:
185
    ToStderr("Invalid master netmask value: %s" % str(err))
186
    return 1
187

    
188
  if opts.disk_state:
189
    disk_state = utils.FlatToDict(opts.disk_state)
190
  else:
191
    disk_state = {}
192

    
193
  hv_state = dict(opts.hv_state)
194

    
195
  bootstrap.InitCluster(cluster_name=args[0],
196
                        secondary_ip=opts.secondary_ip,
197
                        vg_name=vg_name,
198
                        mac_prefix=opts.mac_prefix,
199
                        master_netmask=master_netmask,
200
                        master_netdev=master_netdev,
201
                        file_storage_dir=opts.file_storage_dir,
202
                        shared_file_storage_dir=opts.shared_file_storage_dir,
203
                        enabled_hypervisors=hvlist,
204
                        hvparams=hvparams,
205
                        beparams=beparams,
206
                        nicparams=nicparams,
207
                        ndparams=ndparams,
208
                        diskparams=diskparams,
209
                        ipolicy=ipolicy,
210
                        candidate_pool_size=opts.candidate_pool_size,
211
                        modify_etc_hosts=opts.modify_etc_hosts,
212
                        modify_ssh_setup=opts.modify_ssh_setup,
213
                        maintain_node_health=opts.maintain_node_health,
214
                        drbd_helper=drbd_helper,
215
                        uid_pool=uid_pool,
216
                        default_iallocator=opts.default_iallocator,
217
                        primary_ip_version=primary_ip_version,
218
                        prealloc_wipe_disks=opts.prealloc_wipe_disks,
219
                        use_external_mip_script=external_ip_setup_script,
220
                        hv_state=hv_state,
221
                        disk_state=disk_state,
222
                        )
223
  op = opcodes.OpClusterPostInit()
224
  SubmitOpCode(op, opts=opts)
225
  return 0
226

    
227

    
228
@UsesRPC
229
def DestroyCluster(opts, args):
230
  """Destroy the cluster.
231

232
  @param opts: the command line options selected by the user
233
  @type args: list
234
  @param args: should be an empty list
235
  @rtype: int
236
  @return: the desired exit code
237

238
  """
239
  if not opts.yes_do_it:
240
    ToStderr("Destroying a cluster is irreversible. If you really want"
241
             " destroy this cluster, supply the --yes-do-it option.")
242
    return 1
243

    
244
  op = opcodes.OpClusterDestroy()
245
  master = SubmitOpCode(op, opts=opts)
246
  # if we reached this, the opcode didn't fail; we can proceed to
247
  # shutdown all the daemons
248
  bootstrap.FinalizeClusterDestroy(master)
249
  return 0
250

    
251

    
252
def RenameCluster(opts, args):
253
  """Rename the cluster.
254

255
  @param opts: the command line options selected by the user
256
  @type args: list
257
  @param args: should contain only one element, the new cluster name
258
  @rtype: int
259
  @return: the desired exit code
260

261
  """
262
  cl = GetClient()
263

    
264
  (cluster_name, ) = cl.QueryConfigValues(["cluster_name"])
265

    
266
  new_name = args[0]
267
  if not opts.force:
268
    usertext = ("This will rename the cluster from '%s' to '%s'. If you are"
269
                " connected over the network to the cluster name, the"
270
                " operation is very dangerous as the IP address will be"
271
                " removed from the node and the change may not go through."
272
                " Continue?") % (cluster_name, new_name)
273
    if not AskUser(usertext):
274
      return 1
275

    
276
  op = opcodes.OpClusterRename(name=new_name)
277
  result = SubmitOpCode(op, opts=opts, cl=cl)
278

    
279
  if result:
280
    ToStdout("Cluster renamed from '%s' to '%s'", cluster_name, result)
281

    
282
  return 0
283

    
284

    
285
def ActivateMasterIp(opts, args):
286
  """Activates the master IP.
287

288
  """
289
  op = opcodes.OpClusterActivateMasterIp()
290
  SubmitOpCode(op)
291
  return 0
292

    
293

    
294
def DeactivateMasterIp(opts, args):
295
  """Deactivates the master IP.
296

297
  """
298
  if not opts.confirm:
299
    usertext = ("This will disable the master IP. All the open connections to"
300
                " the master IP will be closed. To reach the master you will"
301
                " need to use its node IP."
302
                " Continue?")
303
    if not AskUser(usertext):
304
      return 1
305

    
306
  op = opcodes.OpClusterDeactivateMasterIp()
307
  SubmitOpCode(op)
308
  return 0
309

    
310

    
311
def RedistributeConfig(opts, args):
312
  """Forces push of the cluster configuration.
313

314
  @param opts: the command line options selected by the user
315
  @type args: list
316
  @param args: empty list
317
  @rtype: int
318
  @return: the desired exit code
319

320
  """
321
  op = opcodes.OpClusterRedistConf()
322
  SubmitOrSend(op, opts)
323
  return 0
324

    
325

    
326
def ShowClusterVersion(opts, args):
327
  """Write version of ganeti software to the standard output.
328

329
  @param opts: the command line options selected by the user
330
  @type args: list
331
  @param args: should be an empty list
332
  @rtype: int
333
  @return: the desired exit code
334

335
  """
336
  cl = GetClient()
337
  result = cl.QueryClusterInfo()
338
  ToStdout("Software version: %s", result["software_version"])
339
  ToStdout("Internode protocol: %s", result["protocol_version"])
340
  ToStdout("Configuration format: %s", result["config_version"])
341
  ToStdout("OS api version: %s", result["os_api_version"])
342
  ToStdout("Export interface: %s", result["export_version"])
343
  return 0
344

    
345

    
346
def ShowClusterMaster(opts, args):
347
  """Write name of master node to the standard output.
348

349
  @param opts: the command line options selected by the user
350
  @type args: list
351
  @param args: should be an empty list
352
  @rtype: int
353
  @return: the desired exit code
354

355
  """
356
  master = bootstrap.GetMaster()
357
  ToStdout(master)
358
  return 0
359

    
360

    
361
def _PrintGroupedParams(paramsdict, level=1, roman=False):
362
  """Print Grouped parameters (be, nic, disk) by group.
363

364
  @type paramsdict: dict of dicts
365
  @param paramsdict: {group: {param: value, ...}, ...}
366
  @type level: int
367
  @param level: Level of indention
368

369
  """
370
  indent = "  " * level
371
  for item, val in sorted(paramsdict.items()):
372
    if isinstance(val, dict):
373
      ToStdout("%s- %s:", indent, item)
374
      _PrintGroupedParams(val, level=level + 1, roman=roman)
375
    elif roman and isinstance(val, int):
376
      ToStdout("%s  %s: %s", indent, item, compat.TryToRoman(val))
377
    else:
378
      ToStdout("%s  %s: %s", indent, item, val)
379

    
380

    
381
def ShowClusterConfig(opts, args):
382
  """Shows cluster information.
383

384
  @param opts: the command line options selected by the user
385
  @type args: list
386
  @param args: should be an empty list
387
  @rtype: int
388
  @return: the desired exit code
389

390
  """
391
  cl = GetClient()
392
  result = cl.QueryClusterInfo()
393

    
394
  ToStdout("Cluster name: %s", result["name"])
395
  ToStdout("Cluster UUID: %s", result["uuid"])
396

    
397
  ToStdout("Creation time: %s", utils.FormatTime(result["ctime"]))
398
  ToStdout("Modification time: %s", utils.FormatTime(result["mtime"]))
399

    
400
  ToStdout("Master node: %s", result["master"])
401

    
402
  ToStdout("Architecture (this node): %s (%s)",
403
           result["architecture"][0], result["architecture"][1])
404

    
405
  if result["tags"]:
406
    tags = utils.CommaJoin(utils.NiceSort(result["tags"]))
407
  else:
408
    tags = "(none)"
409

    
410
  ToStdout("Tags: %s", tags)
411

    
412
  ToStdout("Default hypervisor: %s", result["default_hypervisor"])
413
  ToStdout("Enabled hypervisors: %s",
414
           utils.CommaJoin(result["enabled_hypervisors"]))
415

    
416
  ToStdout("Hypervisor parameters:")
417
  _PrintGroupedParams(result["hvparams"])
418

    
419
  ToStdout("OS-specific hypervisor parameters:")
420
  _PrintGroupedParams(result["os_hvp"])
421

    
422
  ToStdout("OS parameters:")
423
  _PrintGroupedParams(result["osparams"])
424

    
425
  ToStdout("Hidden OSes: %s", utils.CommaJoin(result["hidden_os"]))
426
  ToStdout("Blacklisted OSes: %s", utils.CommaJoin(result["blacklisted_os"]))
427

    
428
  ToStdout("Cluster parameters:")
429
  ToStdout("  - candidate pool size: %s",
430
            compat.TryToRoman(result["candidate_pool_size"],
431
                              convert=opts.roman_integers))
432
  ToStdout("  - master netdev: %s", result["master_netdev"])
433
  ToStdout("  - master netmask: %s", result["master_netmask"])
434
  ToStdout("  - use external master IP address setup script: %s",
435
           result["use_external_mip_script"])
436
  ToStdout("  - lvm volume group: %s", result["volume_group_name"])
437
  if result["reserved_lvs"]:
438
    reserved_lvs = utils.CommaJoin(result["reserved_lvs"])
439
  else:
440
    reserved_lvs = "(none)"
441
  ToStdout("  - lvm reserved volumes: %s", reserved_lvs)
442
  ToStdout("  - drbd usermode helper: %s", result["drbd_usermode_helper"])
443
  ToStdout("  - file storage path: %s", result["file_storage_dir"])
444
  ToStdout("  - shared file storage path: %s",
445
           result["shared_file_storage_dir"])
446
  ToStdout("  - maintenance of node health: %s",
447
           result["maintain_node_health"])
448
  ToStdout("  - uid pool: %s",
449
            uidpool.FormatUidPool(result["uid_pool"],
450
                                  roman=opts.roman_integers))
451
  ToStdout("  - default instance allocator: %s", result["default_iallocator"])
452
  ToStdout("  - primary ip version: %d", result["primary_ip_version"])
453
  ToStdout("  - preallocation wipe disks: %s", result["prealloc_wipe_disks"])
454
  ToStdout("  - OS search path: %s", utils.CommaJoin(constants.OS_SEARCH_PATH))
455

    
456
  ToStdout("Default node parameters:")
457
  _PrintGroupedParams(result["ndparams"], roman=opts.roman_integers)
458

    
459
  ToStdout("Default instance parameters:")
460
  _PrintGroupedParams(result["beparams"], roman=opts.roman_integers)
461

    
462
  ToStdout("Default nic parameters:")
463
  _PrintGroupedParams(result["nicparams"], roman=opts.roman_integers)
464

    
465
  ToStdout("Instance policy - limits for instances:")
466
  for key in constants.IPOLICY_ISPECS:
467
    ToStdout("  - %s", key)
468
    _PrintGroupedParams(result["ipolicy"][key], roman=opts.roman_integers)
469
  ToStdout("  - enabled disk templates: %s",
470
           utils.CommaJoin(result["ipolicy"][constants.IPOLICY_DTS]))
471
  for key in constants.IPOLICY_PARAMETERS:
472
    ToStdout("  - %s: %s", key, result["ipolicy"][key])
473

    
474
  return 0
475

    
476

    
477
def ClusterCopyFile(opts, args):
478
  """Copy a file from master to some nodes.
479

480
  @param opts: the command line options selected by the user
481
  @type args: list
482
  @param args: should contain only one element, the path of
483
      the file to be copied
484
  @rtype: int
485
  @return: the desired exit code
486

487
  """
488
  filename = args[0]
489
  if not os.path.exists(filename):
490
    raise errors.OpPrereqError("No such filename '%s'" % filename,
491
                               errors.ECODE_INVAL)
492

    
493
  cl = GetClient()
494

    
495
  cluster_name = cl.QueryConfigValues(["cluster_name"])[0]
496

    
497
  results = GetOnlineNodes(nodes=opts.nodes, cl=cl, filter_master=True,
498
                           secondary_ips=opts.use_replication_network,
499
                           nodegroup=opts.nodegroup)
500

    
501
  srun = ssh.SshRunner(cluster_name=cluster_name)
502
  for node in results:
503
    if not srun.CopyFileToNode(node, filename):
504
      ToStderr("Copy of file %s to node %s failed", filename, node)
505

    
506
  return 0
507

    
508

    
509
def RunClusterCommand(opts, args):
510
  """Run a command on some nodes.
511

512
  @param opts: the command line options selected by the user
513
  @type args: list
514
  @param args: should contain the command to be run and its arguments
515
  @rtype: int
516
  @return: the desired exit code
517

518
  """
519
  cl = GetClient()
520

    
521
  command = " ".join(args)
522

    
523
  nodes = GetOnlineNodes(nodes=opts.nodes, cl=cl, nodegroup=opts.nodegroup)
524

    
525
  cluster_name, master_node = cl.QueryConfigValues(["cluster_name",
526
                                                    "master_node"])
527

    
528
  srun = ssh.SshRunner(cluster_name=cluster_name)
529

    
530
  # Make sure master node is at list end
531
  if master_node in nodes:
532
    nodes.remove(master_node)
533
    nodes.append(master_node)
534

    
535
  for name in nodes:
536
    result = srun.Run(name, "root", command)
537
    ToStdout("------------------------------------------------")
538
    if opts.show_machine_names:
539
      for line in result.output.splitlines():
540
        ToStdout("%s: %s", name, line)
541
    else:
542
      ToStdout("node: %s", name)
543
      ToStdout("%s", result.output)
544
    ToStdout("return code = %s", result.exit_code)
545

    
546
  return 0
547

    
548

    
549
def VerifyCluster(opts, args):
550
  """Verify integrity of cluster, performing various test on nodes.
551

552
  @param opts: the command line options selected by the user
553
  @type args: list
554
  @param args: should be an empty list
555
  @rtype: int
556
  @return: the desired exit code
557

558
  """
559
  skip_checks = []
560

    
561
  if opts.skip_nplusone_mem:
562
    skip_checks.append(constants.VERIFY_NPLUSONE_MEM)
563

    
564
  cl = GetClient()
565

    
566
  op = opcodes.OpClusterVerify(verbose=opts.verbose,
567
                               error_codes=opts.error_codes,
568
                               debug_simulate_errors=opts.simulate_errors,
569
                               skip_checks=skip_checks,
570
                               ignore_errors=opts.ignore_errors,
571
                               group_name=opts.nodegroup)
572
  result = SubmitOpCode(op, cl=cl, opts=opts)
573

    
574
  # Keep track of submitted jobs
575
  jex = JobExecutor(cl=cl, opts=opts)
576

    
577
  for (status, job_id) in result[constants.JOB_IDS_KEY]:
578
    jex.AddJobId(None, status, job_id)
579

    
580
  results = jex.GetResults()
581

    
582
  (bad_jobs, bad_results) = \
583
    map(len,
584
        # Convert iterators to lists
585
        map(list,
586
            # Count errors
587
            map(compat.partial(itertools.ifilterfalse, bool),
588
                # Convert result to booleans in a tuple
589
                zip(*((job_success, len(op_results) == 1 and op_results[0])
590
                      for (job_success, op_results) in results)))))
591

    
592
  if bad_jobs == 0 and bad_results == 0:
593
    rcode = constants.EXIT_SUCCESS
594
  else:
595
    rcode = constants.EXIT_FAILURE
596
    if bad_jobs > 0:
597
      ToStdout("%s job(s) failed while verifying the cluster.", bad_jobs)
598

    
599
  return rcode
600

    
601

    
602
def VerifyDisks(opts, args):
603
  """Verify integrity of cluster disks.
604

605
  @param opts: the command line options selected by the user
606
  @type args: list
607
  @param args: should be an empty list
608
  @rtype: int
609
  @return: the desired exit code
610

611
  """
612
  cl = GetClient()
613

    
614
  op = opcodes.OpClusterVerifyDisks()
615

    
616
  result = SubmitOpCode(op, cl=cl, opts=opts)
617

    
618
  # Keep track of submitted jobs
619
  jex = JobExecutor(cl=cl, opts=opts)
620

    
621
  for (status, job_id) in result[constants.JOB_IDS_KEY]:
622
    jex.AddJobId(None, status, job_id)
623

    
624
  retcode = constants.EXIT_SUCCESS
625

    
626
  for (status, result) in jex.GetResults():
627
    if not status:
628
      ToStdout("Job failed: %s", result)
629
      continue
630

    
631
    ((bad_nodes, instances, missing), ) = result
632

    
633
    for node, text in bad_nodes.items():
634
      ToStdout("Error gathering data on node %s: %s",
635
               node, utils.SafeEncode(text[-400:]))
636
      retcode = constants.EXIT_FAILURE
637
      ToStdout("You need to fix these nodes first before fixing instances")
638

    
639
    for iname in instances:
640
      if iname in missing:
641
        continue
642
      op = opcodes.OpInstanceActivateDisks(instance_name=iname)
643
      try:
644
        ToStdout("Activating disks for instance '%s'", iname)
645
        SubmitOpCode(op, opts=opts, cl=cl)
646
      except errors.GenericError, err:
647
        nret, msg = FormatError(err)
648
        retcode |= nret
649
        ToStderr("Error activating disks for instance %s: %s", iname, msg)
650

    
651
    if missing:
652
      for iname, ival in missing.iteritems():
653
        all_missing = compat.all(x[0] in bad_nodes for x in ival)
654
        if all_missing:
655
          ToStdout("Instance %s cannot be verified as it lives on"
656
                   " broken nodes", iname)
657
        else:
658
          ToStdout("Instance %s has missing logical volumes:", iname)
659
          ival.sort()
660
          for node, vol in ival:
661
            if node in bad_nodes:
662
              ToStdout("\tbroken node %s /dev/%s", node, vol)
663
            else:
664
              ToStdout("\t%s /dev/%s", node, vol)
665

    
666
      ToStdout("You need to replace or recreate disks for all the above"
667
               " instances if this message persists after fixing broken nodes.")
668
      retcode = constants.EXIT_FAILURE
669

    
670
  return retcode
671

    
672

    
673
def RepairDiskSizes(opts, args):
674
  """Verify sizes of cluster disks.
675

676
  @param opts: the command line options selected by the user
677
  @type args: list
678
  @param args: optional list of instances to restrict check to
679
  @rtype: int
680
  @return: the desired exit code
681

682
  """
683
  op = opcodes.OpClusterRepairDiskSizes(instances=args)
684
  SubmitOpCode(op, opts=opts)
685

    
686

    
687
@UsesRPC
688
def MasterFailover(opts, args):
689
  """Failover the master node.
690

691
  This command, when run on a non-master node, will cause the current
692
  master to cease being master, and the non-master to become new
693
  master.
694

695
  @param opts: the command line options selected by the user
696
  @type args: list
697
  @param args: should be an empty list
698
  @rtype: int
699
  @return: the desired exit code
700

701
  """
702
  if opts.no_voting:
703
    usertext = ("This will perform the failover even if most other nodes"
704
                " are down, or if this node is outdated. This is dangerous"
705
                " as it can lead to a non-consistent cluster. Check the"
706
                " gnt-cluster(8) man page before proceeding. Continue?")
707
    if not AskUser(usertext):
708
      return 1
709

    
710
  return bootstrap.MasterFailover(no_voting=opts.no_voting)
711

    
712

    
713
def MasterPing(opts, args):
714
  """Checks if the master is alive.
715

716
  @param opts: the command line options selected by the user
717
  @type args: list
718
  @param args: should be an empty list
719
  @rtype: int
720
  @return: the desired exit code
721

722
  """
723
  try:
724
    cl = GetClient()
725
    cl.QueryClusterInfo()
726
    return 0
727
  except Exception: # pylint: disable=W0703
728
    return 1
729

    
730

    
731
def SearchTags(opts, args):
732
  """Searches the tags on all the cluster.
733

734
  @param opts: the command line options selected by the user
735
  @type args: list
736
  @param args: should contain only one element, the tag pattern
737
  @rtype: int
738
  @return: the desired exit code
739

740
  """
741
  op = opcodes.OpTagsSearch(pattern=args[0])
742
  result = SubmitOpCode(op, opts=opts)
743
  if not result:
744
    return 1
745
  result = list(result)
746
  result.sort()
747
  for path, tag in result:
748
    ToStdout("%s %s", path, tag)
749

    
750

    
751
def _ReadAndVerifyCert(cert_filename, verify_private_key=False):
752
  """Reads and verifies an X509 certificate.
753

754
  @type cert_filename: string
755
  @param cert_filename: the path of the file containing the certificate to
756
                        verify encoded in PEM format
757
  @type verify_private_key: bool
758
  @param verify_private_key: whether to verify the private key in addition to
759
                             the public certificate
760
  @rtype: string
761
  @return: a string containing the PEM-encoded certificate.
762

763
  """
764
  try:
765
    pem = utils.ReadFile(cert_filename)
766
  except IOError, err:
767
    raise errors.X509CertError(cert_filename,
768
                               "Unable to read certificate: %s" % str(err))
769

    
770
  try:
771
    OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, pem)
772
  except Exception, err:
773
    raise errors.X509CertError(cert_filename,
774
                               "Unable to load certificate: %s" % str(err))
775

    
776
  if verify_private_key:
777
    try:
778
      OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, pem)
779
    except Exception, err:
780
      raise errors.X509CertError(cert_filename,
781
                                 "Unable to load private key: %s" % str(err))
782

    
783
  return pem
784

    
785

    
786
def _RenewCrypto(new_cluster_cert, new_rapi_cert, #pylint: disable=R0911
787
                 rapi_cert_filename, new_spice_cert, spice_cert_filename,
788
                 spice_cacert_filename, new_confd_hmac_key, new_cds,
789
                 cds_filename, force):
790
  """Renews cluster certificates, keys and secrets.
791

792
  @type new_cluster_cert: bool
793
  @param new_cluster_cert: Whether to generate a new cluster certificate
794
  @type new_rapi_cert: bool
795
  @param new_rapi_cert: Whether to generate a new RAPI certificate
796
  @type rapi_cert_filename: string
797
  @param rapi_cert_filename: Path to file containing new RAPI certificate
798
  @type new_spice_cert: bool
799
  @param new_spice_cert: Whether to generate a new SPICE certificate
800
  @type spice_cert_filename: string
801
  @param spice_cert_filename: Path to file containing new SPICE certificate
802
  @type spice_cacert_filename: string
803
  @param spice_cacert_filename: Path to file containing the certificate of the
804
                                CA that signed the SPICE certificate
805
  @type new_confd_hmac_key: bool
806
  @param new_confd_hmac_key: Whether to generate a new HMAC key
807
  @type new_cds: bool
808
  @param new_cds: Whether to generate a new cluster domain secret
809
  @type cds_filename: string
810
  @param cds_filename: Path to file containing new cluster domain secret
811
  @type force: bool
812
  @param force: Whether to ask user for confirmation
813

814
  """
815
  if new_rapi_cert and rapi_cert_filename:
816
    ToStderr("Only one of the --new-rapi-certificate and --rapi-certificate"
817
             " options can be specified at the same time.")
818
    return 1
819

    
820
  if new_cds and cds_filename:
821
    ToStderr("Only one of the --new-cluster-domain-secret and"
822
             " --cluster-domain-secret options can be specified at"
823
             " the same time.")
824
    return 1
825

    
826
  if new_spice_cert and (spice_cert_filename or spice_cacert_filename):
827
    ToStderr("When using --new-spice-certificate, the --spice-certificate"
828
             " and --spice-ca-certificate must not be used.")
829
    return 1
830

    
831
  if bool(spice_cacert_filename) ^ bool(spice_cert_filename):
832
    ToStderr("Both --spice-certificate and --spice-ca-certificate must be"
833
             " specified.")
834
    return 1
835

    
836
  rapi_cert_pem, spice_cert_pem, spice_cacert_pem = (None, None, None)
837
  try:
838
    if rapi_cert_filename:
839
      rapi_cert_pem = _ReadAndVerifyCert(rapi_cert_filename, True)
840
    if spice_cert_filename:
841
      spice_cert_pem = _ReadAndVerifyCert(spice_cert_filename, True)
842
      spice_cacert_pem = _ReadAndVerifyCert(spice_cacert_filename)
843
  except errors.X509CertError, err:
844
    ToStderr("Unable to load X509 certificate from %s: %s", err[0], err[1])
845
    return 1
846

    
847
  if cds_filename:
848
    try:
849
      cds = utils.ReadFile(cds_filename)
850
    except Exception, err: # pylint: disable=W0703
851
      ToStderr("Can't load new cluster domain secret from %s: %s" %
852
               (cds_filename, str(err)))
853
      return 1
854
  else:
855
    cds = None
856

    
857
  if not force:
858
    usertext = ("This requires all daemons on all nodes to be restarted and"
859
                " may take some time. Continue?")
860
    if not AskUser(usertext):
861
      return 1
862

    
863
  def _RenewCryptoInner(ctx):
864
    ctx.feedback_fn("Updating certificates and keys")
865
    bootstrap.GenerateClusterCrypto(new_cluster_cert,
866
                                    new_rapi_cert,
867
                                    new_spice_cert,
868
                                    new_confd_hmac_key,
869
                                    new_cds,
870
                                    rapi_cert_pem=rapi_cert_pem,
871
                                    spice_cert_pem=spice_cert_pem,
872
                                    spice_cacert_pem=spice_cacert_pem,
873
                                    cds=cds)
874

    
875
    files_to_copy = []
876

    
877
    if new_cluster_cert:
878
      files_to_copy.append(constants.NODED_CERT_FILE)
879

    
880
    if new_rapi_cert or rapi_cert_pem:
881
      files_to_copy.append(constants.RAPI_CERT_FILE)
882

    
883
    if new_spice_cert or spice_cert_pem:
884
      files_to_copy.append(constants.SPICE_CERT_FILE)
885
      files_to_copy.append(constants.SPICE_CACERT_FILE)
886

    
887
    if new_confd_hmac_key:
888
      files_to_copy.append(constants.CONFD_HMAC_KEY)
889

    
890
    if new_cds or cds:
891
      files_to_copy.append(constants.CLUSTER_DOMAIN_SECRET_FILE)
892

    
893
    if files_to_copy:
894
      for node_name in ctx.nonmaster_nodes:
895
        ctx.feedback_fn("Copying %s to %s" %
896
                        (", ".join(files_to_copy), node_name))
897
        for file_name in files_to_copy:
898
          ctx.ssh.CopyFileToNode(node_name, file_name)
899

    
900
  RunWhileClusterStopped(ToStdout, _RenewCryptoInner)
901

    
902
  ToStdout("All requested certificates and keys have been replaced."
903
           " Running \"gnt-cluster verify\" now is recommended.")
904

    
905
  return 0
906

    
907

    
908
def RenewCrypto(opts, args):
909
  """Renews cluster certificates, keys and secrets.
910

911
  """
912
  return _RenewCrypto(opts.new_cluster_cert,
913
                      opts.new_rapi_cert,
914
                      opts.rapi_cert,
915
                      opts.new_spice_cert,
916
                      opts.spice_cert,
917
                      opts.spice_cacert,
918
                      opts.new_confd_hmac_key,
919
                      opts.new_cluster_domain_secret,
920
                      opts.cluster_domain_secret,
921
                      opts.force)
922

    
923

    
924
def SetClusterParams(opts, args):
925
  """Modify the cluster.
926

927
  @param opts: the command line options selected by the user
928
  @type args: list
929
  @param args: should be an empty list
930
  @rtype: int
931
  @return: the desired exit code
932

933
  """
934
  if not (not opts.lvm_storage or opts.vg_name or
935
          not opts.drbd_storage or opts.drbd_helper or
936
          opts.enabled_hypervisors or opts.hvparams or
937
          opts.beparams or opts.nicparams or
938
          opts.ndparams or opts.diskparams or
939
          opts.candidate_pool_size is not None or
940
          opts.uid_pool is not None or
941
          opts.maintain_node_health is not None or
942
          opts.add_uids is not None or
943
          opts.remove_uids is not None or
944
          opts.default_iallocator is not None or
945
          opts.reserved_lvs is not None or
946
          opts.master_netdev is not None or
947
          opts.master_netmask is not None or
948
          opts.use_external_mip_script is not None or
949
          opts.prealloc_wipe_disks is not None or
950
          opts.hv_state or
951
          opts.disk_state or
952
          opts.ispecs_mem_size is not None or
953
          opts.ispecs_cpu_count is not None or
954
          opts.ispecs_disk_count is not None or
955
          opts.ispecs_disk_size is not None or
956
          opts.ispecs_nic_count is not None):
957
    ToStderr("Please give at least one of the parameters.")
958
    return 1
959

    
960
  vg_name = opts.vg_name
961
  if not opts.lvm_storage and opts.vg_name:
962
    ToStderr("Options --no-lvm-storage and --vg-name conflict.")
963
    return 1
964

    
965
  if not opts.lvm_storage:
966
    vg_name = ""
967

    
968
  drbd_helper = opts.drbd_helper
969
  if not opts.drbd_storage and opts.drbd_helper:
970
    ToStderr("Options --no-drbd-storage and --drbd-usermode-helper conflict.")
971
    return 1
972

    
973
  if not opts.drbd_storage:
974
    drbd_helper = ""
975

    
976
  hvlist = opts.enabled_hypervisors
977
  if hvlist is not None:
978
    hvlist = hvlist.split(",")
979

    
980
  # a list of (name, dict) we can pass directly to dict() (or [])
981
  hvparams = dict(opts.hvparams)
982
  for hv_params in hvparams.values():
983
    utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
984

    
985
  diskparams = dict(opts.diskparams)
986

    
987
  for dt_params in diskparams.values():
988
    utils.ForceDictType(dt_params, constants.DISK_DT_TYPES)
989

    
990
  beparams = opts.beparams
991
  utils.ForceDictType(beparams, constants.BES_PARAMETER_COMPAT)
992

    
993
  nicparams = opts.nicparams
994
  utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
995

    
996
  ndparams = opts.ndparams
997
  if ndparams is not None:
998
    utils.ForceDictType(ndparams, constants.NDS_PARAMETER_TYPES)
999

    
1000
  ispecs_dts = opts.ipolicy_disk_templates
1001
  ipolicy = \
1002
    objects.CreateIPolicyFromOpts(ispecs_mem_size=opts.ispecs_mem_size,
1003
                                  ispecs_cpu_count=opts.ispecs_cpu_count,
1004
                                  ispecs_disk_count=opts.ispecs_disk_count,
1005
                                  ispecs_disk_size=opts.ispecs_disk_size,
1006
                                  ispecs_nic_count=opts.ispecs_nic_count,
1007
                                  ipolicy_disk_templates=ispecs_dts)
1008

    
1009
  mnh = opts.maintain_node_health
1010

    
1011
  uid_pool = opts.uid_pool
1012
  if uid_pool is not None:
1013
    uid_pool = uidpool.ParseUidPool(uid_pool)
1014

    
1015
  add_uids = opts.add_uids
1016
  if add_uids is not None:
1017
    add_uids = uidpool.ParseUidPool(add_uids)
1018

    
1019
  remove_uids = opts.remove_uids
1020
  if remove_uids is not None:
1021
    remove_uids = uidpool.ParseUidPool(remove_uids)
1022

    
1023
  if opts.reserved_lvs is not None:
1024
    if opts.reserved_lvs == "":
1025
      opts.reserved_lvs = []
1026
    else:
1027
      opts.reserved_lvs = utils.UnescapeAndSplit(opts.reserved_lvs, sep=",")
1028

    
1029
  if opts.master_netmask is not None:
1030
    try:
1031
      opts.master_netmask = int(opts.master_netmask)
1032
    except ValueError:
1033
      ToStderr("The --master-netmask option expects an int parameter.")
1034
      return 1
1035

    
1036
  ext_ip_script = opts.use_external_mip_script
1037

    
1038
  if opts.disk_state:
1039
    disk_state = utils.FlatToDict(opts.disk_state)
1040
  else:
1041
    disk_state = {}
1042

    
1043
  hv_state = dict(opts.hv_state)
1044

    
1045
  op = opcodes.OpClusterSetParams(vg_name=vg_name,
1046
                                  drbd_helper=drbd_helper,
1047
                                  enabled_hypervisors=hvlist,
1048
                                  hvparams=hvparams,
1049
                                  os_hvp=None,
1050
                                  beparams=beparams,
1051
                                  nicparams=nicparams,
1052
                                  ndparams=ndparams,
1053
                                  diskparams=diskparams,
1054
                                  ipolicy=ipolicy,
1055
                                  candidate_pool_size=opts.candidate_pool_size,
1056
                                  maintain_node_health=mnh,
1057
                                  uid_pool=uid_pool,
1058
                                  add_uids=add_uids,
1059
                                  remove_uids=remove_uids,
1060
                                  default_iallocator=opts.default_iallocator,
1061
                                  prealloc_wipe_disks=opts.prealloc_wipe_disks,
1062
                                  master_netdev=opts.master_netdev,
1063
                                  master_netmask=opts.master_netmask,
1064
                                  reserved_lvs=opts.reserved_lvs,
1065
                                  use_external_mip_script=ext_ip_script,
1066
                                  hv_state=hv_state,
1067
                                  disk_state=disk_state,
1068
                                  )
1069
  SubmitOpCode(op, opts=opts)
1070
  return 0
1071

    
1072

    
1073
def QueueOps(opts, args):
1074
  """Queue operations.
1075

1076
  @param opts: the command line options selected by the user
1077
  @type args: list
1078
  @param args: should contain only one element, the subcommand
1079
  @rtype: int
1080
  @return: the desired exit code
1081

1082
  """
1083
  command = args[0]
1084
  client = GetClient()
1085
  if command in ("drain", "undrain"):
1086
    drain_flag = command == "drain"
1087
    client.SetQueueDrainFlag(drain_flag)
1088
  elif command == "info":
1089
    result = client.QueryConfigValues(["drain_flag"])
1090
    if result[0]:
1091
      val = "set"
1092
    else:
1093
      val = "unset"
1094
    ToStdout("The drain flag is %s" % val)
1095
  else:
1096
    raise errors.OpPrereqError("Command '%s' is not valid." % command,
1097
                               errors.ECODE_INVAL)
1098

    
1099
  return 0
1100

    
1101

    
1102
def _ShowWatcherPause(until):
1103
  if until is None or until < time.time():
1104
    ToStdout("The watcher is not paused.")
1105
  else:
1106
    ToStdout("The watcher is paused until %s.", time.ctime(until))
1107

    
1108

    
1109
def WatcherOps(opts, args):
1110
  """Watcher operations.
1111

1112
  @param opts: the command line options selected by the user
1113
  @type args: list
1114
  @param args: should contain only one element, the subcommand
1115
  @rtype: int
1116
  @return: the desired exit code
1117

1118
  """
1119
  command = args[0]
1120
  client = GetClient()
1121

    
1122
  if command == "continue":
1123
    client.SetWatcherPause(None)
1124
    ToStdout("The watcher is no longer paused.")
1125

    
1126
  elif command == "pause":
1127
    if len(args) < 2:
1128
      raise errors.OpPrereqError("Missing pause duration", errors.ECODE_INVAL)
1129

    
1130
    result = client.SetWatcherPause(time.time() + ParseTimespec(args[1]))
1131
    _ShowWatcherPause(result)
1132

    
1133
  elif command == "info":
1134
    result = client.QueryConfigValues(["watcher_pause"])
1135
    _ShowWatcherPause(result[0])
1136

    
1137
  else:
1138
    raise errors.OpPrereqError("Command '%s' is not valid." % command,
1139
                               errors.ECODE_INVAL)
1140

    
1141
  return 0
1142

    
1143

    
1144
def _OobPower(opts, node_list, power):
1145
  """Puts the node in the list to desired power state.
1146

1147
  @param opts: The command line options selected by the user
1148
  @param node_list: The list of nodes to operate on
1149
  @param power: True if they should be powered on, False otherwise
1150
  @return: The success of the operation (none failed)
1151

1152
  """
1153
  if power:
1154
    command = constants.OOB_POWER_ON
1155
  else:
1156
    command = constants.OOB_POWER_OFF
1157

    
1158
  op = opcodes.OpOobCommand(node_names=node_list,
1159
                            command=command,
1160
                            ignore_status=True,
1161
                            timeout=opts.oob_timeout,
1162
                            power_delay=opts.power_delay)
1163
  result = SubmitOpCode(op, opts=opts)
1164
  errs = 0
1165
  for node_result in result:
1166
    (node_tuple, data_tuple) = node_result
1167
    (_, node_name) = node_tuple
1168
    (data_status, _) = data_tuple
1169
    if data_status != constants.RS_NORMAL:
1170
      assert data_status != constants.RS_UNAVAIL
1171
      errs += 1
1172
      ToStderr("There was a problem changing power for %s, please investigate",
1173
               node_name)
1174

    
1175
  if errs > 0:
1176
    return False
1177

    
1178
  return True
1179

    
1180

    
1181
def _InstanceStart(opts, inst_list, start):
1182
  """Puts the instances in the list to desired state.
1183

1184
  @param opts: The command line options selected by the user
1185
  @param inst_list: The list of instances to operate on
1186
  @param start: True if they should be started, False for shutdown
1187
  @return: The success of the operation (none failed)
1188

1189
  """
1190
  if start:
1191
    opcls = opcodes.OpInstanceStartup
1192
    text_submit, text_success, text_failed = ("startup", "started", "starting")
1193
  else:
1194
    opcls = compat.partial(opcodes.OpInstanceShutdown,
1195
                           timeout=opts.shutdown_timeout)
1196
    text_submit, text_success, text_failed = ("shutdown", "stopped", "stopping")
1197

    
1198
  jex = JobExecutor(opts=opts)
1199

    
1200
  for inst in inst_list:
1201
    ToStdout("Submit %s of instance %s", text_submit, inst)
1202
    op = opcls(instance_name=inst)
1203
    jex.QueueJob(inst, op)
1204

    
1205
  results = jex.GetResults()
1206
  bad_cnt = len([1 for (success, _) in results if not success])
1207

    
1208
  if bad_cnt == 0:
1209
    ToStdout("All instances have been %s successfully", text_success)
1210
  else:
1211
    ToStderr("There were errors while %s instances:\n"
1212
             "%d error(s) out of %d instance(s)", text_failed, bad_cnt,
1213
             len(results))
1214
    return False
1215

    
1216
  return True
1217

    
1218

    
1219
class _RunWhenNodesReachableHelper:
1220
  """Helper class to make shared internal state sharing easier.
1221

1222
  @ivar success: Indicates if all action_cb calls were successful
1223

1224
  """
1225
  def __init__(self, node_list, action_cb, node2ip, port, feedback_fn,
1226
               _ping_fn=netutils.TcpPing, _sleep_fn=time.sleep):
1227
    """Init the object.
1228

1229
    @param node_list: The list of nodes to be reachable
1230
    @param action_cb: Callback called when a new host is reachable
1231
    @type node2ip: dict
1232
    @param node2ip: Node to ip mapping
1233
    @param port: The port to use for the TCP ping
1234
    @param feedback_fn: The function used for feedback
1235
    @param _ping_fn: Function to check reachabilty (for unittest use only)
1236
    @param _sleep_fn: Function to sleep (for unittest use only)
1237

1238
    """
1239
    self.down = set(node_list)
1240
    self.up = set()
1241
    self.node2ip = node2ip
1242
    self.success = True
1243
    self.action_cb = action_cb
1244
    self.port = port
1245
    self.feedback_fn = feedback_fn
1246
    self._ping_fn = _ping_fn
1247
    self._sleep_fn = _sleep_fn
1248

    
1249
  def __call__(self):
1250
    """When called we run action_cb.
1251

1252
    @raises utils.RetryAgain: When there are still down nodes
1253

1254
    """
1255
    if not self.action_cb(self.up):
1256
      self.success = False
1257

    
1258
    if self.down:
1259
      raise utils.RetryAgain()
1260
    else:
1261
      return self.success
1262

    
1263
  def Wait(self, secs):
1264
    """Checks if a host is up or waits remaining seconds.
1265

1266
    @param secs: The secs remaining
1267

1268
    """
1269
    start = time.time()
1270
    for node in self.down:
1271
      if self._ping_fn(self.node2ip[node], self.port, timeout=_EPO_PING_TIMEOUT,
1272
                       live_port_needed=True):
1273
        self.feedback_fn("Node %s became available" % node)
1274
        self.up.add(node)
1275
        self.down -= self.up
1276
        # If we have a node available there is the possibility to run the
1277
        # action callback successfully, therefore we don't wait and return
1278
        return
1279

    
1280
    self._sleep_fn(max(0.0, start + secs - time.time()))
1281

    
1282

    
1283
def _RunWhenNodesReachable(node_list, action_cb, interval):
1284
  """Run action_cb when nodes become reachable.
1285

1286
  @param node_list: The list of nodes to be reachable
1287
  @param action_cb: Callback called when a new host is reachable
1288
  @param interval: The earliest time to retry
1289

1290
  """
1291
  client = GetClient()
1292
  cluster_info = client.QueryClusterInfo()
1293
  if cluster_info["primary_ip_version"] == constants.IP4_VERSION:
1294
    family = netutils.IPAddress.family
1295
  else:
1296
    family = netutils.IP6Address.family
1297

    
1298
  node2ip = dict((node, netutils.GetHostname(node, family=family).ip)
1299
                 for node in node_list)
1300

    
1301
  port = netutils.GetDaemonPort(constants.NODED)
1302
  helper = _RunWhenNodesReachableHelper(node_list, action_cb, node2ip, port,
1303
                                        ToStdout)
1304

    
1305
  try:
1306
    return utils.Retry(helper, interval, _EPO_REACHABLE_TIMEOUT,
1307
                       wait_fn=helper.Wait)
1308
  except utils.RetryTimeout:
1309
    ToStderr("Time exceeded while waiting for nodes to become reachable"
1310
             " again:\n  - %s", "  - ".join(helper.down))
1311
    return False
1312

    
1313

    
1314
def _MaybeInstanceStartup(opts, inst_map, nodes_online,
1315
                          _instance_start_fn=_InstanceStart):
1316
  """Start the instances conditional based on node_states.
1317

1318
  @param opts: The command line options selected by the user
1319
  @param inst_map: A dict of inst -> nodes mapping
1320
  @param nodes_online: A list of nodes online
1321
  @param _instance_start_fn: Callback to start instances (unittest use only)
1322
  @return: Success of the operation on all instances
1323

1324
  """
1325
  start_inst_list = []
1326
  for (inst, nodes) in inst_map.items():
1327
    if not (nodes - nodes_online):
1328
      # All nodes the instance lives on are back online
1329
      start_inst_list.append(inst)
1330

    
1331
  for inst in start_inst_list:
1332
    del inst_map[inst]
1333

    
1334
  if start_inst_list:
1335
    return _instance_start_fn(opts, start_inst_list, True)
1336

    
1337
  return True
1338

    
1339

    
1340
def _EpoOn(opts, full_node_list, node_list, inst_map):
1341
  """Does the actual power on.
1342

1343
  @param opts: The command line options selected by the user
1344
  @param full_node_list: All nodes to operate on (includes nodes not supporting
1345
                         OOB)
1346
  @param node_list: The list of nodes to operate on (all need to support OOB)
1347
  @param inst_map: A dict of inst -> nodes mapping
1348
  @return: The desired exit status
1349

1350
  """
1351
  if node_list and not _OobPower(opts, node_list, False):
1352
    ToStderr("Not all nodes seem to get back up, investigate and start"
1353
             " manually if needed")
1354

    
1355
  # Wait for the nodes to be back up
1356
  action_cb = compat.partial(_MaybeInstanceStartup, opts, dict(inst_map))
1357

    
1358
  ToStdout("Waiting until all nodes are available again")
1359
  if not _RunWhenNodesReachable(full_node_list, action_cb, _EPO_PING_INTERVAL):
1360
    ToStderr("Please investigate and start stopped instances manually")
1361
    return constants.EXIT_FAILURE
1362

    
1363
  return constants.EXIT_SUCCESS
1364

    
1365

    
1366
def _EpoOff(opts, node_list, inst_map):
1367
  """Does the actual power off.
1368

1369
  @param opts: The command line options selected by the user
1370
  @param node_list: The list of nodes to operate on (all need to support OOB)
1371
  @param inst_map: A dict of inst -> nodes mapping
1372
  @return: The desired exit status
1373

1374
  """
1375
  if not _InstanceStart(opts, inst_map.keys(), False):
1376
    ToStderr("Please investigate and stop instances manually before continuing")
1377
    return constants.EXIT_FAILURE
1378

    
1379
  if not node_list:
1380
    return constants.EXIT_SUCCESS
1381

    
1382
  if _OobPower(opts, node_list, False):
1383
    return constants.EXIT_SUCCESS
1384
  else:
1385
    return constants.EXIT_FAILURE
1386

    
1387

    
1388
def Epo(opts, args):
1389
  """EPO operations.
1390

1391
  @param opts: the command line options selected by the user
1392
  @type args: list
1393
  @param args: should contain only one element, the subcommand
1394
  @rtype: int
1395
  @return: the desired exit code
1396

1397
  """
1398
  if opts.groups and opts.show_all:
1399
    ToStderr("Only one of --groups or --all are allowed")
1400
    return constants.EXIT_FAILURE
1401
  elif args and opts.show_all:
1402
    ToStderr("Arguments in combination with --all are not allowed")
1403
    return constants.EXIT_FAILURE
1404

    
1405
  client = GetClient()
1406

    
1407
  if opts.groups:
1408
    node_query_list = itertools.chain(*client.QueryGroups(names=args,
1409
                                                          fields=["node_list"],
1410
                                                          use_locking=False))
1411
  else:
1412
    node_query_list = args
1413

    
1414
  result = client.QueryNodes(names=node_query_list,
1415
                             fields=["name", "master", "pinst_list",
1416
                                     "sinst_list", "powered", "offline"],
1417
                             use_locking=False)
1418
  node_list = []
1419
  inst_map = {}
1420
  for (idx, (node, master, pinsts, sinsts, powered,
1421
             offline)) in enumerate(result):
1422
    # Normalize the node_query_list as well
1423
    if not opts.show_all:
1424
      node_query_list[idx] = node
1425
    if not offline:
1426
      for inst in (pinsts + sinsts):
1427
        if inst in inst_map:
1428
          if not master:
1429
            inst_map[inst].add(node)
1430
        elif master:
1431
          inst_map[inst] = set()
1432
        else:
1433
          inst_map[inst] = set([node])
1434

    
1435
    if master and opts.on:
1436
      # We ignore the master for turning on the machines, in fact we are
1437
      # already operating on the master at this point :)
1438
      continue
1439
    elif master and not opts.show_all:
1440
      ToStderr("%s is the master node, please do a master-failover to another"
1441
               " node not affected by the EPO or use --all if you intend to"
1442
               " shutdown the whole cluster", node)
1443
      return constants.EXIT_FAILURE
1444
    elif powered is None:
1445
      ToStdout("Node %s does not support out-of-band handling, it can not be"
1446
               " handled in a fully automated manner", node)
1447
    elif powered == opts.on:
1448
      ToStdout("Node %s is already in desired power state, skipping", node)
1449
    elif not offline or (offline and powered):
1450
      node_list.append(node)
1451

    
1452
  if not opts.force and not ConfirmOperation(node_query_list, "nodes", "epo"):
1453
    return constants.EXIT_FAILURE
1454

    
1455
  if opts.on:
1456
    return _EpoOn(opts, node_query_list, node_list, inst_map)
1457
  else:
1458
    return _EpoOff(opts, node_list, inst_map)
1459

    
1460
commands = {
1461
  "init": (
1462
    InitCluster, [ArgHost(min=1, max=1)],
1463
    [BACKEND_OPT, CP_SIZE_OPT, ENABLED_HV_OPT, GLOBAL_FILEDIR_OPT,
1464
     HVLIST_OPT, MAC_PREFIX_OPT, MASTER_NETDEV_OPT, MASTER_NETMASK_OPT,
1465
     NIC_PARAMS_OPT, NOLVM_STORAGE_OPT, NOMODIFY_ETCHOSTS_OPT,
1466
     NOMODIFY_SSH_SETUP_OPT, SECONDARY_IP_OPT, VG_NAME_OPT,
1467
     MAINTAIN_NODE_HEALTH_OPT, UIDPOOL_OPT, DRBD_HELPER_OPT, NODRBD_STORAGE_OPT,
1468
     DEFAULT_IALLOCATOR_OPT, PRIMARY_IP_VERSION_OPT, PREALLOC_WIPE_DISKS_OPT,
1469
     NODE_PARAMS_OPT, GLOBAL_SHARED_FILEDIR_OPT, USE_EXTERNAL_MIP_SCRIPT,
1470
     DISK_PARAMS_OPT, HV_STATE_OPT, DISK_STATE_OPT] + INSTANCE_POLICY_OPTS,
1471
    "[opts...] <cluster_name>", "Initialises a new cluster configuration"),
1472
  "destroy": (
1473
    DestroyCluster, ARGS_NONE, [YES_DOIT_OPT],
1474
    "", "Destroy cluster"),
1475
  "rename": (
1476
    RenameCluster, [ArgHost(min=1, max=1)],
1477
    [FORCE_OPT, DRY_RUN_OPT],
1478
    "<new_name>",
1479
    "Renames the cluster"),
1480
  "redist-conf": (
1481
    RedistributeConfig, ARGS_NONE, [SUBMIT_OPT, DRY_RUN_OPT, PRIORITY_OPT],
1482
    "", "Forces a push of the configuration file and ssconf files"
1483
    " to the nodes in the cluster"),
1484
  "verify": (
1485
    VerifyCluster, ARGS_NONE,
1486
    [VERBOSE_OPT, DEBUG_SIMERR_OPT, ERROR_CODES_OPT, NONPLUS1_OPT,
1487
     DRY_RUN_OPT, PRIORITY_OPT, NODEGROUP_OPT, IGNORE_ERRORS_OPT],
1488
    "", "Does a check on the cluster configuration"),
1489
  "verify-disks": (
1490
    VerifyDisks, ARGS_NONE, [PRIORITY_OPT],
1491
    "", "Does a check on the cluster disk status"),
1492
  "repair-disk-sizes": (
1493
    RepairDiskSizes, ARGS_MANY_INSTANCES, [DRY_RUN_OPT, PRIORITY_OPT],
1494
    "[instance...]", "Updates mismatches in recorded disk sizes"),
1495
  "master-failover": (
1496
    MasterFailover, ARGS_NONE, [NOVOTING_OPT],
1497
    "", "Makes the current node the master"),
1498
  "master-ping": (
1499
    MasterPing, ARGS_NONE, [],
1500
    "", "Checks if the master is alive"),
1501
  "version": (
1502
    ShowClusterVersion, ARGS_NONE, [],
1503
    "", "Shows the cluster version"),
1504
  "getmaster": (
1505
    ShowClusterMaster, ARGS_NONE, [],
1506
    "", "Shows the cluster master"),
1507
  "copyfile": (
1508
    ClusterCopyFile, [ArgFile(min=1, max=1)],
1509
    [NODE_LIST_OPT, USE_REPL_NET_OPT, NODEGROUP_OPT],
1510
    "[-n node...] <filename>", "Copies a file to all (or only some) nodes"),
1511
  "command": (
1512
    RunClusterCommand, [ArgCommand(min=1)],
1513
    [NODE_LIST_OPT, NODEGROUP_OPT, SHOW_MACHINE_OPT],
1514
    "[-n node...] <command>", "Runs a command on all (or only some) nodes"),
1515
  "info": (
1516
    ShowClusterConfig, ARGS_NONE, [ROMAN_OPT],
1517
    "[--roman]", "Show cluster configuration"),
1518
  "list-tags": (
1519
    ListTags, ARGS_NONE, [], "", "List the tags of the cluster"),
1520
  "add-tags": (
1521
    AddTags, [ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT],
1522
    "tag...", "Add tags to the cluster"),
1523
  "remove-tags": (
1524
    RemoveTags, [ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT],
1525
    "tag...", "Remove tags from the cluster"),
1526
  "search-tags": (
1527
    SearchTags, [ArgUnknown(min=1, max=1)], [PRIORITY_OPT], "",
1528
    "Searches the tags on all objects on"
1529
    " the cluster for a given pattern (regex)"),
1530
  "queue": (
1531
    QueueOps,
1532
    [ArgChoice(min=1, max=1, choices=["drain", "undrain", "info"])],
1533
    [], "drain|undrain|info", "Change queue properties"),
1534
  "watcher": (
1535
    WatcherOps,
1536
    [ArgChoice(min=1, max=1, choices=["pause", "continue", "info"]),
1537
     ArgSuggest(min=0, max=1, choices=["30m", "1h", "4h"])],
1538
    [],
1539
    "{pause <timespec>|continue|info}", "Change watcher properties"),
1540
  "modify": (
1541
    SetClusterParams, ARGS_NONE,
1542
    [BACKEND_OPT, CP_SIZE_OPT, ENABLED_HV_OPT, HVLIST_OPT, MASTER_NETDEV_OPT,
1543
     MASTER_NETMASK_OPT, NIC_PARAMS_OPT, NOLVM_STORAGE_OPT, VG_NAME_OPT,
1544
     MAINTAIN_NODE_HEALTH_OPT, UIDPOOL_OPT, ADD_UIDS_OPT, REMOVE_UIDS_OPT,
1545
     DRBD_HELPER_OPT, NODRBD_STORAGE_OPT, DEFAULT_IALLOCATOR_OPT,
1546
     RESERVED_LVS_OPT, DRY_RUN_OPT, PRIORITY_OPT, PREALLOC_WIPE_DISKS_OPT,
1547
     NODE_PARAMS_OPT, USE_EXTERNAL_MIP_SCRIPT, DISK_PARAMS_OPT, HV_STATE_OPT,
1548
     DISK_STATE_OPT] +
1549
    INSTANCE_POLICY_OPTS,
1550
    "[opts...]",
1551
    "Alters the parameters of the cluster"),
1552
  "renew-crypto": (
1553
    RenewCrypto, ARGS_NONE,
1554
    [NEW_CLUSTER_CERT_OPT, NEW_RAPI_CERT_OPT, RAPI_CERT_OPT,
1555
     NEW_CONFD_HMAC_KEY_OPT, FORCE_OPT,
1556
     NEW_CLUSTER_DOMAIN_SECRET_OPT, CLUSTER_DOMAIN_SECRET_OPT,
1557
     NEW_SPICE_CERT_OPT, SPICE_CERT_OPT, SPICE_CACERT_OPT],
1558
    "[opts...]",
1559
    "Renews cluster certificates, keys and secrets"),
1560
  "epo": (
1561
    Epo, [ArgUnknown()],
1562
    [FORCE_OPT, ON_OPT, GROUPS_OPT, ALL_OPT, OOB_TIMEOUT_OPT,
1563
     SHUTDOWN_TIMEOUT_OPT, POWER_DELAY_OPT],
1564
    "[opts...] [args]",
1565
    "Performs an emergency power-off on given args"),
1566
  "activate-master-ip": (
1567
    ActivateMasterIp, ARGS_NONE, [], "", "Activates the master IP"),
1568
  "deactivate-master-ip": (
1569
    DeactivateMasterIp, ARGS_NONE, [CONFIRM_OPT], "",
1570
    "Deactivates the master IP"),
1571
  }
1572

    
1573

    
1574
#: dictionary with aliases for commands
1575
aliases = {
1576
  "masterfailover": "master-failover",
1577
}
1578

    
1579

    
1580
def Main():
1581
  return GenericMain(commands, override={"tag_type": constants.TAG_CLUSTER},
1582
                     aliases=aliases)