Statistics
| Branch: | Tag: | Revision:

root / lib / cmdlib.py @ a5c30dc2

History | View | Annotate | Download (436 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Module implementing the master-side code."""
23

    
24
# pylint: disable-msg=W0201,C0302
25

    
26
# W0201 since most LU attributes are defined in CheckPrereq or similar
27
# functions
28

    
29
# C0302: since we have waaaay to many lines in this module
30

    
31
import os
32
import os.path
33
import time
34
import re
35
import platform
36
import logging
37
import copy
38
import OpenSSL
39
import socket
40
import tempfile
41
import shutil
42
import itertools
43

    
44
from ganeti import ssh
45
from ganeti import utils
46
from ganeti import errors
47
from ganeti import hypervisor
48
from ganeti import locking
49
from ganeti import constants
50
from ganeti import objects
51
from ganeti import serializer
52
from ganeti import ssconf
53
from ganeti import uidpool
54
from ganeti import compat
55
from ganeti import masterd
56
from ganeti import netutils
57
from ganeti import query
58
from ganeti import qlang
59
from ganeti import opcodes
60
from ganeti import ht
61

    
62
import ganeti.masterd.instance # pylint: disable-msg=W0611
63

    
64

    
65
def _SupportsOob(cfg, node):
66
  """Tells if node supports OOB.
67

68
  @type cfg: L{config.ConfigWriter}
69
  @param cfg: The cluster configuration
70
  @type node: L{objects.Node}
71
  @param node: The node
72
  @return: The OOB script if supported or an empty string otherwise
73

74
  """
75
  return cfg.GetNdParams(node)[constants.ND_OOB_PROGRAM]
76

    
77

    
78
class ResultWithJobs:
79
  """Data container for LU results with jobs.
80

81
  Instances of this class returned from L{LogicalUnit.Exec} will be recognized
82
  by L{mcpu.Processor._ProcessResult}. The latter will then submit the jobs
83
  contained in the C{jobs} attribute and include the job IDs in the opcode
84
  result.
85

86
  """
87
  def __init__(self, jobs, **kwargs):
88
    """Initializes this class.
89

90
    Additional return values can be specified as keyword arguments.
91

92
    @type jobs: list of lists of L{opcode.OpCode}
93
    @param jobs: A list of lists of opcode objects
94

95
    """
96
    self.jobs = jobs
97
    self.other = kwargs
98

    
99

    
100
class LogicalUnit(object):
101
  """Logical Unit base class.
102

103
  Subclasses must follow these rules:
104
    - implement ExpandNames
105
    - implement CheckPrereq (except when tasklets are used)
106
    - implement Exec (except when tasklets are used)
107
    - implement BuildHooksEnv
108
    - implement BuildHooksNodes
109
    - redefine HPATH and HTYPE
110
    - optionally redefine their run requirements:
111
        REQ_BGL: the LU needs to hold the Big Ganeti Lock exclusively
112

113
  Note that all commands require root permissions.
114

115
  @ivar dry_run_result: the value (if any) that will be returned to the caller
116
      in dry-run mode (signalled by opcode dry_run parameter)
117

118
  """
119
  HPATH = None
120
  HTYPE = None
121
  REQ_BGL = True
122

    
123
  def __init__(self, processor, op, context, rpc):
124
    """Constructor for LogicalUnit.
125

126
    This needs to be overridden in derived classes in order to check op
127
    validity.
128

129
    """
130
    self.proc = processor
131
    self.op = op
132
    self.cfg = context.cfg
133
    self.glm = context.glm
134
    self.context = context
135
    self.rpc = rpc
136
    # Dicts used to declare locking needs to mcpu
137
    self.needed_locks = None
138
    self.share_locks = dict.fromkeys(locking.LEVELS, 0)
139
    self.add_locks = {}
140
    self.remove_locks = {}
141
    # Used to force good behavior when calling helper functions
142
    self.recalculate_locks = {}
143
    # logging
144
    self.Log = processor.Log # pylint: disable-msg=C0103
145
    self.LogWarning = processor.LogWarning # pylint: disable-msg=C0103
146
    self.LogInfo = processor.LogInfo # pylint: disable-msg=C0103
147
    self.LogStep = processor.LogStep # pylint: disable-msg=C0103
148
    # support for dry-run
149
    self.dry_run_result = None
150
    # support for generic debug attribute
151
    if (not hasattr(self.op, "debug_level") or
152
        not isinstance(self.op.debug_level, int)):
153
      self.op.debug_level = 0
154

    
155
    # Tasklets
156
    self.tasklets = None
157

    
158
    # Validate opcode parameters and set defaults
159
    self.op.Validate(True)
160

    
161
    self.CheckArguments()
162

    
163
  def CheckArguments(self):
164
    """Check syntactic validity for the opcode arguments.
165

166
    This method is for doing a simple syntactic check and ensure
167
    validity of opcode parameters, without any cluster-related
168
    checks. While the same can be accomplished in ExpandNames and/or
169
    CheckPrereq, doing these separate is better because:
170

171
      - ExpandNames is left as as purely a lock-related function
172
      - CheckPrereq is run after we have acquired locks (and possible
173
        waited for them)
174

175
    The function is allowed to change the self.op attribute so that
176
    later methods can no longer worry about missing parameters.
177

178
    """
179
    pass
180

    
181
  def ExpandNames(self):
182
    """Expand names for this LU.
183

184
    This method is called before starting to execute the opcode, and it should
185
    update all the parameters of the opcode to their canonical form (e.g. a
186
    short node name must be fully expanded after this method has successfully
187
    completed). This way locking, hooks, logging, etc. can work correctly.
188

189
    LUs which implement this method must also populate the self.needed_locks
190
    member, as a dict with lock levels as keys, and a list of needed lock names
191
    as values. Rules:
192

193
      - use an empty dict if you don't need any lock
194
      - if you don't need any lock at a particular level omit that level
195
      - don't put anything for the BGL level
196
      - if you want all locks at a level use locking.ALL_SET as a value
197

198
    If you need to share locks (rather than acquire them exclusively) at one
199
    level you can modify self.share_locks, setting a true value (usually 1) for
200
    that level. By default locks are not shared.
201

202
    This function can also define a list of tasklets, which then will be
203
    executed in order instead of the usual LU-level CheckPrereq and Exec
204
    functions, if those are not defined by the LU.
205

206
    Examples::
207

208
      # Acquire all nodes and one instance
209
      self.needed_locks = {
210
        locking.LEVEL_NODE: locking.ALL_SET,
211
        locking.LEVEL_INSTANCE: ['instance1.example.com'],
212
      }
213
      # Acquire just two nodes
214
      self.needed_locks = {
215
        locking.LEVEL_NODE: ['node1.example.com', 'node2.example.com'],
216
      }
217
      # Acquire no locks
218
      self.needed_locks = {} # No, you can't leave it to the default value None
219

220
    """
221
    # The implementation of this method is mandatory only if the new LU is
222
    # concurrent, so that old LUs don't need to be changed all at the same
223
    # time.
224
    if self.REQ_BGL:
225
      self.needed_locks = {} # Exclusive LUs don't need locks.
226
    else:
227
      raise NotImplementedError
228

    
229
  def DeclareLocks(self, level):
230
    """Declare LU locking needs for a level
231

232
    While most LUs can just declare their locking needs at ExpandNames time,
233
    sometimes there's the need to calculate some locks after having acquired
234
    the ones before. This function is called just before acquiring locks at a
235
    particular level, but after acquiring the ones at lower levels, and permits
236
    such calculations. It can be used to modify self.needed_locks, and by
237
    default it does nothing.
238

239
    This function is only called if you have something already set in
240
    self.needed_locks for the level.
241

242
    @param level: Locking level which is going to be locked
243
    @type level: member of ganeti.locking.LEVELS
244

245
    """
246

    
247
  def CheckPrereq(self):
248
    """Check prerequisites for this LU.
249

250
    This method should check that the prerequisites for the execution
251
    of this LU are fulfilled. It can do internode communication, but
252
    it should be idempotent - no cluster or system changes are
253
    allowed.
254

255
    The method should raise errors.OpPrereqError in case something is
256
    not fulfilled. Its return value is ignored.
257

258
    This method should also update all the parameters of the opcode to
259
    their canonical form if it hasn't been done by ExpandNames before.
260

261
    """
262
    if self.tasklets is not None:
263
      for (idx, tl) in enumerate(self.tasklets):
264
        logging.debug("Checking prerequisites for tasklet %s/%s",
265
                      idx + 1, len(self.tasklets))
266
        tl.CheckPrereq()
267
    else:
268
      pass
269

    
270
  def Exec(self, feedback_fn):
271
    """Execute the LU.
272

273
    This method should implement the actual work. It should raise
274
    errors.OpExecError for failures that are somewhat dealt with in
275
    code, or expected.
276

277
    """
278
    if self.tasklets is not None:
279
      for (idx, tl) in enumerate(self.tasklets):
280
        logging.debug("Executing tasklet %s/%s", idx + 1, len(self.tasklets))
281
        tl.Exec(feedback_fn)
282
    else:
283
      raise NotImplementedError
284

    
285
  def BuildHooksEnv(self):
286
    """Build hooks environment for this LU.
287

288
    @rtype: dict
289
    @return: Dictionary containing the environment that will be used for
290
      running the hooks for this LU. The keys of the dict must not be prefixed
291
      with "GANETI_"--that'll be added by the hooks runner. The hooks runner
292
      will extend the environment with additional variables. If no environment
293
      should be defined, an empty dictionary should be returned (not C{None}).
294
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
295
      will not be called.
296

297
    """
298
    raise NotImplementedError
299

    
300
  def BuildHooksNodes(self):
301
    """Build list of nodes to run LU's hooks.
302

303
    @rtype: tuple; (list, list)
304
    @return: Tuple containing a list of node names on which the hook
305
      should run before the execution and a list of node names on which the
306
      hook should run after the execution. No nodes should be returned as an
307
      empty list (and not None).
308
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
309
      will not be called.
310

311
    """
312
    raise NotImplementedError
313

    
314
  def HooksCallBack(self, phase, hook_results, feedback_fn, lu_result):
315
    """Notify the LU about the results of its hooks.
316

317
    This method is called every time a hooks phase is executed, and notifies
318
    the Logical Unit about the hooks' result. The LU can then use it to alter
319
    its result based on the hooks.  By default the method does nothing and the
320
    previous result is passed back unchanged but any LU can define it if it
321
    wants to use the local cluster hook-scripts somehow.
322

323
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
324
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
325
    @param hook_results: the results of the multi-node hooks rpc call
326
    @param feedback_fn: function used send feedback back to the caller
327
    @param lu_result: the previous Exec result this LU had, or None
328
        in the PRE phase
329
    @return: the new Exec result, based on the previous result
330
        and hook results
331

332
    """
333
    # API must be kept, thus we ignore the unused argument and could
334
    # be a function warnings
335
    # pylint: disable-msg=W0613,R0201
336
    return lu_result
337

    
338
  def _ExpandAndLockInstance(self):
339
    """Helper function to expand and lock an instance.
340

341
    Many LUs that work on an instance take its name in self.op.instance_name
342
    and need to expand it and then declare the expanded name for locking. This
343
    function does it, and then updates self.op.instance_name to the expanded
344
    name. It also initializes needed_locks as a dict, if this hasn't been done
345
    before.
346

347
    """
348
    if self.needed_locks is None:
349
      self.needed_locks = {}
350
    else:
351
      assert locking.LEVEL_INSTANCE not in self.needed_locks, \
352
        "_ExpandAndLockInstance called with instance-level locks set"
353
    self.op.instance_name = _ExpandInstanceName(self.cfg,
354
                                                self.op.instance_name)
355
    self.needed_locks[locking.LEVEL_INSTANCE] = self.op.instance_name
356

    
357
  def _LockInstancesNodes(self, primary_only=False):
358
    """Helper function to declare instances' nodes for locking.
359

360
    This function should be called after locking one or more instances to lock
361
    their nodes. Its effect is populating self.needed_locks[locking.LEVEL_NODE]
362
    with all primary or secondary nodes for instances already locked and
363
    present in self.needed_locks[locking.LEVEL_INSTANCE].
364

365
    It should be called from DeclareLocks, and for safety only works if
366
    self.recalculate_locks[locking.LEVEL_NODE] is set.
367

368
    In the future it may grow parameters to just lock some instance's nodes, or
369
    to just lock primaries or secondary nodes, if needed.
370

371
    If should be called in DeclareLocks in a way similar to::
372

373
      if level == locking.LEVEL_NODE:
374
        self._LockInstancesNodes()
375

376
    @type primary_only: boolean
377
    @param primary_only: only lock primary nodes of locked instances
378

379
    """
380
    assert locking.LEVEL_NODE in self.recalculate_locks, \
381
      "_LockInstancesNodes helper function called with no nodes to recalculate"
382

    
383
    # TODO: check if we're really been called with the instance locks held
384

    
385
    # For now we'll replace self.needed_locks[locking.LEVEL_NODE], but in the
386
    # future we might want to have different behaviors depending on the value
387
    # of self.recalculate_locks[locking.LEVEL_NODE]
388
    wanted_nodes = []
389
    for instance_name in self.glm.list_owned(locking.LEVEL_INSTANCE):
390
      instance = self.context.cfg.GetInstanceInfo(instance_name)
391
      wanted_nodes.append(instance.primary_node)
392
      if not primary_only:
393
        wanted_nodes.extend(instance.secondary_nodes)
394

    
395
    if self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_REPLACE:
396
      self.needed_locks[locking.LEVEL_NODE] = wanted_nodes
397
    elif self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_APPEND:
398
      self.needed_locks[locking.LEVEL_NODE].extend(wanted_nodes)
399

    
400
    del self.recalculate_locks[locking.LEVEL_NODE]
401

    
402

    
403
class NoHooksLU(LogicalUnit): # pylint: disable-msg=W0223
404
  """Simple LU which runs no hooks.
405

406
  This LU is intended as a parent for other LogicalUnits which will
407
  run no hooks, in order to reduce duplicate code.
408

409
  """
410
  HPATH = None
411
  HTYPE = None
412

    
413
  def BuildHooksEnv(self):
414
    """Empty BuildHooksEnv for NoHooksLu.
415

416
    This just raises an error.
417

418
    """
419
    raise AssertionError("BuildHooksEnv called for NoHooksLUs")
420

    
421
  def BuildHooksNodes(self):
422
    """Empty BuildHooksNodes for NoHooksLU.
423

424
    """
425
    raise AssertionError("BuildHooksNodes called for NoHooksLU")
426

    
427

    
428
class Tasklet:
429
  """Tasklet base class.
430

431
  Tasklets are subcomponents for LUs. LUs can consist entirely of tasklets or
432
  they can mix legacy code with tasklets. Locking needs to be done in the LU,
433
  tasklets know nothing about locks.
434

435
  Subclasses must follow these rules:
436
    - Implement CheckPrereq
437
    - Implement Exec
438

439
  """
440
  def __init__(self, lu):
441
    self.lu = lu
442

    
443
    # Shortcuts
444
    self.cfg = lu.cfg
445
    self.rpc = lu.rpc
446

    
447
  def CheckPrereq(self):
448
    """Check prerequisites for this tasklets.
449

450
    This method should check whether the prerequisites for the execution of
451
    this tasklet are fulfilled. It can do internode communication, but it
452
    should be idempotent - no cluster or system changes are allowed.
453

454
    The method should raise errors.OpPrereqError in case something is not
455
    fulfilled. Its return value is ignored.
456

457
    This method should also update all parameters to their canonical form if it
458
    hasn't been done before.
459

460
    """
461
    pass
462

    
463
  def Exec(self, feedback_fn):
464
    """Execute the tasklet.
465

466
    This method should implement the actual work. It should raise
467
    errors.OpExecError for failures that are somewhat dealt with in code, or
468
    expected.
469

470
    """
471
    raise NotImplementedError
472

    
473

    
474
class _QueryBase:
475
  """Base for query utility classes.
476

477
  """
478
  #: Attribute holding field definitions
479
  FIELDS = None
480

    
481
  def __init__(self, filter_, fields, use_locking):
482
    """Initializes this class.
483

484
    """
485
    self.use_locking = use_locking
486

    
487
    self.query = query.Query(self.FIELDS, fields, filter_=filter_,
488
                             namefield="name")
489
    self.requested_data = self.query.RequestedData()
490
    self.names = self.query.RequestedNames()
491

    
492
    # Sort only if no names were requested
493
    self.sort_by_name = not self.names
494

    
495
    self.do_locking = None
496
    self.wanted = None
497

    
498
  def _GetNames(self, lu, all_names, lock_level):
499
    """Helper function to determine names asked for in the query.
500

501
    """
502
    if self.do_locking:
503
      names = lu.glm.list_owned(lock_level)
504
    else:
505
      names = all_names
506

    
507
    if self.wanted == locking.ALL_SET:
508
      assert not self.names
509
      # caller didn't specify names, so ordering is not important
510
      return utils.NiceSort(names)
511

    
512
    # caller specified names and we must keep the same order
513
    assert self.names
514
    assert not self.do_locking or lu.glm.is_owned(lock_level)
515

    
516
    missing = set(self.wanted).difference(names)
517
    if missing:
518
      raise errors.OpExecError("Some items were removed before retrieving"
519
                               " their data: %s" % missing)
520

    
521
    # Return expanded names
522
    return self.wanted
523

    
524
  def ExpandNames(self, lu):
525
    """Expand names for this query.
526

527
    See L{LogicalUnit.ExpandNames}.
528

529
    """
530
    raise NotImplementedError()
531

    
532
  def DeclareLocks(self, lu, level):
533
    """Declare locks for this query.
534

535
    See L{LogicalUnit.DeclareLocks}.
536

537
    """
538
    raise NotImplementedError()
539

    
540
  def _GetQueryData(self, lu):
541
    """Collects all data for this query.
542

543
    @return: Query data object
544

545
    """
546
    raise NotImplementedError()
547

    
548
  def NewStyleQuery(self, lu):
549
    """Collect data and execute query.
550

551
    """
552
    return query.GetQueryResponse(self.query, self._GetQueryData(lu),
553
                                  sort_by_name=self.sort_by_name)
554

    
555
  def OldStyleQuery(self, lu):
556
    """Collect data and execute query.
557

558
    """
559
    return self.query.OldStyleQuery(self._GetQueryData(lu),
560
                                    sort_by_name=self.sort_by_name)
561

    
562

    
563
def _GetWantedNodes(lu, nodes):
564
  """Returns list of checked and expanded node names.
565

566
  @type lu: L{LogicalUnit}
567
  @param lu: the logical unit on whose behalf we execute
568
  @type nodes: list
569
  @param nodes: list of node names or None for all nodes
570
  @rtype: list
571
  @return: the list of nodes, sorted
572
  @raise errors.ProgrammerError: if the nodes parameter is wrong type
573

574
  """
575
  if nodes:
576
    return [_ExpandNodeName(lu.cfg, name) for name in nodes]
577

    
578
  return utils.NiceSort(lu.cfg.GetNodeList())
579

    
580

    
581
def _GetWantedInstances(lu, instances):
582
  """Returns list of checked and expanded instance names.
583

584
  @type lu: L{LogicalUnit}
585
  @param lu: the logical unit on whose behalf we execute
586
  @type instances: list
587
  @param instances: list of instance names or None for all instances
588
  @rtype: list
589
  @return: the list of instances, sorted
590
  @raise errors.OpPrereqError: if the instances parameter is wrong type
591
  @raise errors.OpPrereqError: if any of the passed instances is not found
592

593
  """
594
  if instances:
595
    wanted = [_ExpandInstanceName(lu.cfg, name) for name in instances]
596
  else:
597
    wanted = utils.NiceSort(lu.cfg.GetInstanceList())
598
  return wanted
599

    
600

    
601
def _GetUpdatedParams(old_params, update_dict,
602
                      use_default=True, use_none=False):
603
  """Return the new version of a parameter dictionary.
604

605
  @type old_params: dict
606
  @param old_params: old parameters
607
  @type update_dict: dict
608
  @param update_dict: dict containing new parameter values, or
609
      constants.VALUE_DEFAULT to reset the parameter to its default
610
      value
611
  @param use_default: boolean
612
  @type use_default: whether to recognise L{constants.VALUE_DEFAULT}
613
      values as 'to be deleted' values
614
  @param use_none: boolean
615
  @type use_none: whether to recognise C{None} values as 'to be
616
      deleted' values
617
  @rtype: dict
618
  @return: the new parameter dictionary
619

620
  """
621
  params_copy = copy.deepcopy(old_params)
622
  for key, val in update_dict.iteritems():
623
    if ((use_default and val == constants.VALUE_DEFAULT) or
624
        (use_none and val is None)):
625
      try:
626
        del params_copy[key]
627
      except KeyError:
628
        pass
629
    else:
630
      params_copy[key] = val
631
  return params_copy
632

    
633

    
634
def _ReleaseLocks(lu, level, names=None, keep=None):
635
  """Releases locks owned by an LU.
636

637
  @type lu: L{LogicalUnit}
638
  @param level: Lock level
639
  @type names: list or None
640
  @param names: Names of locks to release
641
  @type keep: list or None
642
  @param keep: Names of locks to retain
643

644
  """
645
  assert not (keep is not None and names is not None), \
646
         "Only one of the 'names' and the 'keep' parameters can be given"
647

    
648
  if names is not None:
649
    should_release = names.__contains__
650
  elif keep:
651
    should_release = lambda name: name not in keep
652
  else:
653
    should_release = None
654

    
655
  if should_release:
656
    retain = []
657
    release = []
658

    
659
    # Determine which locks to release
660
    for name in lu.glm.list_owned(level):
661
      if should_release(name):
662
        release.append(name)
663
      else:
664
        retain.append(name)
665

    
666
    assert len(lu.glm.list_owned(level)) == (len(retain) + len(release))
667

    
668
    # Release just some locks
669
    lu.glm.release(level, names=release)
670

    
671
    assert frozenset(lu.glm.list_owned(level)) == frozenset(retain)
672
  else:
673
    # Release everything
674
    lu.glm.release(level)
675

    
676
    assert not lu.glm.is_owned(level), "No locks should be owned"
677

    
678

    
679
def _RunPostHook(lu, node_name):
680
  """Runs the post-hook for an opcode on a single node.
681

682
  """
683
  hm = lu.proc.hmclass(lu.rpc.call_hooks_runner, lu)
684
  try:
685
    hm.RunPhase(constants.HOOKS_PHASE_POST, nodes=[node_name])
686
  except:
687
    # pylint: disable-msg=W0702
688
    lu.LogWarning("Errors occurred running hooks on %s" % node_name)
689

    
690

    
691
def _CheckOutputFields(static, dynamic, selected):
692
  """Checks whether all selected fields are valid.
693

694
  @type static: L{utils.FieldSet}
695
  @param static: static fields set
696
  @type dynamic: L{utils.FieldSet}
697
  @param dynamic: dynamic fields set
698

699
  """
700
  f = utils.FieldSet()
701
  f.Extend(static)
702
  f.Extend(dynamic)
703

    
704
  delta = f.NonMatching(selected)
705
  if delta:
706
    raise errors.OpPrereqError("Unknown output fields selected: %s"
707
                               % ",".join(delta), errors.ECODE_INVAL)
708

    
709

    
710
def _CheckGlobalHvParams(params):
711
  """Validates that given hypervisor params are not global ones.
712

713
  This will ensure that instances don't get customised versions of
714
  global params.
715

716
  """
717
  used_globals = constants.HVC_GLOBALS.intersection(params)
718
  if used_globals:
719
    msg = ("The following hypervisor parameters are global and cannot"
720
           " be customized at instance level, please modify them at"
721
           " cluster level: %s" % utils.CommaJoin(used_globals))
722
    raise errors.OpPrereqError(msg, errors.ECODE_INVAL)
723

    
724

    
725
def _CheckNodeOnline(lu, node, msg=None):
726
  """Ensure that a given node is online.
727

728
  @param lu: the LU on behalf of which we make the check
729
  @param node: the node to check
730
  @param msg: if passed, should be a message to replace the default one
731
  @raise errors.OpPrereqError: if the node is offline
732

733
  """
734
  if msg is None:
735
    msg = "Can't use offline node"
736
  if lu.cfg.GetNodeInfo(node).offline:
737
    raise errors.OpPrereqError("%s: %s" % (msg, node), errors.ECODE_STATE)
738

    
739

    
740
def _CheckNodeNotDrained(lu, node):
741
  """Ensure that a given node is not drained.
742

743
  @param lu: the LU on behalf of which we make the check
744
  @param node: the node to check
745
  @raise errors.OpPrereqError: if the node is drained
746

747
  """
748
  if lu.cfg.GetNodeInfo(node).drained:
749
    raise errors.OpPrereqError("Can't use drained node %s" % node,
750
                               errors.ECODE_STATE)
751

    
752

    
753
def _CheckNodeVmCapable(lu, node):
754
  """Ensure that a given node is vm capable.
755

756
  @param lu: the LU on behalf of which we make the check
757
  @param node: the node to check
758
  @raise errors.OpPrereqError: if the node is not vm capable
759

760
  """
761
  if not lu.cfg.GetNodeInfo(node).vm_capable:
762
    raise errors.OpPrereqError("Can't use non-vm_capable node %s" % node,
763
                               errors.ECODE_STATE)
764

    
765

    
766
def _CheckNodeHasOS(lu, node, os_name, force_variant):
767
  """Ensure that a node supports a given OS.
768

769
  @param lu: the LU on behalf of which we make the check
770
  @param node: the node to check
771
  @param os_name: the OS to query about
772
  @param force_variant: whether to ignore variant errors
773
  @raise errors.OpPrereqError: if the node is not supporting the OS
774

775
  """
776
  result = lu.rpc.call_os_get(node, os_name)
777
  result.Raise("OS '%s' not in supported OS list for node %s" %
778
               (os_name, node),
779
               prereq=True, ecode=errors.ECODE_INVAL)
780
  if not force_variant:
781
    _CheckOSVariant(result.payload, os_name)
782

    
783

    
784
def _CheckNodeHasSecondaryIP(lu, node, secondary_ip, prereq):
785
  """Ensure that a node has the given secondary ip.
786

787
  @type lu: L{LogicalUnit}
788
  @param lu: the LU on behalf of which we make the check
789
  @type node: string
790
  @param node: the node to check
791
  @type secondary_ip: string
792
  @param secondary_ip: the ip to check
793
  @type prereq: boolean
794
  @param prereq: whether to throw a prerequisite or an execute error
795
  @raise errors.OpPrereqError: if the node doesn't have the ip, and prereq=True
796
  @raise errors.OpExecError: if the node doesn't have the ip, and prereq=False
797

798
  """
799
  result = lu.rpc.call_node_has_ip_address(node, secondary_ip)
800
  result.Raise("Failure checking secondary ip on node %s" % node,
801
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
802
  if not result.payload:
803
    msg = ("Node claims it doesn't have the secondary ip you gave (%s),"
804
           " please fix and re-run this command" % secondary_ip)
805
    if prereq:
806
      raise errors.OpPrereqError(msg, errors.ECODE_ENVIRON)
807
    else:
808
      raise errors.OpExecError(msg)
809

    
810

    
811
def _GetClusterDomainSecret():
812
  """Reads the cluster domain secret.
813

814
  """
815
  return utils.ReadOneLineFile(constants.CLUSTER_DOMAIN_SECRET_FILE,
816
                               strict=True)
817

    
818

    
819
def _CheckInstanceDown(lu, instance, reason):
820
  """Ensure that an instance is not running."""
821
  if instance.admin_up:
822
    raise errors.OpPrereqError("Instance %s is marked to be up, %s" %
823
                               (instance.name, reason), errors.ECODE_STATE)
824

    
825
  pnode = instance.primary_node
826
  ins_l = lu.rpc.call_instance_list([pnode], [instance.hypervisor])[pnode]
827
  ins_l.Raise("Can't contact node %s for instance information" % pnode,
828
              prereq=True, ecode=errors.ECODE_ENVIRON)
829

    
830
  if instance.name in ins_l.payload:
831
    raise errors.OpPrereqError("Instance %s is running, %s" %
832
                               (instance.name, reason), errors.ECODE_STATE)
833

    
834

    
835
def _ExpandItemName(fn, name, kind):
836
  """Expand an item name.
837

838
  @param fn: the function to use for expansion
839
  @param name: requested item name
840
  @param kind: text description ('Node' or 'Instance')
841
  @return: the resolved (full) name
842
  @raise errors.OpPrereqError: if the item is not found
843

844
  """
845
  full_name = fn(name)
846
  if full_name is None:
847
    raise errors.OpPrereqError("%s '%s' not known" % (kind, name),
848
                               errors.ECODE_NOENT)
849
  return full_name
850

    
851

    
852
def _ExpandNodeName(cfg, name):
853
  """Wrapper over L{_ExpandItemName} for nodes."""
854
  return _ExpandItemName(cfg.ExpandNodeName, name, "Node")
855

    
856

    
857
def _ExpandInstanceName(cfg, name):
858
  """Wrapper over L{_ExpandItemName} for instance."""
859
  return _ExpandItemName(cfg.ExpandInstanceName, name, "Instance")
860

    
861

    
862
def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status,
863
                          memory, vcpus, nics, disk_template, disks,
864
                          bep, hvp, hypervisor_name):
865
  """Builds instance related env variables for hooks
866

867
  This builds the hook environment from individual variables.
868

869
  @type name: string
870
  @param name: the name of the instance
871
  @type primary_node: string
872
  @param primary_node: the name of the instance's primary node
873
  @type secondary_nodes: list
874
  @param secondary_nodes: list of secondary nodes as strings
875
  @type os_type: string
876
  @param os_type: the name of the instance's OS
877
  @type status: boolean
878
  @param status: the should_run status of the instance
879
  @type memory: string
880
  @param memory: the memory size of the instance
881
  @type vcpus: string
882
  @param vcpus: the count of VCPUs the instance has
883
  @type nics: list
884
  @param nics: list of tuples (ip, mac, mode, link) representing
885
      the NICs the instance has
886
  @type disk_template: string
887
  @param disk_template: the disk template of the instance
888
  @type disks: list
889
  @param disks: the list of (size, mode) pairs
890
  @type bep: dict
891
  @param bep: the backend parameters for the instance
892
  @type hvp: dict
893
  @param hvp: the hypervisor parameters for the instance
894
  @type hypervisor_name: string
895
  @param hypervisor_name: the hypervisor for the instance
896
  @rtype: dict
897
  @return: the hook environment for this instance
898

899
  """
900
  if status:
901
    str_status = "up"
902
  else:
903
    str_status = "down"
904
  env = {
905
    "OP_TARGET": name,
906
    "INSTANCE_NAME": name,
907
    "INSTANCE_PRIMARY": primary_node,
908
    "INSTANCE_SECONDARIES": " ".join(secondary_nodes),
909
    "INSTANCE_OS_TYPE": os_type,
910
    "INSTANCE_STATUS": str_status,
911
    "INSTANCE_MEMORY": memory,
912
    "INSTANCE_VCPUS": vcpus,
913
    "INSTANCE_DISK_TEMPLATE": disk_template,
914
    "INSTANCE_HYPERVISOR": hypervisor_name,
915
  }
916

    
917
  if nics:
918
    nic_count = len(nics)
919
    for idx, (ip, mac, mode, link) in enumerate(nics):
920
      if ip is None:
921
        ip = ""
922
      env["INSTANCE_NIC%d_IP" % idx] = ip
923
      env["INSTANCE_NIC%d_MAC" % idx] = mac
924
      env["INSTANCE_NIC%d_MODE" % idx] = mode
925
      env["INSTANCE_NIC%d_LINK" % idx] = link
926
      if mode == constants.NIC_MODE_BRIDGED:
927
        env["INSTANCE_NIC%d_BRIDGE" % idx] = link
928
  else:
929
    nic_count = 0
930

    
931
  env["INSTANCE_NIC_COUNT"] = nic_count
932

    
933
  if disks:
934
    disk_count = len(disks)
935
    for idx, (size, mode) in enumerate(disks):
936
      env["INSTANCE_DISK%d_SIZE" % idx] = size
937
      env["INSTANCE_DISK%d_MODE" % idx] = mode
938
  else:
939
    disk_count = 0
940

    
941
  env["INSTANCE_DISK_COUNT"] = disk_count
942

    
943
  for source, kind in [(bep, "BE"), (hvp, "HV")]:
944
    for key, value in source.items():
945
      env["INSTANCE_%s_%s" % (kind, key)] = value
946

    
947
  return env
948

    
949

    
950
def _NICListToTuple(lu, nics):
951
  """Build a list of nic information tuples.
952

953
  This list is suitable to be passed to _BuildInstanceHookEnv or as a return
954
  value in LUInstanceQueryData.
955

956
  @type lu:  L{LogicalUnit}
957
  @param lu: the logical unit on whose behalf we execute
958
  @type nics: list of L{objects.NIC}
959
  @param nics: list of nics to convert to hooks tuples
960

961
  """
962
  hooks_nics = []
963
  cluster = lu.cfg.GetClusterInfo()
964
  for nic in nics:
965
    ip = nic.ip
966
    mac = nic.mac
967
    filled_params = cluster.SimpleFillNIC(nic.nicparams)
968
    mode = filled_params[constants.NIC_MODE]
969
    link = filled_params[constants.NIC_LINK]
970
    hooks_nics.append((ip, mac, mode, link))
971
  return hooks_nics
972

    
973

    
974
def _BuildInstanceHookEnvByObject(lu, instance, override=None):
975
  """Builds instance related env variables for hooks from an object.
976

977
  @type lu: L{LogicalUnit}
978
  @param lu: the logical unit on whose behalf we execute
979
  @type instance: L{objects.Instance}
980
  @param instance: the instance for which we should build the
981
      environment
982
  @type override: dict
983
  @param override: dictionary with key/values that will override
984
      our values
985
  @rtype: dict
986
  @return: the hook environment dictionary
987

988
  """
989
  cluster = lu.cfg.GetClusterInfo()
990
  bep = cluster.FillBE(instance)
991
  hvp = cluster.FillHV(instance)
992
  args = {
993
    'name': instance.name,
994
    'primary_node': instance.primary_node,
995
    'secondary_nodes': instance.secondary_nodes,
996
    'os_type': instance.os,
997
    'status': instance.admin_up,
998
    'memory': bep[constants.BE_MEMORY],
999
    'vcpus': bep[constants.BE_VCPUS],
1000
    'nics': _NICListToTuple(lu, instance.nics),
1001
    'disk_template': instance.disk_template,
1002
    'disks': [(disk.size, disk.mode) for disk in instance.disks],
1003
    'bep': bep,
1004
    'hvp': hvp,
1005
    'hypervisor_name': instance.hypervisor,
1006
  }
1007
  if override:
1008
    args.update(override)
1009
  return _BuildInstanceHookEnv(**args) # pylint: disable-msg=W0142
1010

    
1011

    
1012
def _AdjustCandidatePool(lu, exceptions):
1013
  """Adjust the candidate pool after node operations.
1014

1015
  """
1016
  mod_list = lu.cfg.MaintainCandidatePool(exceptions)
1017
  if mod_list:
1018
    lu.LogInfo("Promoted nodes to master candidate role: %s",
1019
               utils.CommaJoin(node.name for node in mod_list))
1020
    for name in mod_list:
1021
      lu.context.ReaddNode(name)
1022
  mc_now, mc_max, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1023
  if mc_now > mc_max:
1024
    lu.LogInfo("Note: more nodes are candidates (%d) than desired (%d)" %
1025
               (mc_now, mc_max))
1026

    
1027

    
1028
def _DecideSelfPromotion(lu, exceptions=None):
1029
  """Decide whether I should promote myself as a master candidate.
1030

1031
  """
1032
  cp_size = lu.cfg.GetClusterInfo().candidate_pool_size
1033
  mc_now, mc_should, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1034
  # the new node will increase mc_max with one, so:
1035
  mc_should = min(mc_should + 1, cp_size)
1036
  return mc_now < mc_should
1037

    
1038

    
1039
def _CheckNicsBridgesExist(lu, target_nics, target_node):
1040
  """Check that the brigdes needed by a list of nics exist.
1041

1042
  """
1043
  cluster = lu.cfg.GetClusterInfo()
1044
  paramslist = [cluster.SimpleFillNIC(nic.nicparams) for nic in target_nics]
1045
  brlist = [params[constants.NIC_LINK] for params in paramslist
1046
            if params[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED]
1047
  if brlist:
1048
    result = lu.rpc.call_bridges_exist(target_node, brlist)
1049
    result.Raise("Error checking bridges on destination node '%s'" %
1050
                 target_node, prereq=True, ecode=errors.ECODE_ENVIRON)
1051

    
1052

    
1053
def _CheckInstanceBridgesExist(lu, instance, node=None):
1054
  """Check that the brigdes needed by an instance exist.
1055

1056
  """
1057
  if node is None:
1058
    node = instance.primary_node
1059
  _CheckNicsBridgesExist(lu, instance.nics, node)
1060

    
1061

    
1062
def _CheckOSVariant(os_obj, name):
1063
  """Check whether an OS name conforms to the os variants specification.
1064

1065
  @type os_obj: L{objects.OS}
1066
  @param os_obj: OS object to check
1067
  @type name: string
1068
  @param name: OS name passed by the user, to check for validity
1069

1070
  """
1071
  if not os_obj.supported_variants:
1072
    return
1073
  variant = objects.OS.GetVariant(name)
1074
  if not variant:
1075
    raise errors.OpPrereqError("OS name must include a variant",
1076
                               errors.ECODE_INVAL)
1077

    
1078
  if variant not in os_obj.supported_variants:
1079
    raise errors.OpPrereqError("Unsupported OS variant", errors.ECODE_INVAL)
1080

    
1081

    
1082
def _GetNodeInstancesInner(cfg, fn):
1083
  return [i for i in cfg.GetAllInstancesInfo().values() if fn(i)]
1084

    
1085

    
1086
def _GetNodeInstances(cfg, node_name):
1087
  """Returns a list of all primary and secondary instances on a node.
1088

1089
  """
1090

    
1091
  return _GetNodeInstancesInner(cfg, lambda inst: node_name in inst.all_nodes)
1092

    
1093

    
1094
def _GetNodePrimaryInstances(cfg, node_name):
1095
  """Returns primary instances on a node.
1096

1097
  """
1098
  return _GetNodeInstancesInner(cfg,
1099
                                lambda inst: node_name == inst.primary_node)
1100

    
1101

    
1102
def _GetNodeSecondaryInstances(cfg, node_name):
1103
  """Returns secondary instances on a node.
1104

1105
  """
1106
  return _GetNodeInstancesInner(cfg,
1107
                                lambda inst: node_name in inst.secondary_nodes)
1108

    
1109

    
1110
def _GetStorageTypeArgs(cfg, storage_type):
1111
  """Returns the arguments for a storage type.
1112

1113
  """
1114
  # Special case for file storage
1115
  if storage_type == constants.ST_FILE:
1116
    # storage.FileStorage wants a list of storage directories
1117
    return [[cfg.GetFileStorageDir(), cfg.GetSharedFileStorageDir()]]
1118

    
1119
  return []
1120

    
1121

    
1122
def _FindFaultyInstanceDisks(cfg, rpc, instance, node_name, prereq):
1123
  faulty = []
1124

    
1125
  for dev in instance.disks:
1126
    cfg.SetDiskID(dev, node_name)
1127

    
1128
  result = rpc.call_blockdev_getmirrorstatus(node_name, instance.disks)
1129
  result.Raise("Failed to get disk status from node %s" % node_name,
1130
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
1131

    
1132
  for idx, bdev_status in enumerate(result.payload):
1133
    if bdev_status and bdev_status.ldisk_status == constants.LDS_FAULTY:
1134
      faulty.append(idx)
1135

    
1136
  return faulty
1137

    
1138

    
1139
def _CheckIAllocatorOrNode(lu, iallocator_slot, node_slot):
1140
  """Check the sanity of iallocator and node arguments and use the
1141
  cluster-wide iallocator if appropriate.
1142

1143
  Check that at most one of (iallocator, node) is specified. If none is
1144
  specified, then the LU's opcode's iallocator slot is filled with the
1145
  cluster-wide default iallocator.
1146

1147
  @type iallocator_slot: string
1148
  @param iallocator_slot: the name of the opcode iallocator slot
1149
  @type node_slot: string
1150
  @param node_slot: the name of the opcode target node slot
1151

1152
  """
1153
  node = getattr(lu.op, node_slot, None)
1154
  iallocator = getattr(lu.op, iallocator_slot, None)
1155

    
1156
  if node is not None and iallocator is not None:
1157
    raise errors.OpPrereqError("Do not specify both, iallocator and node",
1158
                               errors.ECODE_INVAL)
1159
  elif node is None and iallocator is None:
1160
    default_iallocator = lu.cfg.GetDefaultIAllocator()
1161
    if default_iallocator:
1162
      setattr(lu.op, iallocator_slot, default_iallocator)
1163
    else:
1164
      raise errors.OpPrereqError("No iallocator or node given and no"
1165
                                 " cluster-wide default iallocator found;"
1166
                                 " please specify either an iallocator or a"
1167
                                 " node, or set a cluster-wide default"
1168
                                 " iallocator")
1169

    
1170

    
1171
class LUClusterPostInit(LogicalUnit):
1172
  """Logical unit for running hooks after cluster initialization.
1173

1174
  """
1175
  HPATH = "cluster-init"
1176
  HTYPE = constants.HTYPE_CLUSTER
1177

    
1178
  def BuildHooksEnv(self):
1179
    """Build hooks env.
1180

1181
    """
1182
    return {
1183
      "OP_TARGET": self.cfg.GetClusterName(),
1184
      }
1185

    
1186
  def BuildHooksNodes(self):
1187
    """Build hooks nodes.
1188

1189
    """
1190
    return ([], [self.cfg.GetMasterNode()])
1191

    
1192
  def Exec(self, feedback_fn):
1193
    """Nothing to do.
1194

1195
    """
1196
    return True
1197

    
1198

    
1199
class LUClusterDestroy(LogicalUnit):
1200
  """Logical unit for destroying the cluster.
1201

1202
  """
1203
  HPATH = "cluster-destroy"
1204
  HTYPE = constants.HTYPE_CLUSTER
1205

    
1206
  def BuildHooksEnv(self):
1207
    """Build hooks env.
1208

1209
    """
1210
    return {
1211
      "OP_TARGET": self.cfg.GetClusterName(),
1212
      }
1213

    
1214
  def BuildHooksNodes(self):
1215
    """Build hooks nodes.
1216

1217
    """
1218
    return ([], [])
1219

    
1220
  def CheckPrereq(self):
1221
    """Check prerequisites.
1222

1223
    This checks whether the cluster is empty.
1224

1225
    Any errors are signaled by raising errors.OpPrereqError.
1226

1227
    """
1228
    master = self.cfg.GetMasterNode()
1229

    
1230
    nodelist = self.cfg.GetNodeList()
1231
    if len(nodelist) != 1 or nodelist[0] != master:
1232
      raise errors.OpPrereqError("There are still %d node(s) in"
1233
                                 " this cluster." % (len(nodelist) - 1),
1234
                                 errors.ECODE_INVAL)
1235
    instancelist = self.cfg.GetInstanceList()
1236
    if instancelist:
1237
      raise errors.OpPrereqError("There are still %d instance(s) in"
1238
                                 " this cluster." % len(instancelist),
1239
                                 errors.ECODE_INVAL)
1240

    
1241
  def Exec(self, feedback_fn):
1242
    """Destroys the cluster.
1243

1244
    """
1245
    master = self.cfg.GetMasterNode()
1246

    
1247
    # Run post hooks on master node before it's removed
1248
    _RunPostHook(self, master)
1249

    
1250
    result = self.rpc.call_node_stop_master(master, False)
1251
    result.Raise("Could not disable the master role")
1252

    
1253
    return master
1254

    
1255

    
1256
def _VerifyCertificate(filename):
1257
  """Verifies a certificate for LUClusterVerify.
1258

1259
  @type filename: string
1260
  @param filename: Path to PEM file
1261

1262
  """
1263
  try:
1264
    cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
1265
                                           utils.ReadFile(filename))
1266
  except Exception, err: # pylint: disable-msg=W0703
1267
    return (LUClusterVerify.ETYPE_ERROR,
1268
            "Failed to load X509 certificate %s: %s" % (filename, err))
1269

    
1270
  (errcode, msg) = \
1271
    utils.VerifyX509Certificate(cert, constants.SSL_CERT_EXPIRATION_WARN,
1272
                                constants.SSL_CERT_EXPIRATION_ERROR)
1273

    
1274
  if msg:
1275
    fnamemsg = "While verifying %s: %s" % (filename, msg)
1276
  else:
1277
    fnamemsg = None
1278

    
1279
  if errcode is None:
1280
    return (None, fnamemsg)
1281
  elif errcode == utils.CERT_WARNING:
1282
    return (LUClusterVerify.ETYPE_WARNING, fnamemsg)
1283
  elif errcode == utils.CERT_ERROR:
1284
    return (LUClusterVerify.ETYPE_ERROR, fnamemsg)
1285

    
1286
  raise errors.ProgrammerError("Unhandled certificate error code %r" % errcode)
1287

    
1288

    
1289
class _VerifyErrors(object):
1290
  """Mix-in for cluster/group verify LUs.
1291

1292
  It provides _Error and _ErrorIf, and updates the self.bad boolean. (Expects
1293
  self.op and self._feedback_fn to be available.)
1294

1295
  """
1296
  TCLUSTER = "cluster"
1297
  TNODE = "node"
1298
  TINSTANCE = "instance"
1299

    
1300
  ECLUSTERCFG = (TCLUSTER, "ECLUSTERCFG")
1301
  ECLUSTERCERT = (TCLUSTER, "ECLUSTERCERT")
1302
  ECLUSTERFILECHECK = (TCLUSTER, "ECLUSTERFILECHECK")
1303
  EINSTANCEBADNODE = (TINSTANCE, "EINSTANCEBADNODE")
1304
  EINSTANCEDOWN = (TINSTANCE, "EINSTANCEDOWN")
1305
  EINSTANCELAYOUT = (TINSTANCE, "EINSTANCELAYOUT")
1306
  EINSTANCEMISSINGDISK = (TINSTANCE, "EINSTANCEMISSINGDISK")
1307
  EINSTANCEFAULTYDISK = (TINSTANCE, "EINSTANCEFAULTYDISK")
1308
  EINSTANCEWRONGNODE = (TINSTANCE, "EINSTANCEWRONGNODE")
1309
  EINSTANCESPLITGROUPS = (TINSTANCE, "EINSTANCESPLITGROUPS")
1310
  ENODEDRBD = (TNODE, "ENODEDRBD")
1311
  ENODEDRBDHELPER = (TNODE, "ENODEDRBDHELPER")
1312
  ENODEFILECHECK = (TNODE, "ENODEFILECHECK")
1313
  ENODEHOOKS = (TNODE, "ENODEHOOKS")
1314
  ENODEHV = (TNODE, "ENODEHV")
1315
  ENODELVM = (TNODE, "ENODELVM")
1316
  ENODEN1 = (TNODE, "ENODEN1")
1317
  ENODENET = (TNODE, "ENODENET")
1318
  ENODEOS = (TNODE, "ENODEOS")
1319
  ENODEORPHANINSTANCE = (TNODE, "ENODEORPHANINSTANCE")
1320
  ENODEORPHANLV = (TNODE, "ENODEORPHANLV")
1321
  ENODERPC = (TNODE, "ENODERPC")
1322
  ENODESSH = (TNODE, "ENODESSH")
1323
  ENODEVERSION = (TNODE, "ENODEVERSION")
1324
  ENODESETUP = (TNODE, "ENODESETUP")
1325
  ENODETIME = (TNODE, "ENODETIME")
1326
  ENODEOOBPATH = (TNODE, "ENODEOOBPATH")
1327

    
1328
  ETYPE_FIELD = "code"
1329
  ETYPE_ERROR = "ERROR"
1330
  ETYPE_WARNING = "WARNING"
1331

    
1332
  def _Error(self, ecode, item, msg, *args, **kwargs):
1333
    """Format an error message.
1334

1335
    Based on the opcode's error_codes parameter, either format a
1336
    parseable error code, or a simpler error string.
1337

1338
    This must be called only from Exec and functions called from Exec.
1339

1340
    """
1341
    ltype = kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR)
1342
    itype, etxt = ecode
1343
    # first complete the msg
1344
    if args:
1345
      msg = msg % args
1346
    # then format the whole message
1347
    if self.op.error_codes: # This is a mix-in. pylint: disable-msg=E1101
1348
      msg = "%s:%s:%s:%s:%s" % (ltype, etxt, itype, item, msg)
1349
    else:
1350
      if item:
1351
        item = " " + item
1352
      else:
1353
        item = ""
1354
      msg = "%s: %s%s: %s" % (ltype, itype, item, msg)
1355
    # and finally report it via the feedback_fn
1356
    self._feedback_fn("  - %s" % msg) # Mix-in. pylint: disable-msg=E1101
1357

    
1358
  def _ErrorIf(self, cond, *args, **kwargs):
1359
    """Log an error message if the passed condition is True.
1360

1361
    """
1362
    cond = (bool(cond)
1363
            or self.op.debug_simulate_errors) # pylint: disable-msg=E1101
1364
    if cond:
1365
      self._Error(*args, **kwargs)
1366
    # do not mark the operation as failed for WARN cases only
1367
    if kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR) == self.ETYPE_ERROR:
1368
      self.bad = self.bad or cond
1369

    
1370

    
1371
class LUClusterVerify(LogicalUnit, _VerifyErrors):
1372
  """Verifies the cluster status.
1373

1374
  """
1375
  HPATH = "cluster-verify"
1376
  HTYPE = constants.HTYPE_CLUSTER
1377
  REQ_BGL = False
1378

    
1379
  _HOOKS_INDENT_RE = re.compile("^", re.M)
1380

    
1381
  class NodeImage(object):
1382
    """A class representing the logical and physical status of a node.
1383

1384
    @type name: string
1385
    @ivar name: the node name to which this object refers
1386
    @ivar volumes: a structure as returned from
1387
        L{ganeti.backend.GetVolumeList} (runtime)
1388
    @ivar instances: a list of running instances (runtime)
1389
    @ivar pinst: list of configured primary instances (config)
1390
    @ivar sinst: list of configured secondary instances (config)
1391
    @ivar sbp: dictionary of {primary-node: list of instances} for all
1392
        instances for which this node is secondary (config)
1393
    @ivar mfree: free memory, as reported by hypervisor (runtime)
1394
    @ivar dfree: free disk, as reported by the node (runtime)
1395
    @ivar offline: the offline status (config)
1396
    @type rpc_fail: boolean
1397
    @ivar rpc_fail: whether the RPC verify call was successfull (overall,
1398
        not whether the individual keys were correct) (runtime)
1399
    @type lvm_fail: boolean
1400
    @ivar lvm_fail: whether the RPC call didn't return valid LVM data
1401
    @type hyp_fail: boolean
1402
    @ivar hyp_fail: whether the RPC call didn't return the instance list
1403
    @type ghost: boolean
1404
    @ivar ghost: whether this is a known node or not (config)
1405
    @type os_fail: boolean
1406
    @ivar os_fail: whether the RPC call didn't return valid OS data
1407
    @type oslist: list
1408
    @ivar oslist: list of OSes as diagnosed by DiagnoseOS
1409
    @type vm_capable: boolean
1410
    @ivar vm_capable: whether the node can host instances
1411

1412
    """
1413
    def __init__(self, offline=False, name=None, vm_capable=True):
1414
      self.name = name
1415
      self.volumes = {}
1416
      self.instances = []
1417
      self.pinst = []
1418
      self.sinst = []
1419
      self.sbp = {}
1420
      self.mfree = 0
1421
      self.dfree = 0
1422
      self.offline = offline
1423
      self.vm_capable = vm_capable
1424
      self.rpc_fail = False
1425
      self.lvm_fail = False
1426
      self.hyp_fail = False
1427
      self.ghost = False
1428
      self.os_fail = False
1429
      self.oslist = {}
1430

    
1431
  def ExpandNames(self):
1432
    self.needed_locks = {
1433
      locking.LEVEL_NODE: locking.ALL_SET,
1434
      locking.LEVEL_INSTANCE: locking.ALL_SET,
1435
    }
1436
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
1437

    
1438
  def CheckPrereq(self):
1439
    self.all_node_info = self.cfg.GetAllNodesInfo()
1440
    self.all_inst_info = self.cfg.GetAllInstancesInfo()
1441
    self.my_node_names = utils.NiceSort(list(self.all_node_info))
1442
    self.my_node_info = self.all_node_info
1443
    self.my_inst_names = utils.NiceSort(list(self.all_inst_info))
1444
    self.my_inst_info = self.all_inst_info
1445

    
1446
  def _VerifyNode(self, ninfo, nresult):
1447
    """Perform some basic validation on data returned from a node.
1448

1449
      - check the result data structure is well formed and has all the
1450
        mandatory fields
1451
      - check ganeti version
1452

1453
    @type ninfo: L{objects.Node}
1454
    @param ninfo: the node to check
1455
    @param nresult: the results from the node
1456
    @rtype: boolean
1457
    @return: whether overall this call was successful (and we can expect
1458
         reasonable values in the respose)
1459

1460
    """
1461
    node = ninfo.name
1462
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1463

    
1464
    # main result, nresult should be a non-empty dict
1465
    test = not nresult or not isinstance(nresult, dict)
1466
    _ErrorIf(test, self.ENODERPC, node,
1467
                  "unable to verify node: no data returned")
1468
    if test:
1469
      return False
1470

    
1471
    # compares ganeti version
1472
    local_version = constants.PROTOCOL_VERSION
1473
    remote_version = nresult.get("version", None)
1474
    test = not (remote_version and
1475
                isinstance(remote_version, (list, tuple)) and
1476
                len(remote_version) == 2)
1477
    _ErrorIf(test, self.ENODERPC, node,
1478
             "connection to node returned invalid data")
1479
    if test:
1480
      return False
1481

    
1482
    test = local_version != remote_version[0]
1483
    _ErrorIf(test, self.ENODEVERSION, node,
1484
             "incompatible protocol versions: master %s,"
1485
             " node %s", local_version, remote_version[0])
1486
    if test:
1487
      return False
1488

    
1489
    # node seems compatible, we can actually try to look into its results
1490

    
1491
    # full package version
1492
    self._ErrorIf(constants.RELEASE_VERSION != remote_version[1],
1493
                  self.ENODEVERSION, node,
1494
                  "software version mismatch: master %s, node %s",
1495
                  constants.RELEASE_VERSION, remote_version[1],
1496
                  code=self.ETYPE_WARNING)
1497

    
1498
    hyp_result = nresult.get(constants.NV_HYPERVISOR, None)
1499
    if ninfo.vm_capable and isinstance(hyp_result, dict):
1500
      for hv_name, hv_result in hyp_result.iteritems():
1501
        test = hv_result is not None
1502
        _ErrorIf(test, self.ENODEHV, node,
1503
                 "hypervisor %s verify failure: '%s'", hv_name, hv_result)
1504

    
1505
    hvp_result = nresult.get(constants.NV_HVPARAMS, None)
1506
    if ninfo.vm_capable and isinstance(hvp_result, list):
1507
      for item, hv_name, hv_result in hvp_result:
1508
        _ErrorIf(True, self.ENODEHV, node,
1509
                 "hypervisor %s parameter verify failure (source %s): %s",
1510
                 hv_name, item, hv_result)
1511

    
1512
    test = nresult.get(constants.NV_NODESETUP,
1513
                       ["Missing NODESETUP results"])
1514
    _ErrorIf(test, self.ENODESETUP, node, "node setup error: %s",
1515
             "; ".join(test))
1516

    
1517
    return True
1518

    
1519
  def _VerifyNodeTime(self, ninfo, nresult,
1520
                      nvinfo_starttime, nvinfo_endtime):
1521
    """Check the node time.
1522

1523
    @type ninfo: L{objects.Node}
1524
    @param ninfo: the node to check
1525
    @param nresult: the remote results for the node
1526
    @param nvinfo_starttime: the start time of the RPC call
1527
    @param nvinfo_endtime: the end time of the RPC call
1528

1529
    """
1530
    node = ninfo.name
1531
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1532

    
1533
    ntime = nresult.get(constants.NV_TIME, None)
1534
    try:
1535
      ntime_merged = utils.MergeTime(ntime)
1536
    except (ValueError, TypeError):
1537
      _ErrorIf(True, self.ENODETIME, node, "Node returned invalid time")
1538
      return
1539

    
1540
    if ntime_merged < (nvinfo_starttime - constants.NODE_MAX_CLOCK_SKEW):
1541
      ntime_diff = "%.01fs" % abs(nvinfo_starttime - ntime_merged)
1542
    elif ntime_merged > (nvinfo_endtime + constants.NODE_MAX_CLOCK_SKEW):
1543
      ntime_diff = "%.01fs" % abs(ntime_merged - nvinfo_endtime)
1544
    else:
1545
      ntime_diff = None
1546

    
1547
    _ErrorIf(ntime_diff is not None, self.ENODETIME, node,
1548
             "Node time diverges by at least %s from master node time",
1549
             ntime_diff)
1550

    
1551
  def _VerifyNodeLVM(self, ninfo, nresult, vg_name):
1552
    """Check the node LVM results.
1553

1554
    @type ninfo: L{objects.Node}
1555
    @param ninfo: the node to check
1556
    @param nresult: the remote results for the node
1557
    @param vg_name: the configured VG name
1558

1559
    """
1560
    if vg_name is None:
1561
      return
1562

    
1563
    node = ninfo.name
1564
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1565

    
1566
    # checks vg existence and size > 20G
1567
    vglist = nresult.get(constants.NV_VGLIST, None)
1568
    test = not vglist
1569
    _ErrorIf(test, self.ENODELVM, node, "unable to check volume groups")
1570
    if not test:
1571
      vgstatus = utils.CheckVolumeGroupSize(vglist, vg_name,
1572
                                            constants.MIN_VG_SIZE)
1573
      _ErrorIf(vgstatus, self.ENODELVM, node, vgstatus)
1574

    
1575
    # check pv names
1576
    pvlist = nresult.get(constants.NV_PVLIST, None)
1577
    test = pvlist is None
1578
    _ErrorIf(test, self.ENODELVM, node, "Can't get PV list from node")
1579
    if not test:
1580
      # check that ':' is not present in PV names, since it's a
1581
      # special character for lvcreate (denotes the range of PEs to
1582
      # use on the PV)
1583
      for _, pvname, owner_vg in pvlist:
1584
        test = ":" in pvname
1585
        _ErrorIf(test, self.ENODELVM, node, "Invalid character ':' in PV"
1586
                 " '%s' of VG '%s'", pvname, owner_vg)
1587

    
1588
  def _VerifyNodeBridges(self, ninfo, nresult, bridges):
1589
    """Check the node bridges.
1590

1591
    @type ninfo: L{objects.Node}
1592
    @param ninfo: the node to check
1593
    @param nresult: the remote results for the node
1594
    @param bridges: the expected list of bridges
1595

1596
    """
1597
    if not bridges:
1598
      return
1599

    
1600
    node = ninfo.name
1601
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1602

    
1603
    missing = nresult.get(constants.NV_BRIDGES, None)
1604
    test = not isinstance(missing, list)
1605
    _ErrorIf(test, self.ENODENET, node,
1606
             "did not return valid bridge information")
1607
    if not test:
1608
      _ErrorIf(bool(missing), self.ENODENET, node, "missing bridges: %s" %
1609
               utils.CommaJoin(sorted(missing)))
1610

    
1611
  def _VerifyNodeNetwork(self, ninfo, nresult):
1612
    """Check the node network connectivity results.
1613

1614
    @type ninfo: L{objects.Node}
1615
    @param ninfo: the node to check
1616
    @param nresult: the remote results for the node
1617

1618
    """
1619
    node = ninfo.name
1620
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1621

    
1622
    test = constants.NV_NODELIST not in nresult
1623
    _ErrorIf(test, self.ENODESSH, node,
1624
             "node hasn't returned node ssh connectivity data")
1625
    if not test:
1626
      if nresult[constants.NV_NODELIST]:
1627
        for a_node, a_msg in nresult[constants.NV_NODELIST].items():
1628
          _ErrorIf(True, self.ENODESSH, node,
1629
                   "ssh communication with node '%s': %s", a_node, a_msg)
1630

    
1631
    test = constants.NV_NODENETTEST not in nresult
1632
    _ErrorIf(test, self.ENODENET, node,
1633
             "node hasn't returned node tcp connectivity data")
1634
    if not test:
1635
      if nresult[constants.NV_NODENETTEST]:
1636
        nlist = utils.NiceSort(nresult[constants.NV_NODENETTEST].keys())
1637
        for anode in nlist:
1638
          _ErrorIf(True, self.ENODENET, node,
1639
                   "tcp communication with node '%s': %s",
1640
                   anode, nresult[constants.NV_NODENETTEST][anode])
1641

    
1642
    test = constants.NV_MASTERIP not in nresult
1643
    _ErrorIf(test, self.ENODENET, node,
1644
             "node hasn't returned node master IP reachability data")
1645
    if not test:
1646
      if not nresult[constants.NV_MASTERIP]:
1647
        if node == self.master_node:
1648
          msg = "the master node cannot reach the master IP (not configured?)"
1649
        else:
1650
          msg = "cannot reach the master IP"
1651
        _ErrorIf(True, self.ENODENET, node, msg)
1652

    
1653
  def _VerifyInstance(self, instance, instanceconfig, node_image,
1654
                      diskstatus):
1655
    """Verify an instance.
1656

1657
    This function checks to see if the required block devices are
1658
    available on the instance's node.
1659

1660
    """
1661
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1662
    node_current = instanceconfig.primary_node
1663

    
1664
    node_vol_should = {}
1665
    instanceconfig.MapLVsByNode(node_vol_should)
1666

    
1667
    for node in node_vol_should:
1668
      n_img = node_image[node]
1669
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1670
        # ignore missing volumes on offline or broken nodes
1671
        continue
1672
      for volume in node_vol_should[node]:
1673
        test = volume not in n_img.volumes
1674
        _ErrorIf(test, self.EINSTANCEMISSINGDISK, instance,
1675
                 "volume %s missing on node %s", volume, node)
1676

    
1677
    if instanceconfig.admin_up:
1678
      pri_img = node_image[node_current]
1679
      test = instance not in pri_img.instances and not pri_img.offline
1680
      _ErrorIf(test, self.EINSTANCEDOWN, instance,
1681
               "instance not running on its primary node %s",
1682
               node_current)
1683

    
1684
    diskdata = [(nname, success, status, idx)
1685
                for (nname, disks) in diskstatus.items()
1686
                for idx, (success, status) in enumerate(disks)]
1687

    
1688
    for nname, success, bdev_status, idx in diskdata:
1689
      # the 'ghost node' construction in Exec() ensures that we have a
1690
      # node here
1691
      snode = node_image[nname]
1692
      bad_snode = snode.ghost or snode.offline
1693
      _ErrorIf(instanceconfig.admin_up and not success and not bad_snode,
1694
               self.EINSTANCEFAULTYDISK, instance,
1695
               "couldn't retrieve status for disk/%s on %s: %s",
1696
               idx, nname, bdev_status)
1697
      _ErrorIf((instanceconfig.admin_up and success and
1698
                bdev_status.ldisk_status == constants.LDS_FAULTY),
1699
               self.EINSTANCEFAULTYDISK, instance,
1700
               "disk/%s on %s is faulty", idx, nname)
1701

    
1702
  def _VerifyOrphanVolumes(self, node_vol_should, node_image, reserved):
1703
    """Verify if there are any unknown volumes in the cluster.
1704

1705
    The .os, .swap and backup volumes are ignored. All other volumes are
1706
    reported as unknown.
1707

1708
    @type reserved: L{ganeti.utils.FieldSet}
1709
    @param reserved: a FieldSet of reserved volume names
1710

1711
    """
1712
    for node, n_img in node_image.items():
1713
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1714
        # skip non-healthy nodes
1715
        continue
1716
      for volume in n_img.volumes:
1717
        test = ((node not in node_vol_should or
1718
                volume not in node_vol_should[node]) and
1719
                not reserved.Matches(volume))
1720
        self._ErrorIf(test, self.ENODEORPHANLV, node,
1721
                      "volume %s is unknown", volume)
1722

    
1723
  def _VerifyNPlusOneMemory(self, node_image, instance_cfg):
1724
    """Verify N+1 Memory Resilience.
1725

1726
    Check that if one single node dies we can still start all the
1727
    instances it was primary for.
1728

1729
    """
1730
    cluster_info = self.cfg.GetClusterInfo()
1731
    for node, n_img in node_image.items():
1732
      # This code checks that every node which is now listed as
1733
      # secondary has enough memory to host all instances it is
1734
      # supposed to should a single other node in the cluster fail.
1735
      # FIXME: not ready for failover to an arbitrary node
1736
      # FIXME: does not support file-backed instances
1737
      # WARNING: we currently take into account down instances as well
1738
      # as up ones, considering that even if they're down someone
1739
      # might want to start them even in the event of a node failure.
1740
      if n_img.offline:
1741
        # we're skipping offline nodes from the N+1 warning, since
1742
        # most likely we don't have good memory infromation from them;
1743
        # we already list instances living on such nodes, and that's
1744
        # enough warning
1745
        continue
1746
      for prinode, instances in n_img.sbp.items():
1747
        needed_mem = 0
1748
        for instance in instances:
1749
          bep = cluster_info.FillBE(instance_cfg[instance])
1750
          if bep[constants.BE_AUTO_BALANCE]:
1751
            needed_mem += bep[constants.BE_MEMORY]
1752
        test = n_img.mfree < needed_mem
1753
        self._ErrorIf(test, self.ENODEN1, node,
1754
                      "not enough memory to accomodate instance failovers"
1755
                      " should node %s fail (%dMiB needed, %dMiB available)",
1756
                      prinode, needed_mem, n_img.mfree)
1757

    
1758
  @classmethod
1759
  def _VerifyFiles(cls, errorif, nodeinfo, master_node, all_nvinfo,
1760
                   (files_all, files_all_opt, files_mc, files_vm)):
1761
    """Verifies file checksums collected from all nodes.
1762

1763
    @param errorif: Callback for reporting errors
1764
    @param nodeinfo: List of L{objects.Node} objects
1765
    @param master_node: Name of master node
1766
    @param all_nvinfo: RPC results
1767

1768
    """
1769
    node_names = frozenset(node.name for node in nodeinfo)
1770

    
1771
    assert master_node in node_names
1772
    assert (len(files_all | files_all_opt | files_mc | files_vm) ==
1773
            sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
1774
           "Found file listed in more than one file list"
1775

    
1776
    # Define functions determining which nodes to consider for a file
1777
    file2nodefn = dict([(filename, fn)
1778
      for (files, fn) in [(files_all, None),
1779
                          (files_all_opt, None),
1780
                          (files_mc, lambda node: (node.master_candidate or
1781
                                                   node.name == master_node)),
1782
                          (files_vm, lambda node: node.vm_capable)]
1783
      for filename in files])
1784

    
1785
    fileinfo = dict((filename, {}) for filename in file2nodefn.keys())
1786

    
1787
    for node in nodeinfo:
1788
      nresult = all_nvinfo[node.name]
1789

    
1790
      if nresult.fail_msg or not nresult.payload:
1791
        node_files = None
1792
      else:
1793
        node_files = nresult.payload.get(constants.NV_FILELIST, None)
1794

    
1795
      test = not (node_files and isinstance(node_files, dict))
1796
      errorif(test, cls.ENODEFILECHECK, node.name,
1797
              "Node did not return file checksum data")
1798
      if test:
1799
        continue
1800

    
1801
      for (filename, checksum) in node_files.items():
1802
        # Check if the file should be considered for a node
1803
        fn = file2nodefn[filename]
1804
        if fn is None or fn(node):
1805
          fileinfo[filename].setdefault(checksum, set()).add(node.name)
1806

    
1807
    for (filename, checksums) in fileinfo.items():
1808
      assert compat.all(len(i) > 10 for i in checksums), "Invalid checksum"
1809

    
1810
      # Nodes having the file
1811
      with_file = frozenset(node_name
1812
                            for nodes in fileinfo[filename].values()
1813
                            for node_name in nodes)
1814

    
1815
      # Nodes missing file
1816
      missing_file = node_names - with_file
1817

    
1818
      if filename in files_all_opt:
1819
        # All or no nodes
1820
        errorif(missing_file and missing_file != node_names,
1821
                cls.ECLUSTERFILECHECK, None,
1822
                "File %s is optional, but it must exist on all or no nodes (not"
1823
                " found on %s)",
1824
                filename, utils.CommaJoin(utils.NiceSort(missing_file)))
1825
      else:
1826
        errorif(missing_file, cls.ECLUSTERFILECHECK, None,
1827
                "File %s is missing from node(s) %s", filename,
1828
                utils.CommaJoin(utils.NiceSort(missing_file)))
1829

    
1830
      # See if there are multiple versions of the file
1831
      test = len(checksums) > 1
1832
      if test:
1833
        variants = ["variant %s on %s" %
1834
                    (idx + 1, utils.CommaJoin(utils.NiceSort(nodes)))
1835
                    for (idx, (checksum, nodes)) in
1836
                      enumerate(sorted(checksums.items()))]
1837
      else:
1838
        variants = []
1839

    
1840
      errorif(test, cls.ECLUSTERFILECHECK, None,
1841
              "File %s found with %s different checksums (%s)",
1842
              filename, len(checksums), "; ".join(variants))
1843

    
1844
  def _VerifyNodeDrbd(self, ninfo, nresult, instanceinfo, drbd_helper,
1845
                      drbd_map):
1846
    """Verifies and the node DRBD status.
1847

1848
    @type ninfo: L{objects.Node}
1849
    @param ninfo: the node to check
1850
    @param nresult: the remote results for the node
1851
    @param instanceinfo: the dict of instances
1852
    @param drbd_helper: the configured DRBD usermode helper
1853
    @param drbd_map: the DRBD map as returned by
1854
        L{ganeti.config.ConfigWriter.ComputeDRBDMap}
1855

1856
    """
1857
    node = ninfo.name
1858
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1859

    
1860
    if drbd_helper:
1861
      helper_result = nresult.get(constants.NV_DRBDHELPER, None)
1862
      test = (helper_result == None)
1863
      _ErrorIf(test, self.ENODEDRBDHELPER, node,
1864
               "no drbd usermode helper returned")
1865
      if helper_result:
1866
        status, payload = helper_result
1867
        test = not status
1868
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
1869
                 "drbd usermode helper check unsuccessful: %s", payload)
1870
        test = status and (payload != drbd_helper)
1871
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
1872
                 "wrong drbd usermode helper: %s", payload)
1873

    
1874
    # compute the DRBD minors
1875
    node_drbd = {}
1876
    for minor, instance in drbd_map[node].items():
1877
      test = instance not in instanceinfo
1878
      _ErrorIf(test, self.ECLUSTERCFG, None,
1879
               "ghost instance '%s' in temporary DRBD map", instance)
1880
        # ghost instance should not be running, but otherwise we
1881
        # don't give double warnings (both ghost instance and
1882
        # unallocated minor in use)
1883
      if test:
1884
        node_drbd[minor] = (instance, False)
1885
      else:
1886
        instance = instanceinfo[instance]
1887
        node_drbd[minor] = (instance.name, instance.admin_up)
1888

    
1889
    # and now check them
1890
    used_minors = nresult.get(constants.NV_DRBDLIST, [])
1891
    test = not isinstance(used_minors, (tuple, list))
1892
    _ErrorIf(test, self.ENODEDRBD, node,
1893
             "cannot parse drbd status file: %s", str(used_minors))
1894
    if test:
1895
      # we cannot check drbd status
1896
      return
1897

    
1898
    for minor, (iname, must_exist) in node_drbd.items():
1899
      test = minor not in used_minors and must_exist
1900
      _ErrorIf(test, self.ENODEDRBD, node,
1901
               "drbd minor %d of instance %s is not active", minor, iname)
1902
    for minor in used_minors:
1903
      test = minor not in node_drbd
1904
      _ErrorIf(test, self.ENODEDRBD, node,
1905
               "unallocated drbd minor %d is in use", minor)
1906

    
1907
  def _UpdateNodeOS(self, ninfo, nresult, nimg):
1908
    """Builds the node OS structures.
1909

1910
    @type ninfo: L{objects.Node}
1911
    @param ninfo: the node to check
1912
    @param nresult: the remote results for the node
1913
    @param nimg: the node image object
1914

1915
    """
1916
    node = ninfo.name
1917
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1918

    
1919
    remote_os = nresult.get(constants.NV_OSLIST, None)
1920
    test = (not isinstance(remote_os, list) or
1921
            not compat.all(isinstance(v, list) and len(v) == 7
1922
                           for v in remote_os))
1923

    
1924
    _ErrorIf(test, self.ENODEOS, node,
1925
             "node hasn't returned valid OS data")
1926

    
1927
    nimg.os_fail = test
1928

    
1929
    if test:
1930
      return
1931

    
1932
    os_dict = {}
1933

    
1934
    for (name, os_path, status, diagnose,
1935
         variants, parameters, api_ver) in nresult[constants.NV_OSLIST]:
1936

    
1937
      if name not in os_dict:
1938
        os_dict[name] = []
1939

    
1940
      # parameters is a list of lists instead of list of tuples due to
1941
      # JSON lacking a real tuple type, fix it:
1942
      parameters = [tuple(v) for v in parameters]
1943
      os_dict[name].append((os_path, status, diagnose,
1944
                            set(variants), set(parameters), set(api_ver)))
1945

    
1946
    nimg.oslist = os_dict
1947

    
1948
  def _VerifyNodeOS(self, ninfo, nimg, base):
1949
    """Verifies the node OS list.
1950

1951
    @type ninfo: L{objects.Node}
1952
    @param ninfo: the node to check
1953
    @param nimg: the node image object
1954
    @param base: the 'template' node we match against (e.g. from the master)
1955

1956
    """
1957
    node = ninfo.name
1958
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1959

    
1960
    assert not nimg.os_fail, "Entered _VerifyNodeOS with failed OS rpc?"
1961

    
1962
    beautify_params = lambda l: ["%s: %s" % (k, v) for (k, v) in l]
1963
    for os_name, os_data in nimg.oslist.items():
1964
      assert os_data, "Empty OS status for OS %s?!" % os_name
1965
      f_path, f_status, f_diag, f_var, f_param, f_api = os_data[0]
1966
      _ErrorIf(not f_status, self.ENODEOS, node,
1967
               "Invalid OS %s (located at %s): %s", os_name, f_path, f_diag)
1968
      _ErrorIf(len(os_data) > 1, self.ENODEOS, node,
1969
               "OS '%s' has multiple entries (first one shadows the rest): %s",
1970
               os_name, utils.CommaJoin([v[0] for v in os_data]))
1971
      # this will catched in backend too
1972
      _ErrorIf(compat.any(v >= constants.OS_API_V15 for v in f_api)
1973
               and not f_var, self.ENODEOS, node,
1974
               "OS %s with API at least %d does not declare any variant",
1975
               os_name, constants.OS_API_V15)
1976
      # comparisons with the 'base' image
1977
      test = os_name not in base.oslist
1978
      _ErrorIf(test, self.ENODEOS, node,
1979
               "Extra OS %s not present on reference node (%s)",
1980
               os_name, base.name)
1981
      if test:
1982
        continue
1983
      assert base.oslist[os_name], "Base node has empty OS status?"
1984
      _, b_status, _, b_var, b_param, b_api = base.oslist[os_name][0]
1985
      if not b_status:
1986
        # base OS is invalid, skipping
1987
        continue
1988
      for kind, a, b in [("API version", f_api, b_api),
1989
                         ("variants list", f_var, b_var),
1990
                         ("parameters", beautify_params(f_param),
1991
                          beautify_params(b_param))]:
1992
        _ErrorIf(a != b, self.ENODEOS, node,
1993
                 "OS %s for %s differs from reference node %s: [%s] vs. [%s]",
1994
                 kind, os_name, base.name,
1995
                 utils.CommaJoin(sorted(a)), utils.CommaJoin(sorted(b)))
1996

    
1997
    # check any missing OSes
1998
    missing = set(base.oslist.keys()).difference(nimg.oslist.keys())
1999
    _ErrorIf(missing, self.ENODEOS, node,
2000
             "OSes present on reference node %s but missing on this node: %s",
2001
             base.name, utils.CommaJoin(missing))
2002

    
2003
  def _VerifyOob(self, ninfo, nresult):
2004
    """Verifies out of band functionality of a node.
2005

2006
    @type ninfo: L{objects.Node}
2007
    @param ninfo: the node to check
2008
    @param nresult: the remote results for the node
2009

2010
    """
2011
    node = ninfo.name
2012
    # We just have to verify the paths on master and/or master candidates
2013
    # as the oob helper is invoked on the master
2014
    if ((ninfo.master_candidate or ninfo.master_capable) and
2015
        constants.NV_OOB_PATHS in nresult):
2016
      for path_result in nresult[constants.NV_OOB_PATHS]:
2017
        self._ErrorIf(path_result, self.ENODEOOBPATH, node, path_result)
2018

    
2019
  def _UpdateNodeVolumes(self, ninfo, nresult, nimg, vg_name):
2020
    """Verifies and updates the node volume data.
2021

2022
    This function will update a L{NodeImage}'s internal structures
2023
    with data from the remote call.
2024

2025
    @type ninfo: L{objects.Node}
2026
    @param ninfo: the node to check
2027
    @param nresult: the remote results for the node
2028
    @param nimg: the node image object
2029
    @param vg_name: the configured VG name
2030

2031
    """
2032
    node = ninfo.name
2033
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2034

    
2035
    nimg.lvm_fail = True
2036
    lvdata = nresult.get(constants.NV_LVLIST, "Missing LV data")
2037
    if vg_name is None:
2038
      pass
2039
    elif isinstance(lvdata, basestring):
2040
      _ErrorIf(True, self.ENODELVM, node, "LVM problem on node: %s",
2041
               utils.SafeEncode(lvdata))
2042
    elif not isinstance(lvdata, dict):
2043
      _ErrorIf(True, self.ENODELVM, node, "rpc call to node failed (lvlist)")
2044
    else:
2045
      nimg.volumes = lvdata
2046
      nimg.lvm_fail = False
2047

    
2048
  def _UpdateNodeInstances(self, ninfo, nresult, nimg):
2049
    """Verifies and updates the node instance list.
2050

2051
    If the listing was successful, then updates this node's instance
2052
    list. Otherwise, it marks the RPC call as failed for the instance
2053
    list key.
2054

2055
    @type ninfo: L{objects.Node}
2056
    @param ninfo: the node to check
2057
    @param nresult: the remote results for the node
2058
    @param nimg: the node image object
2059

2060
    """
2061
    idata = nresult.get(constants.NV_INSTANCELIST, None)
2062
    test = not isinstance(idata, list)
2063
    self._ErrorIf(test, self.ENODEHV, ninfo.name, "rpc call to node failed"
2064
                  " (instancelist): %s", utils.SafeEncode(str(idata)))
2065
    if test:
2066
      nimg.hyp_fail = True
2067
    else:
2068
      nimg.instances = idata
2069

    
2070
  def _UpdateNodeInfo(self, ninfo, nresult, nimg, vg_name):
2071
    """Verifies and computes a node information map
2072

2073
    @type ninfo: L{objects.Node}
2074
    @param ninfo: the node to check
2075
    @param nresult: the remote results for the node
2076
    @param nimg: the node image object
2077
    @param vg_name: the configured VG name
2078

2079
    """
2080
    node = ninfo.name
2081
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2082

    
2083
    # try to read free memory (from the hypervisor)
2084
    hv_info = nresult.get(constants.NV_HVINFO, None)
2085
    test = not isinstance(hv_info, dict) or "memory_free" not in hv_info
2086
    _ErrorIf(test, self.ENODEHV, node, "rpc call to node failed (hvinfo)")
2087
    if not test:
2088
      try:
2089
        nimg.mfree = int(hv_info["memory_free"])
2090
      except (ValueError, TypeError):
2091
        _ErrorIf(True, self.ENODERPC, node,
2092
                 "node returned invalid nodeinfo, check hypervisor")
2093

    
2094
    # FIXME: devise a free space model for file based instances as well
2095
    if vg_name is not None:
2096
      test = (constants.NV_VGLIST not in nresult or
2097
              vg_name not in nresult[constants.NV_VGLIST])
2098
      _ErrorIf(test, self.ENODELVM, node,
2099
               "node didn't return data for the volume group '%s'"
2100
               " - it is either missing or broken", vg_name)
2101
      if not test:
2102
        try:
2103
          nimg.dfree = int(nresult[constants.NV_VGLIST][vg_name])
2104
        except (ValueError, TypeError):
2105
          _ErrorIf(True, self.ENODERPC, node,
2106
                   "node returned invalid LVM info, check LVM status")
2107

    
2108
  def _CollectDiskInfo(self, nodelist, node_image, instanceinfo):
2109
    """Gets per-disk status information for all instances.
2110

2111
    @type nodelist: list of strings
2112
    @param nodelist: Node names
2113
    @type node_image: dict of (name, L{objects.Node})
2114
    @param node_image: Node objects
2115
    @type instanceinfo: dict of (name, L{objects.Instance})
2116
    @param instanceinfo: Instance objects
2117
    @rtype: {instance: {node: [(succes, payload)]}}
2118
    @return: a dictionary of per-instance dictionaries with nodes as
2119
        keys and disk information as values; the disk information is a
2120
        list of tuples (success, payload)
2121

2122
    """
2123
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2124

    
2125
    node_disks = {}
2126
    node_disks_devonly = {}
2127
    diskless_instances = set()
2128
    diskless = constants.DT_DISKLESS
2129

    
2130
    for nname in nodelist:
2131
      node_instances = list(itertools.chain(node_image[nname].pinst,
2132
                                            node_image[nname].sinst))
2133
      diskless_instances.update(inst for inst in node_instances
2134
                                if instanceinfo[inst].disk_template == diskless)
2135
      disks = [(inst, disk)
2136
               for inst in node_instances
2137
               for disk in instanceinfo[inst].disks]
2138

    
2139
      if not disks:
2140
        # No need to collect data
2141
        continue
2142

    
2143
      node_disks[nname] = disks
2144

    
2145
      # Creating copies as SetDiskID below will modify the objects and that can
2146
      # lead to incorrect data returned from nodes
2147
      devonly = [dev.Copy() for (_, dev) in disks]
2148

    
2149
      for dev in devonly:
2150
        self.cfg.SetDiskID(dev, nname)
2151

    
2152
      node_disks_devonly[nname] = devonly
2153

    
2154
    assert len(node_disks) == len(node_disks_devonly)
2155

    
2156
    # Collect data from all nodes with disks
2157
    result = self.rpc.call_blockdev_getmirrorstatus_multi(node_disks.keys(),
2158
                                                          node_disks_devonly)
2159

    
2160
    assert len(result) == len(node_disks)
2161

    
2162
    instdisk = {}
2163

    
2164
    for (nname, nres) in result.items():
2165
      disks = node_disks[nname]
2166

    
2167
      if nres.offline:
2168
        # No data from this node
2169
        data = len(disks) * [(False, "node offline")]
2170
      else:
2171
        msg = nres.fail_msg
2172
        _ErrorIf(msg, self.ENODERPC, nname,
2173
                 "while getting disk information: %s", msg)
2174
        if msg:
2175
          # No data from this node
2176
          data = len(disks) * [(False, msg)]
2177
        else:
2178
          data = []
2179
          for idx, i in enumerate(nres.payload):
2180
            if isinstance(i, (tuple, list)) and len(i) == 2:
2181
              data.append(i)
2182
            else:
2183
              logging.warning("Invalid result from node %s, entry %d: %s",
2184
                              nname, idx, i)
2185
              data.append((False, "Invalid result from the remote node"))
2186

    
2187
      for ((inst, _), status) in zip(disks, data):
2188
        instdisk.setdefault(inst, {}).setdefault(nname, []).append(status)
2189

    
2190
    # Add empty entries for diskless instances.
2191
    for inst in diskless_instances:
2192
      assert inst not in instdisk
2193
      instdisk[inst] = {}
2194

    
2195
    assert compat.all(len(statuses) == len(instanceinfo[inst].disks) and
2196
                      len(nnames) <= len(instanceinfo[inst].all_nodes) and
2197
                      compat.all(isinstance(s, (tuple, list)) and
2198
                                 len(s) == 2 for s in statuses)
2199
                      for inst, nnames in instdisk.items()
2200
                      for nname, statuses in nnames.items())
2201
    assert set(instdisk) == set(instanceinfo), "instdisk consistency failure"
2202

    
2203
    return instdisk
2204

    
2205
  def _VerifyHVP(self, hvp_data):
2206
    """Verifies locally the syntax of the hypervisor parameters.
2207

2208
    """
2209
    for item, hv_name, hv_params in hvp_data:
2210
      msg = ("hypervisor %s parameters syntax check (source %s): %%s" %
2211
             (item, hv_name))
2212
      try:
2213
        hv_class = hypervisor.GetHypervisor(hv_name)
2214
        utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
2215
        hv_class.CheckParameterSyntax(hv_params)
2216
      except errors.GenericError, err:
2217
        self._ErrorIf(True, self.ECLUSTERCFG, None, msg % str(err))
2218

    
2219
  def BuildHooksEnv(self):
2220
    """Build hooks env.
2221

2222
    Cluster-Verify hooks just ran in the post phase and their failure makes
2223
    the output be logged in the verify output and the verification to fail.
2224

2225
    """
2226
    env = {
2227
      "CLUSTER_TAGS": " ".join(self.cfg.GetClusterInfo().GetTags())
2228
      }
2229

    
2230
    env.update(("NODE_TAGS_%s" % node.name, " ".join(node.GetTags()))
2231
               for node in self.my_node_info.values())
2232

    
2233
    return env
2234

    
2235
  def BuildHooksNodes(self):
2236
    """Build hooks nodes.
2237

2238
    """
2239
    assert self.my_node_names, ("Node list not gathered,"
2240
      " has CheckPrereq been executed?")
2241
    return ([], self.my_node_names)
2242

    
2243
  def Exec(self, feedback_fn):
2244
    """Verify integrity of cluster, performing various test on nodes.
2245

2246
    """
2247
    # This method has too many local variables. pylint: disable-msg=R0914
2248
    self.bad = False
2249
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2250
    verbose = self.op.verbose
2251
    self._feedback_fn = feedback_fn
2252
    feedback_fn("* Verifying global settings")
2253
    for msg in self.cfg.VerifyConfig():
2254
      _ErrorIf(True, self.ECLUSTERCFG, None, msg)
2255

    
2256
    # Check the cluster certificates
2257
    for cert_filename in constants.ALL_CERT_FILES:
2258
      (errcode, msg) = _VerifyCertificate(cert_filename)
2259
      _ErrorIf(errcode, self.ECLUSTERCERT, None, msg, code=errcode)
2260

    
2261
    vg_name = self.cfg.GetVGName()
2262
    drbd_helper = self.cfg.GetDRBDHelper()
2263
    hypervisors = self.cfg.GetClusterInfo().enabled_hypervisors
2264
    cluster = self.cfg.GetClusterInfo()
2265
    groupinfo = self.cfg.GetAllNodeGroupsInfo()
2266
    node_data_list = [self.my_node_info[name] for name in self.my_node_names]
2267

    
2268
    i_non_redundant = [] # Non redundant instances
2269
    i_non_a_balanced = [] # Non auto-balanced instances
2270
    n_offline = 0 # Count of offline nodes
2271
    n_drained = 0 # Count of nodes being drained
2272
    node_vol_should = {}
2273

    
2274
    # FIXME: verify OS list
2275

    
2276
    # File verification
2277
    filemap = _ComputeAncillaryFiles(cluster, False)
2278

    
2279
    # do local checksums
2280
    master_node = self.master_node = self.cfg.GetMasterNode()
2281
    master_ip = self.cfg.GetMasterIP()
2282

    
2283
    # Compute the set of hypervisor parameters
2284
    hvp_data = []
2285
    for hv_name in hypervisors:
2286
      hvp_data.append(("cluster", hv_name, cluster.GetHVDefaults(hv_name)))
2287
    for os_name, os_hvp in cluster.os_hvp.items():
2288
      for hv_name, hv_params in os_hvp.items():
2289
        if not hv_params:
2290
          continue
2291
        full_params = cluster.GetHVDefaults(hv_name, os_name=os_name)
2292
        hvp_data.append(("os %s" % os_name, hv_name, full_params))
2293
    # TODO: collapse identical parameter values in a single one
2294
    for instance in self.all_inst_info.values():
2295
      if not instance.hvparams:
2296
        continue
2297
      hvp_data.append(("instance %s" % instance.name, instance.hypervisor,
2298
                       cluster.FillHV(instance)))
2299
    # and verify them locally
2300
    self._VerifyHVP(hvp_data)
2301

    
2302
    feedback_fn("* Gathering data (%d nodes)" % len(self.my_node_names))
2303
    node_verify_param = {
2304
      constants.NV_FILELIST:
2305
        utils.UniqueSequence(filename
2306
                             for files in filemap
2307
                             for filename in files),
2308
      constants.NV_NODELIST: [node.name for node in self.all_node_info.values()
2309
                              if not node.offline],
2310
      constants.NV_HYPERVISOR: hypervisors,
2311
      constants.NV_HVPARAMS: hvp_data,
2312
      constants.NV_NODENETTEST: [(node.name, node.primary_ip, node.secondary_ip)
2313
                                 for node in node_data_list
2314
                                 if not node.offline],
2315
      constants.NV_INSTANCELIST: hypervisors,
2316
      constants.NV_VERSION: None,
2317
      constants.NV_HVINFO: self.cfg.GetHypervisorType(),
2318
      constants.NV_NODESETUP: None,
2319
      constants.NV_TIME: None,
2320
      constants.NV_MASTERIP: (master_node, master_ip),
2321
      constants.NV_OSLIST: None,
2322
      constants.NV_VMNODES: self.cfg.GetNonVmCapableNodeList(),
2323
      }
2324

    
2325
    if vg_name is not None:
2326
      node_verify_param[constants.NV_VGLIST] = None
2327
      node_verify_param[constants.NV_LVLIST] = vg_name
2328
      node_verify_param[constants.NV_PVLIST] = [vg_name]
2329
      node_verify_param[constants.NV_DRBDLIST] = None
2330

    
2331
    if drbd_helper:
2332
      node_verify_param[constants.NV_DRBDHELPER] = drbd_helper
2333

    
2334
    # bridge checks
2335
    # FIXME: this needs to be changed per node-group, not cluster-wide
2336
    bridges = set()
2337
    default_nicpp = cluster.nicparams[constants.PP_DEFAULT]
2338
    if default_nicpp[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2339
      bridges.add(default_nicpp[constants.NIC_LINK])
2340
    for instance in self.my_inst_info.values():
2341
      for nic in instance.nics:
2342
        full_nic = cluster.SimpleFillNIC(nic.nicparams)
2343
        if full_nic[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2344
          bridges.add(full_nic[constants.NIC_LINK])
2345

    
2346
    if bridges:
2347
      node_verify_param[constants.NV_BRIDGES] = list(bridges)
2348

    
2349
    # Build our expected cluster state
2350
    node_image = dict((node.name, self.NodeImage(offline=node.offline,
2351
                                                 name=node.name,
2352
                                                 vm_capable=node.vm_capable))
2353
                      for node in node_data_list)
2354

    
2355
    # Gather OOB paths
2356
    oob_paths = []
2357
    for node in self.all_node_info.values():
2358
      path = _SupportsOob(self.cfg, node)
2359
      if path and path not in oob_paths:
2360
        oob_paths.append(path)
2361

    
2362
    if oob_paths:
2363
      node_verify_param[constants.NV_OOB_PATHS] = oob_paths
2364

    
2365
    for instance in self.my_inst_names:
2366
      inst_config = self.my_inst_info[instance]
2367

    
2368
      for nname in inst_config.all_nodes:
2369
        if nname not in node_image:
2370
          # ghost node
2371
          gnode = self.NodeImage(name=nname)
2372
          gnode.ghost = True
2373
          node_image[nname] = gnode
2374

    
2375
      inst_config.MapLVsByNode(node_vol_should)
2376

    
2377
      pnode = inst_config.primary_node
2378
      node_image[pnode].pinst.append(instance)
2379

    
2380
      for snode in inst_config.secondary_nodes:
2381
        nimg = node_image[snode]
2382
        nimg.sinst.append(instance)
2383
        if pnode not in nimg.sbp:
2384
          nimg.sbp[pnode] = []
2385
        nimg.sbp[pnode].append(instance)
2386

    
2387
    # At this point, we have the in-memory data structures complete,
2388
    # except for the runtime information, which we'll gather next
2389

    
2390
    # Due to the way our RPC system works, exact response times cannot be
2391
    # guaranteed (e.g. a broken node could run into a timeout). By keeping the
2392
    # time before and after executing the request, we can at least have a time
2393
    # window.
2394
    nvinfo_starttime = time.time()
2395
    all_nvinfo = self.rpc.call_node_verify(self.my_node_names,
2396
                                           node_verify_param,
2397
                                           self.cfg.GetClusterName())
2398
    nvinfo_endtime = time.time()
2399

    
2400
    all_drbd_map = self.cfg.ComputeDRBDMap()
2401

    
2402
    feedback_fn("* Gathering disk information (%s nodes)" %
2403
                len(self.my_node_names))
2404
    instdisk = self._CollectDiskInfo(self.my_node_names, node_image,
2405
                                     self.my_inst_info)
2406

    
2407
    feedback_fn("* Verifying configuration file consistency")
2408

    
2409
    # If not all nodes are being checked, we need to make sure the master node
2410
    # and a non-checked vm_capable node are in the list.
2411
    absent_nodes = set(self.all_node_info).difference(self.my_node_info)
2412
    if absent_nodes:
2413
      vf_nvinfo = all_nvinfo.copy()
2414
      vf_node_info = list(self.my_node_info.values())
2415
      additional_nodes = []
2416
      if master_node not in self.my_node_info:
2417
        additional_nodes.append(master_node)
2418
        vf_node_info.append(self.all_node_info[master_node])
2419
      # Add the first vm_capable node we find which is not included
2420
      for node in absent_nodes:
2421
        nodeinfo = self.all_node_info[node]
2422
        if nodeinfo.vm_capable and not nodeinfo.offline:
2423
          additional_nodes.append(node)
2424
          vf_node_info.append(self.all_node_info[node])
2425
          break
2426
      key = constants.NV_FILELIST
2427
      vf_nvinfo.update(self.rpc.call_node_verify(additional_nodes,
2428
                                                 {key: node_verify_param[key]},
2429
                                                 self.cfg.GetClusterName()))
2430
    else:
2431
      vf_nvinfo = all_nvinfo
2432
      vf_node_info = self.my_node_info.values()
2433

    
2434
    self._VerifyFiles(_ErrorIf, vf_node_info, master_node, vf_nvinfo, filemap)
2435

    
2436
    feedback_fn("* Verifying node status")
2437

    
2438
    refos_img = None
2439

    
2440
    for node_i in node_data_list:
2441
      node = node_i.name
2442
      nimg = node_image[node]
2443

    
2444
      if node_i.offline:
2445
        if verbose:
2446
          feedback_fn("* Skipping offline node %s" % (node,))
2447
        n_offline += 1
2448
        continue
2449

    
2450
      if node == master_node:
2451
        ntype = "master"
2452
      elif node_i.master_candidate:
2453
        ntype = "master candidate"
2454
      elif node_i.drained:
2455
        ntype = "drained"
2456
        n_drained += 1
2457
      else:
2458
        ntype = "regular"
2459
      if verbose:
2460
        feedback_fn("* Verifying node %s (%s)" % (node, ntype))
2461

    
2462
      msg = all_nvinfo[node].fail_msg
2463
      _ErrorIf(msg, self.ENODERPC, node, "while contacting node: %s", msg)
2464
      if msg:
2465
        nimg.rpc_fail = True
2466
        continue
2467

    
2468
      nresult = all_nvinfo[node].payload
2469

    
2470
      nimg.call_ok = self._VerifyNode(node_i, nresult)
2471
      self._VerifyNodeTime(node_i, nresult, nvinfo_starttime, nvinfo_endtime)
2472
      self._VerifyNodeNetwork(node_i, nresult)
2473
      self._VerifyOob(node_i, nresult)
2474

    
2475
      if nimg.vm_capable:
2476
        self._VerifyNodeLVM(node_i, nresult, vg_name)
2477
        self._VerifyNodeDrbd(node_i, nresult, self.all_inst_info, drbd_helper,
2478
                             all_drbd_map)
2479

    
2480
        self._UpdateNodeVolumes(node_i, nresult, nimg, vg_name)
2481
        self._UpdateNodeInstances(node_i, nresult, nimg)
2482
        self._UpdateNodeInfo(node_i, nresult, nimg, vg_name)
2483
        self._UpdateNodeOS(node_i, nresult, nimg)
2484

    
2485
        if not nimg.os_fail:
2486
          if refos_img is None:
2487
            refos_img = nimg
2488
          self._VerifyNodeOS(node_i, nimg, refos_img)
2489
        self._VerifyNodeBridges(node_i, nresult, bridges)
2490

    
2491
        # Check whether all running instancies are primary for the node. (This
2492
        # can no longer be done from _VerifyInstance below, since some of the
2493
        # wrong instances could be from other node groups.)
2494
        non_primary_inst = set(nimg.instances).difference(nimg.pinst)
2495

    
2496
        for inst in non_primary_inst:
2497
          test = inst in self.all_inst_info
2498
          _ErrorIf(test, self.EINSTANCEWRONGNODE, inst,
2499
                   "instance should not run on node %s", node_i.name)
2500
          _ErrorIf(not test, self.ENODEORPHANINSTANCE, node_i.name,
2501
                   "node is running unknown instance %s", inst)
2502

    
2503
    feedback_fn("* Verifying instance status")
2504
    for instance in self.my_inst_names:
2505
      if verbose:
2506
        feedback_fn("* Verifying instance %s" % instance)
2507
      inst_config = self.my_inst_info[instance]
2508
      self._VerifyInstance(instance, inst_config, node_image,
2509
                           instdisk[instance])
2510
      inst_nodes_offline = []
2511

    
2512
      pnode = inst_config.primary_node
2513
      pnode_img = node_image[pnode]
2514
      _ErrorIf(pnode_img.rpc_fail and not pnode_img.offline,
2515
               self.ENODERPC, pnode, "instance %s, connection to"
2516
               " primary node failed", instance)
2517

    
2518
      _ErrorIf(inst_config.admin_up and pnode_img.offline,
2519
               self.EINSTANCEBADNODE, instance,
2520
               "instance is marked as running and lives on offline node %s",
2521
               inst_config.primary_node)
2522

    
2523
      # If the instance is non-redundant we cannot survive losing its primary
2524
      # node, so we are not N+1 compliant. On the other hand we have no disk
2525
      # templates with more than one secondary so that situation is not well
2526
      # supported either.
2527
      # FIXME: does not support file-backed instances
2528
      if not inst_config.secondary_nodes:
2529
        i_non_redundant.append(instance)
2530

    
2531
      _ErrorIf(len(inst_config.secondary_nodes) > 1, self.EINSTANCELAYOUT,
2532
               instance, "instance has multiple secondary nodes: %s",
2533
               utils.CommaJoin(inst_config.secondary_nodes),
2534
               code=self.ETYPE_WARNING)
2535

    
2536
      if inst_config.disk_template in constants.DTS_INT_MIRROR:
2537
        pnode = inst_config.primary_node
2538
        instance_nodes = utils.NiceSort(inst_config.all_nodes)
2539
        instance_groups = {}
2540

    
2541
        for node in instance_nodes:
2542
          instance_groups.setdefault(self.all_node_info[node].group,
2543
                                     []).append(node)
2544

    
2545
        pretty_list = [
2546
          "%s (group %s)" % (utils.CommaJoin(nodes), groupinfo[group].name)
2547
          # Sort so that we always list the primary node first.
2548
          for group, nodes in sorted(instance_groups.items(),
2549
                                     key=lambda (_, nodes): pnode in nodes,
2550
                                     reverse=True)]
2551

    
2552
        self._ErrorIf(len(instance_groups) > 1, self.EINSTANCESPLITGROUPS,
2553
                      instance, "instance has primary and secondary nodes in"
2554
                      " different groups: %s", utils.CommaJoin(pretty_list),
2555
                      code=self.ETYPE_WARNING)
2556

    
2557
      if not cluster.FillBE(inst_config)[constants.BE_AUTO_BALANCE]:
2558
        i_non_a_balanced.append(instance)
2559

    
2560
      for snode in inst_config.secondary_nodes:
2561
        s_img = node_image[snode]
2562
        _ErrorIf(s_img.rpc_fail and not s_img.offline, self.ENODERPC, snode,
2563
                 "instance %s, connection to secondary node failed", instance)
2564

    
2565
        if s_img.offline:
2566
          inst_nodes_offline.append(snode)
2567

    
2568
      # warn that the instance lives on offline nodes
2569
      _ErrorIf(inst_nodes_offline, self.EINSTANCEBADNODE, instance,
2570
               "instance has offline secondary node(s) %s",
2571
               utils.CommaJoin(inst_nodes_offline))
2572
      # ... or ghost/non-vm_capable nodes
2573
      for node in inst_config.all_nodes:
2574
        _ErrorIf(node_image[node].ghost, self.EINSTANCEBADNODE, instance,
2575
                 "instance lives on ghost node %s", node)
2576
        _ErrorIf(not node_image[node].vm_capable, self.EINSTANCEBADNODE,
2577
                 instance, "instance lives on non-vm_capable node %s", node)
2578

    
2579
    feedback_fn("* Verifying orphan volumes")
2580
    reserved = utils.FieldSet(*cluster.reserved_lvs)
2581
    self._VerifyOrphanVolumes(node_vol_should, node_image, reserved)
2582

    
2583
    if constants.VERIFY_NPLUSONE_MEM not in self.op.skip_checks:
2584
      feedback_fn("* Verifying N+1 Memory redundancy")
2585
      self._VerifyNPlusOneMemory(node_image, self.my_inst_info)
2586

    
2587
    feedback_fn("* Other Notes")
2588
    if i_non_redundant:
2589
      feedback_fn("  - NOTICE: %d non-redundant instance(s) found."
2590
                  % len(i_non_redundant))
2591

    
2592
    if i_non_a_balanced:
2593
      feedback_fn("  - NOTICE: %d non-auto-balanced instance(s) found."
2594
                  % len(i_non_a_balanced))
2595

    
2596
    if n_offline:
2597
      feedback_fn("  - NOTICE: %d offline node(s) found." % n_offline)
2598

    
2599
    if n_drained:
2600
      feedback_fn("  - NOTICE: %d drained node(s) found." % n_drained)
2601

    
2602
    return not self.bad
2603

    
2604
  def HooksCallBack(self, phase, hooks_results, feedback_fn, lu_result):
2605
    """Analyze the post-hooks' result
2606

2607
    This method analyses the hook result, handles it, and sends some
2608
    nicely-formatted feedback back to the user.
2609

2610
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
2611
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
2612
    @param hooks_results: the results of the multi-node hooks rpc call
2613
    @param feedback_fn: function used send feedback back to the caller
2614
    @param lu_result: previous Exec result
2615
    @return: the new Exec result, based on the previous result
2616
        and hook results
2617

2618
    """
2619
    # We only really run POST phase hooks, and are only interested in
2620
    # their results
2621
    if phase == constants.HOOKS_PHASE_POST:
2622
      # Used to change hooks' output to proper indentation
2623
      feedback_fn("* Hooks Results")
2624
      assert hooks_results, "invalid result from hooks"
2625

    
2626
      for node_name in hooks_results:
2627
        res = hooks_results[node_name]
2628
        msg = res.fail_msg
2629
        test = msg and not res.offline
2630
        self._ErrorIf(test, self.ENODEHOOKS, node_name,
2631
                      "Communication failure in hooks execution: %s", msg)
2632
        if res.offline or msg:
2633
          # No need to investigate payload if node is offline or gave an error.
2634
          # override manually lu_result here as _ErrorIf only
2635
          # overrides self.bad
2636
          lu_result = 1
2637
          continue
2638
        for script, hkr, output in res.payload:
2639
          test = hkr == constants.HKR_FAIL
2640
          self._ErrorIf(test, self.ENODEHOOKS, node_name,
2641
                        "Script %s failed, output:", script)
2642
          if test:
2643
            output = self._HOOKS_INDENT_RE.sub('      ', output)
2644
            feedback_fn("%s" % output)
2645
            lu_result = 0
2646

    
2647
      return lu_result
2648

    
2649

    
2650
class LUClusterVerifyDisks(NoHooksLU):
2651
  """Verifies the cluster disks status.
2652

2653
  """
2654
  REQ_BGL = False
2655

    
2656
  def ExpandNames(self):
2657
    self.needed_locks = {
2658
      locking.LEVEL_NODE: locking.ALL_SET,
2659
      locking.LEVEL_INSTANCE: locking.ALL_SET,
2660
    }
2661
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2662

    
2663
  def Exec(self, feedback_fn):
2664
    """Verify integrity of cluster disks.
2665

2666
    @rtype: tuple of three items
2667
    @return: a tuple of (dict of node-to-node_error, list of instances
2668
        which need activate-disks, dict of instance: (node, volume) for
2669
        missing volumes
2670

2671
    """
2672
    result = res_nodes, res_instances, res_missing = {}, [], {}
2673

    
2674
    nodes = utils.NiceSort(self.cfg.GetVmCapableNodeList())
2675
    instances = self.cfg.GetAllInstancesInfo().values()
2676

    
2677
    nv_dict = {}
2678
    for inst in instances:
2679
      inst_lvs = {}
2680
      if not inst.admin_up:
2681
        continue
2682
      inst.MapLVsByNode(inst_lvs)
2683
      # transform { iname: {node: [vol,],},} to {(node, vol): iname}
2684
      for node, vol_list in inst_lvs.iteritems():
2685
        for vol in vol_list:
2686
          nv_dict[(node, vol)] = inst
2687

    
2688
    if not nv_dict:
2689
      return result
2690

    
2691
    node_lvs = self.rpc.call_lv_list(nodes, [])
2692
    for node, node_res in node_lvs.items():
2693
      if node_res.offline:
2694
        continue
2695
      msg = node_res.fail_msg
2696
      if msg:
2697
        logging.warning("Error enumerating LVs on node %s: %s", node, msg)
2698
        res_nodes[node] = msg
2699
        continue
2700

    
2701
      lvs = node_res.payload
2702
      for lv_name, (_, _, lv_online) in lvs.items():
2703
        inst = nv_dict.pop((node, lv_name), None)
2704
        if (not lv_online and inst is not None
2705
            and inst.name not in res_instances):
2706
          res_instances.append(inst.name)
2707

    
2708
    # any leftover items in nv_dict are missing LVs, let's arrange the
2709
    # data better
2710
    for key, inst in nv_dict.iteritems():
2711
      if inst.name not in res_missing:
2712
        res_missing[inst.name] = []
2713
      res_missing[inst.name].append(key)
2714

    
2715
    return result
2716

    
2717

    
2718
class LUClusterRepairDiskSizes(NoHooksLU):
2719
  """Verifies the cluster disks sizes.
2720

2721
  """
2722
  REQ_BGL = False
2723

    
2724
  def ExpandNames(self):
2725
    if self.op.instances:
2726
      self.wanted_names = _GetWantedInstances(self, self.op.instances)
2727
      self.needed_locks = {
2728
        locking.LEVEL_NODE: [],
2729
        locking.LEVEL_INSTANCE: self.wanted_names,
2730
        }
2731
      self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE
2732
    else:
2733
      self.wanted_names = None
2734
      self.needed_locks = {
2735
        locking.LEVEL_NODE: locking.ALL_SET,
2736
        locking.LEVEL_INSTANCE: locking.ALL_SET,
2737
        }
2738
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2739

    
2740
  def DeclareLocks(self, level):
2741
    if level == locking.LEVEL_NODE and self.wanted_names is not None:
2742
      self._LockInstancesNodes(primary_only=True)
2743

    
2744
  def CheckPrereq(self):
2745
    """Check prerequisites.
2746

2747
    This only checks the optional instance list against the existing names.
2748

2749
    """
2750
    if self.wanted_names is None:
2751
      self.wanted_names = self.glm.list_owned(locking.LEVEL_INSTANCE)
2752

    
2753
    self.wanted_instances = [self.cfg.GetInstanceInfo(name) for name
2754
                             in self.wanted_names]
2755

    
2756
  def _EnsureChildSizes(self, disk):
2757
    """Ensure children of the disk have the needed disk size.
2758

2759
    This is valid mainly for DRBD8 and fixes an issue where the
2760
    children have smaller disk size.
2761

2762
    @param disk: an L{ganeti.objects.Disk} object
2763

2764
    """
2765
    if disk.dev_type == constants.LD_DRBD8:
2766
      assert disk.children, "Empty children for DRBD8?"
2767
      fchild = disk.children[0]
2768
      mismatch = fchild.size < disk.size
2769
      if mismatch:
2770
        self.LogInfo("Child disk has size %d, parent %d, fixing",
2771
                     fchild.size, disk.size)
2772
        fchild.size = disk.size
2773

    
2774
      # and we recurse on this child only, not on the metadev
2775
      return self._EnsureChildSizes(fchild) or mismatch
2776
    else:
2777
      return False
2778

    
2779
  def Exec(self, feedback_fn):
2780
    """Verify the size of cluster disks.
2781

2782
    """
2783
    # TODO: check child disks too
2784
    # TODO: check differences in size between primary/secondary nodes
2785
    per_node_disks = {}
2786
    for instance in self.wanted_instances:
2787
      pnode = instance.primary_node
2788
      if pnode not in per_node_disks:
2789
        per_node_disks[pnode] = []
2790
      for idx, disk in enumerate(instance.disks):
2791
        per_node_disks[pnode].append((instance, idx, disk))
2792

    
2793
    changed = []
2794
    for node, dskl in per_node_disks.items():
2795
      newl = [v[2].Copy() for v in dskl]
2796
      for dsk in newl:
2797
        self.cfg.SetDiskID(dsk, node)
2798
      result = self.rpc.call_blockdev_getsize(node, newl)
2799
      if result.fail_msg:
2800
        self.LogWarning("Failure in blockdev_getsize call to node"
2801
                        " %s, ignoring", node)
2802
        continue
2803
      if len(result.payload) != len(dskl):
2804
        logging.warning("Invalid result from node %s: len(dksl)=%d,"
2805
                        " result.payload=%s", node, len(dskl), result.payload)
2806
        self.LogWarning("Invalid result from node %s, ignoring node results",
2807
                        node)
2808
        continue
2809
      for ((instance, idx, disk), size) in zip(dskl, result.payload):
2810
        if size is None:
2811
          self.LogWarning("Disk %d of instance %s did not return size"
2812
                          " information, ignoring", idx, instance.name)
2813
          continue
2814
        if not isinstance(size, (int, long)):
2815
          self.LogWarning("Disk %d of instance %s did not return valid"
2816
                          " size information, ignoring", idx, instance.name)
2817
          continue
2818
        size = size >> 20
2819
        if size != disk.size:
2820
          self.LogInfo("Disk %d of instance %s has mismatched size,"
2821
                       " correcting: recorded %d, actual %d", idx,
2822
                       instance.name, disk.size, size)
2823
          disk.size = size
2824
          self.cfg.Update(instance, feedback_fn)
2825
          changed.append((instance.name, idx, size))
2826
        if self._EnsureChildSizes(disk):
2827
          self.cfg.Update(instance, feedback_fn)
2828
          changed.append((instance.name, idx, disk.size))
2829
    return changed
2830

    
2831

    
2832
class LUClusterRename(LogicalUnit):
2833
  """Rename the cluster.
2834

2835
  """
2836
  HPATH = "cluster-rename"
2837
  HTYPE = constants.HTYPE_CLUSTER
2838

    
2839
  def BuildHooksEnv(self):
2840
    """Build hooks env.
2841

2842
    """
2843
    return {
2844
      "OP_TARGET": self.cfg.GetClusterName(),
2845
      "NEW_NAME": self.op.name,
2846
      }
2847

    
2848
  def BuildHooksNodes(self):
2849
    """Build hooks nodes.
2850

2851
    """
2852
    return ([self.cfg.GetMasterNode()], self.cfg.GetNodeList())
2853

    
2854
  def CheckPrereq(self):
2855
    """Verify that the passed name is a valid one.
2856

2857
    """
2858
    hostname = netutils.GetHostname(name=self.op.name,
2859
                                    family=self.cfg.GetPrimaryIPFamily())
2860

    
2861
    new_name = hostname.name
2862
    self.ip = new_ip = hostname.ip
2863
    old_name = self.cfg.GetClusterName()
2864
    old_ip = self.cfg.GetMasterIP()
2865
    if new_name == old_name and new_ip == old_ip:
2866
      raise errors.OpPrereqError("Neither the name nor the IP address of the"
2867
                                 " cluster has changed",
2868
                                 errors.ECODE_INVAL)
2869
    if new_ip != old_ip:
2870
      if netutils.TcpPing(new_ip, constants.DEFAULT_NODED_PORT):
2871
        raise errors.OpPrereqError("The given cluster IP address (%s) is"
2872
                                   " reachable on the network" %
2873
                                   new_ip, errors.ECODE_NOTUNIQUE)
2874

    
2875
    self.op.name = new_name
2876

    
2877
  def Exec(self, feedback_fn):
2878
    """Rename the cluster.
2879

2880
    """
2881
    clustername = self.op.name
2882
    ip = self.ip
2883

    
2884
    # shutdown the master IP
2885
    master = self.cfg.GetMasterNode()
2886
    result = self.rpc.call_node_stop_master(master, False)
2887
    result.Raise("Could not disable the master role")
2888

    
2889
    try:
2890
      cluster = self.cfg.GetClusterInfo()
2891
      cluster.cluster_name = clustername
2892
      cluster.master_ip = ip
2893
      self.cfg.Update(cluster, feedback_fn)
2894

    
2895
      # update the known hosts file
2896
      ssh.WriteKnownHostsFile(self.cfg, constants.SSH_KNOWN_HOSTS_FILE)
2897
      node_list = self.cfg.GetOnlineNodeList()
2898
      try:
2899
        node_list.remove(master)
2900
      except ValueError:
2901
        pass
2902
      _UploadHelper(self, node_list, constants.SSH_KNOWN_HOSTS_FILE)
2903
    finally:
2904
      result = self.rpc.call_node_start_master(master, False, False)
2905
      msg = result.fail_msg
2906
      if msg:
2907
        self.LogWarning("Could not re-enable the master role on"
2908
                        " the master, please restart manually: %s", msg)
2909

    
2910
    return clustername
2911

    
2912

    
2913
class LUClusterSetParams(LogicalUnit):
2914
  """Change the parameters of the cluster.
2915

2916
  """
2917
  HPATH = "cluster-modify"
2918
  HTYPE = constants.HTYPE_CLUSTER
2919
  REQ_BGL = False
2920

    
2921
  def CheckArguments(self):
2922
    """Check parameters
2923

2924
    """
2925
    if self.op.uid_pool:
2926
      uidpool.CheckUidPool(self.op.uid_pool)
2927

    
2928
    if self.op.add_uids:
2929
      uidpool.CheckUidPool(self.op.add_uids)
2930

    
2931
    if self.op.remove_uids:
2932
      uidpool.CheckUidPool(self.op.remove_uids)
2933

    
2934
  def ExpandNames(self):
2935
    # FIXME: in the future maybe other cluster params won't require checking on
2936
    # all nodes to be modified.
2937
    self.needed_locks = {
2938
      locking.LEVEL_NODE: locking.ALL_SET,
2939
    }
2940
    self.share_locks[locking.LEVEL_NODE] = 1
2941

    
2942
  def BuildHooksEnv(self):
2943
    """Build hooks env.
2944

2945
    """
2946
    return {
2947
      "OP_TARGET": self.cfg.GetClusterName(),
2948
      "NEW_VG_NAME": self.op.vg_name,
2949
      }
2950

    
2951
  def BuildHooksNodes(self):
2952
    """Build hooks nodes.
2953

2954
    """
2955
    mn = self.cfg.GetMasterNode()
2956
    return ([mn], [mn])
2957

    
2958
  def CheckPrereq(self):
2959
    """Check prerequisites.
2960

2961
    This checks whether the given params don't conflict and
2962
    if the given volume group is valid.
2963

2964
    """
2965
    if self.op.vg_name is not None and not self.op.vg_name:
2966
      if self.cfg.HasAnyDiskOfType(constants.LD_LV):
2967
        raise errors.OpPrereqError("Cannot disable lvm storage while lvm-based"
2968
                                   " instances exist", errors.ECODE_INVAL)
2969

    
2970
    if self.op.drbd_helper is not None and not self.op.drbd_helper:
2971
      if self.cfg.HasAnyDiskOfType(constants.LD_DRBD8):
2972
        raise errors.OpPrereqError("Cannot disable drbd helper while"
2973
                                   " drbd-based instances exist",
2974
                                   errors.ECODE_INVAL)
2975

    
2976
    node_list = self.glm.list_owned(locking.LEVEL_NODE)
2977

    
2978
    # if vg_name not None, checks given volume group on all nodes
2979
    if self.op.vg_name:
2980
      vglist = self.rpc.call_vg_list(node_list)
2981
      for node in node_list:
2982
        msg = vglist[node].fail_msg
2983
        if msg:
2984
          # ignoring down node
2985
          self.LogWarning("Error while gathering data on node %s"
2986
                          " (ignoring node): %s", node, msg)
2987
          continue
2988
        vgstatus = utils.CheckVolumeGroupSize(vglist[node].payload,
2989
                                              self.op.vg_name,
2990
                                              constants.MIN_VG_SIZE)
2991
        if vgstatus:
2992
          raise errors.OpPrereqError("Error on node '%s': %s" %
2993
                                     (node, vgstatus), errors.ECODE_ENVIRON)
2994

    
2995
    if self.op.drbd_helper:
2996
      # checks given drbd helper on all nodes
2997
      helpers = self.rpc.call_drbd_helper(node_list)
2998
      for node in node_list:
2999
        ninfo = self.cfg.GetNodeInfo(node)
3000
        if ninfo.offline:
3001
          self.LogInfo("Not checking drbd helper on offline node %s", node)
3002
          continue
3003
        msg = helpers[node].fail_msg
3004
        if msg:
3005
          raise errors.OpPrereqError("Error checking drbd helper on node"
3006
                                     " '%s': %s" % (node, msg),
3007
                                     errors.ECODE_ENVIRON)
3008
        node_helper = helpers[node].payload
3009
        if node_helper != self.op.drbd_helper:
3010
          raise errors.OpPrereqError("Error on node '%s': drbd helper is %s" %
3011
                                     (node, node_helper), errors.ECODE_ENVIRON)
3012

    
3013
    self.cluster = cluster = self.cfg.GetClusterInfo()
3014
    # validate params changes
3015
    if self.op.beparams:
3016
      utils.ForceDictType(self.op.beparams, constants.BES_PARAMETER_TYPES)
3017
      self.new_beparams = cluster.SimpleFillBE(self.op.beparams)
3018

    
3019
    if self.op.ndparams:
3020
      utils.ForceDictType(self.op.ndparams, constants.NDS_PARAMETER_TYPES)
3021
      self.new_ndparams = cluster.SimpleFillND(self.op.ndparams)
3022

    
3023
      # TODO: we need a more general way to handle resetting
3024
      # cluster-level parameters to default values
3025
      if self.new_ndparams["oob_program"] == "":
3026
        self.new_ndparams["oob_program"] = \
3027
            constants.NDC_DEFAULTS[constants.ND_OOB_PROGRAM]
3028

    
3029
    if self.op.nicparams:
3030
      utils.ForceDictType(self.op.nicparams, constants.NICS_PARAMETER_TYPES)
3031
      self.new_nicparams = cluster.SimpleFillNIC(self.op.nicparams)
3032
      objects.NIC.CheckParameterSyntax(self.new_nicparams)
3033
      nic_errors = []
3034

    
3035
      # check all instances for consistency
3036
      for instance in self.cfg.GetAllInstancesInfo().values():
3037
        for nic_idx, nic in enumerate(instance.nics):
3038
          params_copy = copy.deepcopy(nic.nicparams)
3039
          params_filled = objects.FillDict(self.new_nicparams, params_copy)
3040

    
3041
          # check parameter syntax
3042
          try:
3043
            objects.NIC.CheckParameterSyntax(params_filled)
3044
          except errors.ConfigurationError, err:
3045
            nic_errors.append("Instance %s, nic/%d: %s" %
3046
                              (instance.name, nic_idx, err))
3047

    
3048
          # if we're moving instances to routed, check that they have an ip
3049
          target_mode = params_filled[constants.NIC_MODE]
3050
          if target_mode == constants.NIC_MODE_ROUTED and not nic.ip:
3051
            nic_errors.append("Instance %s, nic/%d: routed NIC with no ip"
3052
                              " address" % (instance.name, nic_idx))
3053
      if nic_errors:
3054
        raise errors.OpPrereqError("Cannot apply the change, errors:\n%s" %
3055
                                   "\n".join(nic_errors))
3056

    
3057
    # hypervisor list/parameters
3058
    self.new_hvparams = new_hvp = objects.FillDict(cluster.hvparams, {})
3059
    if self.op.hvparams:
3060
      for hv_name, hv_dict in self.op.hvparams.items():
3061
        if hv_name not in self.new_hvparams:
3062
          self.new_hvparams[hv_name] = hv_dict
3063
        else:
3064
          self.new_hvparams[hv_name].update(hv_dict)
3065

    
3066
    # os hypervisor parameters
3067
    self.new_os_hvp = objects.FillDict(cluster.os_hvp, {})
3068
    if self.op.os_hvp:
3069
      for os_name, hvs in self.op.os_hvp.items():
3070
        if os_name not in self.new_os_hvp:
3071
          self.new_os_hvp[os_name] = hvs
3072
        else:
3073
          for hv_name, hv_dict in hvs.items():
3074
            if hv_name not in self.new_os_hvp[os_name]:
3075
              self.new_os_hvp[os_name][hv_name] = hv_dict
3076
            else:
3077
              self.new_os_hvp[os_name][hv_name].update(hv_dict)
3078

    
3079
    # os parameters
3080
    self.new_osp = objects.FillDict(cluster.osparams, {})
3081
    if self.op.osparams:
3082
      for os_name, osp in self.op.osparams.items():
3083
        if os_name not in self.new_osp:
3084
          self.new_osp[os_name] = {}
3085

    
3086
        self.new_osp[os_name] = _GetUpdatedParams(self.new_osp[os_name], osp,
3087
                                                  use_none=True)
3088

    
3089
        if not self.new_osp[os_name]:
3090
          # we removed all parameters
3091
          del self.new_osp[os_name]
3092
        else:
3093
          # check the parameter validity (remote check)
3094
          _CheckOSParams(self, False, [self.cfg.GetMasterNode()],
3095
                         os_name, self.new_osp[os_name])
3096

    
3097
    # changes to the hypervisor list
3098
    if self.op.enabled_hypervisors is not None:
3099
      self.hv_list = self.op.enabled_hypervisors
3100
      for hv in self.hv_list:
3101
        # if the hypervisor doesn't already exist in the cluster
3102
        # hvparams, we initialize it to empty, and then (in both
3103
        # cases) we make sure to fill the defaults, as we might not
3104
        # have a complete defaults list if the hypervisor wasn't
3105
        # enabled before
3106
        if hv not in new_hvp:
3107
          new_hvp[hv] = {}
3108
        new_hvp[hv] = objects.FillDict(constants.HVC_DEFAULTS[hv], new_hvp[hv])
3109
        utils.ForceDictType(new_hvp[hv], constants.HVS_PARAMETER_TYPES)
3110
    else:
3111
      self.hv_list = cluster.enabled_hypervisors
3112

    
3113
    if self.op.hvparams or self.op.enabled_hypervisors is not None:
3114
      # either the enabled list has changed, or the parameters have, validate
3115
      for hv_name, hv_params in self.new_hvparams.items():
3116
        if ((self.op.hvparams and hv_name in self.op.hvparams) or
3117
            (self.op.enabled_hypervisors and
3118
             hv_name in self.op.enabled_hypervisors)):
3119
          # either this is a new hypervisor, or its parameters have changed
3120
          hv_class = hypervisor.GetHypervisor(hv_name)
3121
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3122
          hv_class.CheckParameterSyntax(hv_params)
3123
          _CheckHVParams(self, node_list, hv_name, hv_params)
3124

    
3125
    if self.op.os_hvp:
3126
      # no need to check any newly-enabled hypervisors, since the
3127
      # defaults have already been checked in the above code-block
3128
      for os_name, os_hvp in self.new_os_hvp.items():
3129
        for hv_name, hv_params in os_hvp.items():
3130
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3131
          # we need to fill in the new os_hvp on top of the actual hv_p
3132
          cluster_defaults = self.new_hvparams.get(hv_name, {})
3133
          new_osp = objects.FillDict(cluster_defaults, hv_params)
3134
          hv_class = hypervisor.GetHypervisor(hv_name)
3135
          hv_class.CheckParameterSyntax(new_osp)
3136
          _CheckHVParams(self, node_list, hv_name, new_osp)
3137

    
3138
    if self.op.default_iallocator:
3139
      alloc_script = utils.FindFile(self.op.default_iallocator,
3140
                                    constants.IALLOCATOR_SEARCH_PATH,
3141
                                    os.path.isfile)
3142
      if alloc_script is None:
3143
        raise errors.OpPrereqError("Invalid default iallocator script '%s'"
3144
                                   " specified" % self.op.default_iallocator,
3145
                                   errors.ECODE_INVAL)
3146

    
3147
  def Exec(self, feedback_fn):
3148
    """Change the parameters of the cluster.
3149

3150
    """
3151
    if self.op.vg_name is not None:
3152
      new_volume = self.op.vg_name
3153
      if not new_volume:
3154
        new_volume = None
3155
      if new_volume != self.cfg.GetVGName():
3156
        self.cfg.SetVGName(new_volume)
3157
      else:
3158
        feedback_fn("Cluster LVM configuration already in desired"
3159
                    " state, not changing")
3160
    if self.op.drbd_helper is not None:
3161
      new_helper = self.op.drbd_helper
3162
      if not new_helper:
3163
        new_helper = None
3164
      if new_helper != self.cfg.GetDRBDHelper():
3165
        self.cfg.SetDRBDHelper(new_helper)
3166
      else:
3167
        feedback_fn("Cluster DRBD helper already in desired state,"
3168
                    " not changing")
3169
    if self.op.hvparams:
3170
      self.cluster.hvparams = self.new_hvparams
3171
    if self.op.os_hvp:
3172
      self.cluster.os_hvp = self.new_os_hvp
3173
    if self.op.enabled_hypervisors is not None:
3174
      self.cluster.hvparams = self.new_hvparams
3175
      self.cluster.enabled_hypervisors = self.op.enabled_hypervisors
3176
    if self.op.beparams:
3177
      self.cluster.beparams[constants.PP_DEFAULT] = self.new_beparams
3178
    if self.op.nicparams:
3179
      self.cluster.nicparams[constants.PP_DEFAULT] = self.new_nicparams
3180
    if self.op.osparams:
3181
      self.cluster.osparams = self.new_osp
3182
    if self.op.ndparams:
3183
      self.cluster.ndparams = self.new_ndparams
3184

    
3185
    if self.op.candidate_pool_size is not None:
3186
      self.cluster.candidate_pool_size = self.op.candidate_pool_size
3187
      # we need to update the pool size here, otherwise the save will fail
3188
      _AdjustCandidatePool(self, [])
3189

    
3190
    if self.op.maintain_node_health is not None:
3191
      self.cluster.maintain_node_health = self.op.maintain_node_health
3192

    
3193
    if self.op.prealloc_wipe_disks is not None:
3194
      self.cluster.prealloc_wipe_disks = self.op.prealloc_wipe_disks
3195

    
3196
    if self.op.add_uids is not None:
3197
      uidpool.AddToUidPool(self.cluster.uid_pool, self.op.add_uids)
3198

    
3199
    if self.op.remove_uids is not None:
3200
      uidpool.RemoveFromUidPool(self.cluster.uid_pool, self.op.remove_uids)
3201

    
3202
    if self.op.uid_pool is not None:
3203
      self.cluster.uid_pool = self.op.uid_pool
3204

    
3205
    if self.op.default_iallocator is not None:
3206
      self.cluster.default_iallocator = self.op.default_iallocator
3207

    
3208
    if self.op.reserved_lvs is not None:
3209
      self.cluster.reserved_lvs = self.op.reserved_lvs
3210

    
3211
    def helper_os(aname, mods, desc):
3212
      desc += " OS list"
3213
      lst = getattr(self.cluster, aname)
3214
      for key, val in mods:
3215
        if key == constants.DDM_ADD:
3216
          if val in lst:
3217
            feedback_fn("OS %s already in %s, ignoring" % (val, desc))
3218
          else:
3219
            lst.append(val)
3220
        elif key == constants.DDM_REMOVE:
3221
          if val in lst:
3222
            lst.remove(val)
3223
          else:
3224
            feedback_fn("OS %s not found in %s, ignoring" % (val, desc))
3225
        else:
3226
          raise errors.ProgrammerError("Invalid modification '%s'" % key)
3227

    
3228
    if self.op.hidden_os:
3229
      helper_os("hidden_os", self.op.hidden_os, "hidden")
3230

    
3231
    if self.op.blacklisted_os:
3232
      helper_os("blacklisted_os", self.op.blacklisted_os, "blacklisted")
3233

    
3234
    if self.op.master_netdev:
3235
      master = self.cfg.GetMasterNode()
3236
      feedback_fn("Shutting down master ip on the current netdev (%s)" %
3237
                  self.cluster.master_netdev)
3238
      result = self.rpc.call_node_stop_master(master, False)
3239
      result.Raise("Could not disable the master ip")
3240
      feedback_fn("Changing master_netdev from %s to %s" %
3241
                  (self.cluster.master_netdev, self.op.master_netdev))
3242
      self.cluster.master_netdev = self.op.master_netdev
3243

    
3244
    self.cfg.Update(self.cluster, feedback_fn)
3245

    
3246
    if self.op.master_netdev:
3247
      feedback_fn("Starting the master ip on the new master netdev (%s)" %
3248
                  self.op.master_netdev)
3249
      result = self.rpc.call_node_start_master(master, False, False)
3250
      if result.fail_msg:
3251
        self.LogWarning("Could not re-enable the master ip on"
3252
                        " the master, please restart manually: %s",
3253
                        result.fail_msg)
3254

    
3255

    
3256
def _UploadHelper(lu, nodes, fname):
3257
  """Helper for uploading a file and showing warnings.
3258

3259
  """
3260
  if os.path.exists(fname):
3261
    result = lu.rpc.call_upload_file(nodes, fname)
3262
    for to_node, to_result in result.items():
3263
      msg = to_result.fail_msg
3264
      if msg:
3265
        msg = ("Copy of file %s to node %s failed: %s" %
3266
               (fname, to_node, msg))
3267
        lu.proc.LogWarning(msg)
3268

    
3269

    
3270
def _ComputeAncillaryFiles(cluster, redist):
3271
  """Compute files external to Ganeti which need to be consistent.
3272

3273
  @type redist: boolean
3274
  @param redist: Whether to include files which need to be redistributed
3275

3276
  """
3277
  # Compute files for all nodes
3278
  files_all = set([
3279
    constants.SSH_KNOWN_HOSTS_FILE,
3280
    constants.CONFD_HMAC_KEY,
3281
    constants.CLUSTER_DOMAIN_SECRET_FILE,
3282
    ])
3283

    
3284
  if not redist:
3285
    files_all.update(constants.ALL_CERT_FILES)
3286
    files_all.update(ssconf.SimpleStore().GetFileList())
3287

    
3288
  if cluster.modify_etc_hosts:
3289
    files_all.add(constants.ETC_HOSTS)
3290

    
3291
  # Files which must either exist on all nodes or on none
3292
  files_all_opt = set([
3293
    constants.RAPI_USERS_FILE,
3294
    ])
3295

    
3296
  # Files which should only be on master candidates
3297
  files_mc = set()
3298
  if not redist:
3299
    files_mc.add(constants.CLUSTER_CONF_FILE)
3300

    
3301
  # Files which should only be on VM-capable nodes
3302
  files_vm = set(filename
3303
    for hv_name in cluster.enabled_hypervisors
3304
    for filename in hypervisor.GetHypervisor(hv_name).GetAncillaryFiles())
3305

    
3306
  # Filenames must be unique
3307
  assert (len(files_all | files_all_opt | files_mc | files_vm) ==
3308
          sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
3309
         "Found file listed in more than one file list"
3310

    
3311
  return (files_all, files_all_opt, files_mc, files_vm)
3312

    
3313

    
3314
def _RedistributeAncillaryFiles(lu, additional_nodes=None, additional_vm=True):
3315
  """Distribute additional files which are part of the cluster configuration.
3316

3317
  ConfigWriter takes care of distributing the config and ssconf files, but
3318
  there are more files which should be distributed to all nodes. This function
3319
  makes sure those are copied.
3320

3321
  @param lu: calling logical unit
3322
  @param additional_nodes: list of nodes not in the config to distribute to
3323
  @type additional_vm: boolean
3324
  @param additional_vm: whether the additional nodes are vm-capable or not
3325

3326
  """
3327
  # Gather target nodes
3328
  cluster = lu.cfg.GetClusterInfo()
3329
  master_info = lu.cfg.GetNodeInfo(lu.cfg.GetMasterNode())
3330

    
3331
  online_nodes = lu.cfg.GetOnlineNodeList()
3332
  vm_nodes = lu.cfg.GetVmCapableNodeList()
3333

    
3334
  if additional_nodes is not None:
3335
    online_nodes.extend(additional_nodes)
3336
    if additional_vm:
3337
      vm_nodes.extend(additional_nodes)
3338

    
3339
  # Never distribute to master node
3340
  for nodelist in [online_nodes, vm_nodes]:
3341
    if master_info.name in nodelist:
3342
      nodelist.remove(master_info.name)
3343

    
3344
  # Gather file lists
3345
  (files_all, files_all_opt, files_mc, files_vm) = \
3346
    _ComputeAncillaryFiles(cluster, True)
3347

    
3348
  # Never re-distribute configuration file from here
3349
  assert not (constants.CLUSTER_CONF_FILE in files_all or
3350
              constants.CLUSTER_CONF_FILE in files_vm)
3351
  assert not files_mc, "Master candidates not handled in this function"
3352

    
3353
  filemap = [
3354
    (online_nodes, files_all),
3355
    (online_nodes, files_all_opt),
3356
    (vm_nodes, files_vm),
3357
    ]
3358

    
3359
  # Upload the files
3360
  for (node_list, files) in filemap:
3361
    for fname in files:
3362
      _UploadHelper(lu, node_list, fname)
3363

    
3364

    
3365
class LUClusterRedistConf(NoHooksLU):
3366
  """Force the redistribution of cluster configuration.
3367

3368
  This is a very simple LU.
3369

3370
  """
3371
  REQ_BGL = False
3372

    
3373
  def ExpandNames(self):
3374
    self.needed_locks = {
3375
      locking.LEVEL_NODE: locking.ALL_SET,
3376
    }
3377
    self.share_locks[locking.LEVEL_NODE] = 1
3378

    
3379
  def Exec(self, feedback_fn):
3380
    """Redistribute the configuration.
3381

3382
    """
3383
    self.cfg.Update(self.cfg.GetClusterInfo(), feedback_fn)
3384
    _RedistributeAncillaryFiles(self)
3385

    
3386

    
3387
def _WaitForSync(lu, instance, disks=None, oneshot=False):
3388
  """Sleep and poll for an instance's disk to sync.
3389

3390
  """
3391
  if not instance.disks or disks is not None and not disks:
3392
    return True
3393

    
3394
  disks = _ExpandCheckDisks(instance, disks)
3395

    
3396
  if not oneshot:
3397
    lu.proc.LogInfo("Waiting for instance %s to sync disks." % instance.name)
3398

    
3399
  node = instance.primary_node
3400

    
3401
  for dev in disks:
3402
    lu.cfg.SetDiskID(dev, node)
3403

    
3404
  # TODO: Convert to utils.Retry
3405

    
3406
  retries = 0
3407
  degr_retries = 10 # in seconds, as we sleep 1 second each time
3408
  while True:
3409
    max_time = 0
3410
    done = True
3411
    cumul_degraded = False
3412
    rstats = lu.rpc.call_blockdev_getmirrorstatus(node, disks)
3413
    msg = rstats.fail_msg
3414
    if msg:
3415
      lu.LogWarning("Can't get any data from node %s: %s", node, msg)
3416
      retries += 1
3417
      if retries >= 10:
3418
        raise errors.RemoteError("Can't contact node %s for mirror data,"
3419
                                 " aborting." % node)
3420
      time.sleep(6)
3421
      continue
3422
    rstats = rstats.payload
3423
    retries = 0
3424
    for i, mstat in enumerate(rstats):
3425
      if mstat is None:
3426
        lu.LogWarning("Can't compute data for node %s/%s",
3427
                           node, disks[i].iv_name)
3428
        continue
3429

    
3430
      cumul_degraded = (cumul_degraded or
3431
                        (mstat.is_degraded and mstat.sync_percent is None))
3432
      if mstat.sync_percent is not None:
3433
        done = False
3434
        if mstat.estimated_time is not None:
3435
          rem_time = ("%s remaining (estimated)" %
3436
                      utils.FormatSeconds(mstat.estimated_time))
3437
          max_time = mstat.estimated_time
3438
        else:
3439
          rem_time = "no time estimate"
3440
        lu.proc.LogInfo("- device %s: %5.2f%% done, %s" %
3441
                        (disks[i].iv_name, mstat.sync_percent, rem_time))
3442

    
3443
    # if we're done but degraded, let's do a few small retries, to
3444
    # make sure we see a stable and not transient situation; therefore
3445
    # we force restart of the loop
3446
    if (done or oneshot) and cumul_degraded and degr_retries > 0:
3447
      logging.info("Degraded disks found, %d retries left", degr_retries)
3448
      degr_retries -= 1
3449
      time.sleep(1)
3450
      continue
3451

    
3452
    if done or oneshot:
3453
      break
3454

    
3455
    time.sleep(min(60, max_time))
3456

    
3457
  if done:
3458
    lu.proc.LogInfo("Instance %s's disks are in sync." % instance.name)
3459
  return not cumul_degraded
3460

    
3461

    
3462
def _CheckDiskConsistency(lu, dev, node, on_primary, ldisk=False):
3463
  """Check that mirrors are not degraded.
3464

3465
  The ldisk parameter, if True, will change the test from the
3466
  is_degraded attribute (which represents overall non-ok status for
3467
  the device(s)) to the ldisk (representing the local storage status).
3468

3469
  """
3470
  lu.cfg.SetDiskID(dev, node)
3471

    
3472
  result = True
3473

    
3474
  if on_primary or dev.AssembleOnSecondary():
3475
    rstats = lu.rpc.call_blockdev_find(node, dev)
3476
    msg = rstats.fail_msg
3477
    if msg:
3478
      lu.LogWarning("Can't find disk on node %s: %s", node, msg)
3479
      result = False
3480
    elif not rstats.payload:
3481
      lu.LogWarning("Can't find disk on node %s", node)
3482
      result = False
3483
    else:
3484
      if ldisk:
3485
        result = result and rstats.payload.ldisk_status == constants.LDS_OKAY
3486
      else:
3487
        result = result and not rstats.payload.is_degraded
3488

    
3489
  if dev.children:
3490
    for child in dev.children:
3491
      result = result and _CheckDiskConsistency(lu, child, node, on_primary)
3492

    
3493
  return result
3494

    
3495

    
3496
class LUOobCommand(NoHooksLU):
3497
  """Logical unit for OOB handling.
3498

3499
  """
3500
  REG_BGL = False
3501
  _SKIP_MASTER = (constants.OOB_POWER_OFF, constants.OOB_POWER_CYCLE)
3502

    
3503
  def ExpandNames(self):
3504
    """Gather locks we need.
3505

3506
    """
3507
    if self.op.node_names:
3508
      self.op.node_names = _GetWantedNodes(self, self.op.node_names)
3509
      lock_names = self.op.node_names
3510
    else:
3511
      lock_names = locking.ALL_SET
3512

    
3513
    self.needed_locks = {
3514
      locking.LEVEL_NODE: lock_names,
3515
      }
3516

    
3517
  def CheckPrereq(self):
3518
    """Check prerequisites.
3519

3520
    This checks:
3521
     - the node exists in the configuration
3522
     - OOB is supported
3523

3524
    Any errors are signaled by raising errors.OpPrereqError.
3525

3526
    """
3527
    self.nodes = []
3528
    self.master_node = self.cfg.GetMasterNode()
3529

    
3530
    assert self.op.power_delay >= 0.0
3531

    
3532
    if self.op.node_names:
3533
      if (self.op.command in self._SKIP_MASTER and
3534
          self.master_node in self.op.node_names):
3535
        master_node_obj = self.cfg.GetNodeInfo(self.master_node)
3536
        master_oob_handler = _SupportsOob(self.cfg, master_node_obj)
3537

    
3538
        if master_oob_handler:
3539
          additional_text = ("run '%s %s %s' if you want to operate on the"
3540
                             " master regardless") % (master_oob_handler,
3541
                                                      self.op.command,
3542
                                                      self.master_node)
3543
        else:
3544
          additional_text = "it does not support out-of-band operations"
3545

    
3546
        raise errors.OpPrereqError(("Operating on the master node %s is not"
3547
                                    " allowed for %s; %s") %
3548
                                   (self.master_node, self.op.command,
3549
                                    additional_text), errors.ECODE_INVAL)
3550
    else:
3551
      self.op.node_names = self.cfg.GetNodeList()
3552
      if self.op.command in self._SKIP_MASTER:
3553
        self.op.node_names.remove(self.master_node)
3554

    
3555
    if self.op.command in self._SKIP_MASTER:
3556
      assert self.master_node not in self.op.node_names
3557

    
3558
    for node_name in self.op.node_names:
3559
      node = self.cfg.GetNodeInfo(node_name)
3560

    
3561
      if node is None:
3562
        raise errors.OpPrereqError("Node %s not found" % node_name,
3563
                                   errors.ECODE_NOENT)
3564
      else:
3565
        self.nodes.append(node)
3566

    
3567
      if (not self.op.ignore_status and
3568
          (self.op.command == constants.OOB_POWER_OFF and not node.offline)):
3569
        raise errors.OpPrereqError(("Cannot power off node %s because it is"
3570
                                    " not marked offline") % node_name,
3571
                                   errors.ECODE_STATE)
3572

    
3573
  def Exec(self, feedback_fn):
3574
    """Execute OOB and return result if we expect any.
3575

3576
    """
3577
    master_node = self.master_node
3578
    ret = []
3579

    
3580
    for idx, node in enumerate(utils.NiceSort(self.nodes,
3581
                                              key=lambda node: node.name)):
3582
      node_entry = [(constants.RS_NORMAL, node.name)]
3583
      ret.append(node_entry)
3584

    
3585
      oob_program = _SupportsOob(self.cfg, node)
3586

    
3587
      if not oob_program:
3588
        node_entry.append((constants.RS_UNAVAIL, None))
3589
        continue
3590

    
3591
      logging.info("Executing out-of-band command '%s' using '%s' on %s",
3592
                   self.op.command, oob_program, node.name)
3593
      result = self.rpc.call_run_oob(master_node, oob_program,
3594
                                     self.op.command, node.name,
3595
                                     self.op.timeout)
3596

    
3597
      if result.fail_msg:
3598
        self.LogWarning("Out-of-band RPC failed on node '%s': %s",
3599
                        node.name, result.fail_msg)
3600
        node_entry.append((constants.RS_NODATA, None))
3601
      else:
3602
        try:
3603
          self._CheckPayload(result)
3604
        except errors.OpExecError, err:
3605
          self.LogWarning("Payload returned by node '%s' is not valid: %s",
3606
                          node.name, err)
3607
          node_entry.append((constants.RS_NODATA, None))
3608
        else:
3609
          if self.op.command == constants.OOB_HEALTH:
3610
            # For health we should log important events
3611
            for item, status in result.payload:
3612
              if status in [constants.OOB_STATUS_WARNING,
3613
                            constants.OOB_STATUS_CRITICAL]:
3614
                self.LogWarning("Item '%s' on node '%s' has status '%s'",
3615
                                item, node.name, status)
3616

    
3617
          if self.op.command == constants.OOB_POWER_ON:
3618
            node.powered = True
3619
          elif self.op.command == constants.OOB_POWER_OFF:
3620
            node.powered = False
3621
          elif self.op.command == constants.OOB_POWER_STATUS:
3622
            powered = result.payload[constants.OOB_POWER_STATUS_POWERED]
3623
            if powered != node.powered:
3624
              logging.warning(("Recorded power state (%s) of node '%s' does not"
3625
                               " match actual power state (%s)"), node.powered,
3626
                              node.name, powered)
3627

    
3628
          # For configuration changing commands we should update the node
3629
          if self.op.command in (constants.OOB_POWER_ON,
3630
                                 constants.OOB_POWER_OFF):
3631
            self.cfg.Update(node, feedback_fn)
3632

    
3633
          node_entry.append((constants.RS_NORMAL, result.payload))
3634

    
3635
          if (self.op.command == constants.OOB_POWER_ON and
3636
              idx < len(self.nodes) - 1):
3637
            time.sleep(self.op.power_delay)
3638

    
3639
    return ret
3640

    
3641
  def _CheckPayload(self, result):
3642
    """Checks if the payload is valid.
3643

3644
    @param result: RPC result
3645
    @raises errors.OpExecError: If payload is not valid
3646

3647
    """
3648
    errs = []
3649
    if self.op.command == constants.OOB_HEALTH:
3650
      if not isinstance(result.payload, list):
3651
        errs.append("command 'health' is expected to return a list but got %s" %
3652
                    type(result.payload))
3653
      else:
3654
        for item, status in result.payload:
3655
          if status not in constants.OOB_STATUSES:
3656
            errs.append("health item '%s' has invalid status '%s'" %
3657
                        (item, status))
3658

    
3659
    if self.op.command == constants.OOB_POWER_STATUS:
3660
      if not isinstance(result.payload, dict):
3661
        errs.append("power-status is expected to return a dict but got %s" %
3662
                    type(result.payload))
3663

    
3664
    if self.op.command in [
3665
        constants.OOB_POWER_ON,
3666
        constants.OOB_POWER_OFF,
3667
        constants.OOB_POWER_CYCLE,
3668
        ]:
3669
      if result.payload is not None:
3670
        errs.append("%s is expected to not return payload but got '%s'" %
3671
                    (self.op.command, result.payload))
3672

    
3673
    if errs:
3674
      raise errors.OpExecError("Check of out-of-band payload failed due to %s" %
3675
                               utils.CommaJoin(errs))
3676

    
3677
class _OsQuery(_QueryBase):
3678
  FIELDS = query.OS_FIELDS
3679

    
3680
  def ExpandNames(self, lu):
3681
    # Lock all nodes in shared mode
3682
    # Temporary removal of locks, should be reverted later
3683
    # TODO: reintroduce locks when they are lighter-weight
3684
    lu.needed_locks = {}
3685
    #self.share_locks[locking.LEVEL_NODE] = 1
3686
    #self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET
3687

    
3688
    # The following variables interact with _QueryBase._GetNames
3689
    if self.names:
3690
      self.wanted = self.names
3691
    else:
3692
      self.wanted = locking.ALL_SET
3693

    
3694
    self.do_locking = self.use_locking
3695

    
3696
  def DeclareLocks(self, lu, level):
3697
    pass
3698

    
3699
  @staticmethod
3700
  def _DiagnoseByOS(rlist):
3701
    """Remaps a per-node return list into an a per-os per-node dictionary
3702

3703
    @param rlist: a map with node names as keys and OS objects as values
3704

3705
    @rtype: dict
3706
    @return: a dictionary with osnames as keys and as value another
3707
        map, with nodes as keys and tuples of (path, status, diagnose,
3708
        variants, parameters, api_versions) as values, eg::
3709

3710
          {"debian-etch": {"node1": [(/usr/lib/..., True, "", [], []),
3711
                                     (/srv/..., False, "invalid api")],
3712
                           "node2": [(/srv/..., True, "", [], [])]}
3713
          }
3714

3715
    """
3716
    all_os = {}
3717
    # we build here the list of nodes that didn't fail the RPC (at RPC
3718
    # level), so that nodes with a non-responding node daemon don't
3719
    # make all OSes invalid
3720
    good_nodes = [node_name for node_name in rlist
3721
                  if not rlist[node_name].fail_msg]
3722
    for node_name, nr in rlist.items():
3723
      if nr.fail_msg or not nr.payload:
3724
        continue
3725
      for (name, path, status, diagnose, variants,
3726
           params, api_versions) in nr.payload:
3727
        if name not in all_os:
3728
          # build a list of nodes for this os containing empty lists
3729
          # for each node in node_list
3730
          all_os[name] = {}
3731
          for nname in good_nodes:
3732
            all_os[name][nname] = []
3733
        # convert params from [name, help] to (name, help)
3734
        params = [tuple(v) for v in params]
3735
        all_os[name][node_name].append((path, status, diagnose,
3736
                                        variants, params, api_versions))
3737
    return all_os
3738

    
3739
  def _GetQueryData(self, lu):
3740
    """Computes the list of nodes and their attributes.
3741

3742
    """
3743
    # Locking is not used
3744
    assert not (compat.any(lu.glm.is_owned(level)
3745
                           for level in locking.LEVELS
3746
                           if level != locking.LEVEL_CLUSTER) or
3747
                self.do_locking or self.use_locking)
3748

    
3749
    valid_nodes = [node.name
3750
                   for node in lu.cfg.GetAllNodesInfo().values()
3751
                   if not node.offline and node.vm_capable]
3752
    pol = self._DiagnoseByOS(lu.rpc.call_os_diagnose(valid_nodes))
3753
    cluster = lu.cfg.GetClusterInfo()
3754

    
3755
    data = {}
3756

    
3757
    for (os_name, os_data) in pol.items():
3758
      info = query.OsInfo(name=os_name, valid=True, node_status=os_data,
3759
                          hidden=(os_name in cluster.hidden_os),
3760
                          blacklisted=(os_name in cluster.blacklisted_os))
3761

    
3762
      variants = set()
3763
      parameters = set()
3764
      api_versions = set()
3765

    
3766
      for idx, osl in enumerate(os_data.values()):
3767
        info.valid = bool(info.valid and osl and osl[0][1])
3768
        if not info.valid:
3769
          break
3770

    
3771
        (node_variants, node_params, node_api) = osl[0][3:6]
3772
        if idx == 0:
3773
          # First entry
3774
          variants.update(node_variants)
3775
          parameters.update(node_params)
3776
          api_versions.update(node_api)
3777
        else:
3778
          # Filter out inconsistent values
3779
          variants.intersection_update(node_variants)
3780
          parameters.intersection_update(node_params)
3781
          api_versions.intersection_update(node_api)
3782

    
3783
      info.variants = list(variants)
3784
      info.parameters = list(parameters)
3785
      info.api_versions = list(api_versions)
3786

    
3787
      data[os_name] = info
3788

    
3789
    # Prepare data in requested order
3790
    return [data[name] for name in self._GetNames(lu, pol.keys(), None)
3791
            if name in data]
3792

    
3793

    
3794
class LUOsDiagnose(NoHooksLU):
3795
  """Logical unit for OS diagnose/query.
3796

3797
  """
3798
  REQ_BGL = False
3799

    
3800
  @staticmethod
3801
  def _BuildFilter(fields, names):
3802
    """Builds a filter for querying OSes.
3803

3804
    """
3805
    name_filter = qlang.MakeSimpleFilter("name", names)
3806

    
3807
    # Legacy behaviour: Hide hidden, blacklisted or invalid OSes if the
3808
    # respective field is not requested
3809
    status_filter = [[qlang.OP_NOT, [qlang.OP_TRUE, fname]]
3810
                     for fname in ["hidden", "blacklisted"]
3811
                     if fname not in fields]
3812
    if "valid" not in fields:
3813
      status_filter.append([qlang.OP_TRUE, "valid"])
3814

    
3815
    if status_filter:
3816
      status_filter.insert(0, qlang.OP_AND)
3817
    else:
3818
      status_filter = None
3819

    
3820
    if name_filter and status_filter:
3821
      return [qlang.OP_AND, name_filter, status_filter]
3822
    elif name_filter:
3823
      return name_filter
3824
    else:
3825
      return status_filter
3826

    
3827
  def CheckArguments(self):
3828
    self.oq = _OsQuery(self._BuildFilter(self.op.output_fields, self.op.names),
3829
                       self.op.output_fields, False)
3830

    
3831
  def ExpandNames(self):
3832
    self.oq.ExpandNames(self)
3833

    
3834
  def Exec(self, feedback_fn):
3835
    return self.oq.OldStyleQuery(self)
3836

    
3837

    
3838
class LUNodeRemove(LogicalUnit):
3839
  """Logical unit for removing a node.
3840

3841
  """
3842
  HPATH = "node-remove"
3843
  HTYPE = constants.HTYPE_NODE
3844

    
3845
  def BuildHooksEnv(self):
3846
    """Build hooks env.
3847

3848
    This doesn't run on the target node in the pre phase as a failed
3849
    node would then be impossible to remove.
3850

3851
    """
3852
    return {
3853
      "OP_TARGET": self.op.node_name,
3854
      "NODE_NAME": self.op.node_name,
3855
      }
3856

    
3857
  def BuildHooksNodes(self):
3858
    """Build hooks nodes.
3859

3860
    """
3861
    all_nodes = self.cfg.GetNodeList()
3862
    try:
3863
      all_nodes.remove(self.op.node_name)
3864
    except ValueError:
3865
      logging.warning("Node '%s', which is about to be removed, was not found"
3866
                      " in the list of all nodes", self.op.node_name)
3867
    return (all_nodes, all_nodes)
3868

    
3869
  def CheckPrereq(self):
3870
    """Check prerequisites.
3871

3872
    This checks:
3873
     - the node exists in the configuration
3874
     - it does not have primary or secondary instances
3875
     - it's not the master
3876

3877
    Any errors are signaled by raising errors.OpPrereqError.
3878

3879
    """
3880
    self.op.node_name = _ExpandNodeName(self.cfg, self.op.node_name)
3881
    node = self.cfg.GetNodeInfo(self.op.node_name)
3882
    assert node is not None
3883

    
3884
    instance_list = self.cfg.GetInstanceList()
3885

    
3886
    masternode = self.cfg.GetMasterNode()
3887
    if node.name == masternode:
3888
      raise errors.OpPrereqError("Node is the master node, failover to another"
3889
                                 " node is required", errors.ECODE_INVAL)
3890

    
3891
    for instance_name in instance_list:
3892
      instance = self.cfg.GetInstanceInfo(instance_name)
3893
      if node.name in instance.all_nodes:
3894
        raise errors.OpPrereqError("Instance %s is still running on the node,"
3895
                                   " please remove first" % instance_name,
3896
                                   errors.ECODE_INVAL)
3897
    self.op.node_name = node.name
3898
    self.node = node
3899

    
3900
  def Exec(self, feedback_fn):
3901
    """Removes the node from the cluster.
3902

3903
    """
3904
    node = self.node
3905
    logging.info("Stopping the node daemon and removing configs from node %s",
3906
                 node.name)
3907

    
3908
    modify_ssh_setup = self.cfg.GetClusterInfo().modify_ssh_setup
3909

    
3910
    # Promote nodes to master candidate as needed
3911
    _AdjustCandidatePool(self, exceptions=[node.name])
3912
    self.context.RemoveNode(node.name)
3913

    
3914
    # Run post hooks on the node before it's removed
3915
    _RunPostHook(self, node.name)
3916

    
3917
    result = self.rpc.call_node_leave_cluster(node.name, modify_ssh_setup)
3918
    msg = result.fail_msg
3919
    if msg:
3920
      self.LogWarning("Errors encountered on the remote node while leaving"
3921
                      " the cluster: %s", msg)
3922

    
3923
    # Remove node from our /etc/hosts
3924
    if self.cfg.GetClusterInfo().modify_etc_hosts:
3925
      master_node = self.cfg.GetMasterNode()
3926
      result = self.rpc.call_etc_hosts_modify(master_node,
3927
                                              constants.ETC_HOSTS_REMOVE,
3928
                                              node.name, None)
3929
      result.Raise("Can't update hosts file with new host data")
3930
      _RedistributeAncillaryFiles(self)
3931

    
3932

    
3933
class _NodeQuery(_QueryBase):
3934
  FIELDS = query.NODE_FIELDS
3935

    
3936
  def ExpandNames(self, lu):
3937
    lu.needed_locks = {}
3938
    lu.share_locks[locking.LEVEL_NODE] = 1
3939

    
3940
    if self.names:
3941
      self.wanted = _GetWantedNodes(lu, self.names)
3942
    else:
3943
      self.wanted = locking.ALL_SET
3944

    
3945
    self.do_locking = (self.use_locking and
3946
                       query.NQ_LIVE in self.requested_data)
3947

    
3948
    if self.do_locking:
3949
      # if we don't request only static fields, we need to lock the nodes
3950
      lu.needed_locks[locking.LEVEL_NODE] = self.wanted
3951

    
3952
  def DeclareLocks(self, lu, level):
3953
    pass
3954

    
3955
  def _GetQueryData(self, lu):
3956
    """Computes the list of nodes and their attributes.
3957

3958
    """
3959
    all_info = lu.cfg.GetAllNodesInfo()
3960

    
3961
    nodenames = self._GetNames(lu, all_info.keys(), locking.LEVEL_NODE)
3962

    
3963
    # Gather data as requested
3964
    if query.NQ_LIVE in self.requested_data:
3965
      # filter out non-vm_capable nodes
3966
      toquery_nodes = [name for name in nodenames if all_info[name].vm_capable]
3967

    
3968
      node_data = lu.rpc.call_node_info(toquery_nodes, lu.cfg.GetVGName(),
3969
                                        lu.cfg.GetHypervisorType())
3970
      live_data = dict((name, nresult.payload)
3971
                       for (name, nresult) in node_data.items()
3972
                       if not nresult.fail_msg and nresult.payload)
3973
    else:
3974
      live_data = None
3975

    
3976
    if query.NQ_INST in self.requested_data:
3977
      node_to_primary = dict([(name, set()) for name in nodenames])
3978
      node_to_secondary = dict([(name, set()) for name in nodenames])
3979

    
3980
      inst_data = lu.cfg.GetAllInstancesInfo()
3981

    
3982
      for inst in inst_data.values():
3983
        if inst.primary_node in node_to_primary:
3984
          node_to_primary[inst.primary_node].add(inst.name)
3985
        for secnode in inst.secondary_nodes:
3986
          if secnode in node_to_secondary:
3987
            node_to_secondary[secnode].add(inst.name)
3988
    else:
3989
      node_to_primary = None
3990
      node_to_secondary = None
3991

    
3992
    if query.NQ_OOB in self.requested_data:
3993
      oob_support = dict((name, bool(_SupportsOob(lu.cfg, node)))
3994
                         for name, node in all_info.iteritems())
3995
    else:
3996
      oob_support = None
3997

    
3998
    if query.NQ_GROUP in self.requested_data:
3999
      groups = lu.cfg.GetAllNodeGroupsInfo()
4000
    else:
4001
      groups = {}
4002

    
4003
    return query.NodeQueryData([all_info[name] for name in nodenames],
4004
                               live_data, lu.cfg.GetMasterNode(),
4005
                               node_to_primary, node_to_secondary, groups,
4006
                               oob_support, lu.cfg.GetClusterInfo())
4007

    
4008

    
4009
class LUNodeQuery(NoHooksLU):
4010
  """Logical unit for querying nodes.
4011

4012
  """
4013
  # pylint: disable-msg=W0142
4014
  REQ_BGL = False
4015

    
4016
  def CheckArguments(self):
4017
    self.nq = _NodeQuery(qlang.MakeSimpleFilter("name", self.op.names),
4018
                         self.op.output_fields, self.op.use_locking)
4019

    
4020
  def ExpandNames(self):
4021
    self.nq.ExpandNames(self)
4022

    
4023
  def Exec(self, feedback_fn):
4024
    return self.nq.OldStyleQuery(self)
4025

    
4026

    
4027
class LUNodeQueryvols(NoHooksLU):
4028
  """Logical unit for getting volumes on node(s).
4029

4030
  """
4031
  REQ_BGL = False
4032
  _FIELDS_DYNAMIC = utils.FieldSet("phys", "vg", "name", "size", "instance")
4033
  _FIELDS_STATIC = utils.FieldSet("node")
4034

    
4035
  def CheckArguments(self):
4036
    _CheckOutputFields(static=self._FIELDS_STATIC,
4037
                       dynamic=self._FIELDS_DYNAMIC,
4038
                       selected=self.op.output_fields)
4039

    
4040
  def ExpandNames(self):
4041
    self.needed_locks = {}
4042
    self.share_locks[locking.LEVEL_NODE] = 1
4043
    if not self.op.nodes:
4044
      self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET
4045
    else:
4046
      self.needed_locks[locking.LEVEL_NODE] = \
4047
        _GetWantedNodes(self, self.op.nodes)
4048

    
4049
  def Exec(self, feedback_fn):
4050
    """Computes the list of nodes and their attributes.
4051

4052
    """
4053
    nodenames = self.glm.list_owned(locking.LEVEL_NODE)
4054
    volumes = self.rpc.call_node_volumes(nodenames)
4055

    
4056
    ilist = [self.cfg.GetInstanceInfo(iname) for iname
4057
             in self.cfg.GetInstanceList()]
4058

    
4059
    lv_by_node = dict([(inst, inst.MapLVsByNode()) for inst in ilist])
4060

    
4061
    output = []
4062
    for node in nodenames:
4063
      nresult = volumes[node]
4064
      if nresult.offline:
4065
        continue
4066
      msg = nresult.fail_msg
4067
      if msg:
4068
        self.LogWarning("Can't compute volume data on node %s: %s", node, msg)
4069
        continue
4070

    
4071
      node_vols = nresult.payload[:]
4072
      node_vols.sort(key=lambda vol: vol['dev'])
4073

    
4074
      for vol in node_vols:
4075
        node_output = []
4076
        for field in self.op.output_fields:
4077
          if field == "node":
4078
            val = node
4079
          elif field == "phys":
4080
            val = vol['dev']
4081
          elif field == "vg":
4082
            val = vol['vg']
4083
          elif field == "name":
4084
            val = vol['name']
4085
          elif field == "size":
4086
            val = int(float(vol['size']))
4087
          elif field == "instance":
4088
            for inst in ilist:
4089
              if node not in lv_by_node[inst]:
4090
                continue
4091</