Statistics
| Branch: | Tag: | Revision:

root / lib / cmdlib.py @ 14970c32

History | View | Annotate | Download (435.7 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Module implementing the master-side code."""
23

    
24
# pylint: disable-msg=W0201,C0302
25

    
26
# W0201 since most LU attributes are defined in CheckPrereq or similar
27
# functions
28

    
29
# C0302: since we have waaaay to many lines in this module
30

    
31
import os
32
import os.path
33
import time
34
import re
35
import platform
36
import logging
37
import copy
38
import OpenSSL
39
import socket
40
import tempfile
41
import shutil
42
import itertools
43

    
44
from ganeti import ssh
45
from ganeti import utils
46
from ganeti import errors
47
from ganeti import hypervisor
48
from ganeti import locking
49
from ganeti import constants
50
from ganeti import objects
51
from ganeti import serializer
52
from ganeti import ssconf
53
from ganeti import uidpool
54
from ganeti import compat
55
from ganeti import masterd
56
from ganeti import netutils
57
from ganeti import query
58
from ganeti import qlang
59
from ganeti import opcodes
60
from ganeti import ht
61

    
62
import ganeti.masterd.instance # pylint: disable-msg=W0611
63

    
64

    
65
def _SupportsOob(cfg, node):
66
  """Tells if node supports OOB.
67

68
  @type cfg: L{config.ConfigWriter}
69
  @param cfg: The cluster configuration
70
  @type node: L{objects.Node}
71
  @param node: The node
72
  @return: The OOB script if supported or an empty string otherwise
73

74
  """
75
  return cfg.GetNdParams(node)[constants.ND_OOB_PROGRAM]
76

    
77

    
78
class ResultWithJobs:
79
  """Data container for LU results with jobs.
80

81
  Instances of this class returned from L{LogicalUnit.Exec} will be recognized
82
  by L{mcpu.Processor._ProcessResult}. The latter will then submit the jobs
83
  contained in the C{jobs} attribute and include the job IDs in the opcode
84
  result.
85

86
  """
87
  def __init__(self, jobs, **kwargs):
88
    """Initializes this class.
89

90
    Additional return values can be specified as keyword arguments.
91

92
    @type jobs: list of lists of L{opcode.OpCode}
93
    @param jobs: A list of lists of opcode objects
94

95
    """
96
    self.jobs = jobs
97
    self.other = kwargs
98

    
99

    
100
class LogicalUnit(object):
101
  """Logical Unit base class.
102

103
  Subclasses must follow these rules:
104
    - implement ExpandNames
105
    - implement CheckPrereq (except when tasklets are used)
106
    - implement Exec (except when tasklets are used)
107
    - implement BuildHooksEnv
108
    - implement BuildHooksNodes
109
    - redefine HPATH and HTYPE
110
    - optionally redefine their run requirements:
111
        REQ_BGL: the LU needs to hold the Big Ganeti Lock exclusively
112

113
  Note that all commands require root permissions.
114

115
  @ivar dry_run_result: the value (if any) that will be returned to the caller
116
      in dry-run mode (signalled by opcode dry_run parameter)
117

118
  """
119
  HPATH = None
120
  HTYPE = None
121
  REQ_BGL = True
122

    
123
  def __init__(self, processor, op, context, rpc):
124
    """Constructor for LogicalUnit.
125

126
    This needs to be overridden in derived classes in order to check op
127
    validity.
128

129
    """
130
    self.proc = processor
131
    self.op = op
132
    self.cfg = context.cfg
133
    self.glm = context.glm
134
    self.context = context
135
    self.rpc = rpc
136
    # Dicts used to declare locking needs to mcpu
137
    self.needed_locks = None
138
    self.share_locks = dict.fromkeys(locking.LEVELS, 0)
139
    self.add_locks = {}
140
    self.remove_locks = {}
141
    # Used to force good behavior when calling helper functions
142
    self.recalculate_locks = {}
143
    # logging
144
    self.Log = processor.Log # pylint: disable-msg=C0103
145
    self.LogWarning = processor.LogWarning # pylint: disable-msg=C0103
146
    self.LogInfo = processor.LogInfo # pylint: disable-msg=C0103
147
    self.LogStep = processor.LogStep # pylint: disable-msg=C0103
148
    # support for dry-run
149
    self.dry_run_result = None
150
    # support for generic debug attribute
151
    if (not hasattr(self.op, "debug_level") or
152
        not isinstance(self.op.debug_level, int)):
153
      self.op.debug_level = 0
154

    
155
    # Tasklets
156
    self.tasklets = None
157

    
158
    # Validate opcode parameters and set defaults
159
    self.op.Validate(True)
160

    
161
    self.CheckArguments()
162

    
163
  def CheckArguments(self):
164
    """Check syntactic validity for the opcode arguments.
165

166
    This method is for doing a simple syntactic check and ensure
167
    validity of opcode parameters, without any cluster-related
168
    checks. While the same can be accomplished in ExpandNames and/or
169
    CheckPrereq, doing these separate is better because:
170

171
      - ExpandNames is left as as purely a lock-related function
172
      - CheckPrereq is run after we have acquired locks (and possible
173
        waited for them)
174

175
    The function is allowed to change the self.op attribute so that
176
    later methods can no longer worry about missing parameters.
177

178
    """
179
    pass
180

    
181
  def ExpandNames(self):
182
    """Expand names for this LU.
183

184
    This method is called before starting to execute the opcode, and it should
185
    update all the parameters of the opcode to their canonical form (e.g. a
186
    short node name must be fully expanded after this method has successfully
187
    completed). This way locking, hooks, logging, etc. can work correctly.
188

189
    LUs which implement this method must also populate the self.needed_locks
190
    member, as a dict with lock levels as keys, and a list of needed lock names
191
    as values. Rules:
192

193
      - use an empty dict if you don't need any lock
194
      - if you don't need any lock at a particular level omit that level
195
      - don't put anything for the BGL level
196
      - if you want all locks at a level use locking.ALL_SET as a value
197

198
    If you need to share locks (rather than acquire them exclusively) at one
199
    level you can modify self.share_locks, setting a true value (usually 1) for
200
    that level. By default locks are not shared.
201

202
    This function can also define a list of tasklets, which then will be
203
    executed in order instead of the usual LU-level CheckPrereq and Exec
204
    functions, if those are not defined by the LU.
205

206
    Examples::
207

208
      # Acquire all nodes and one instance
209
      self.needed_locks = {
210
        locking.LEVEL_NODE: locking.ALL_SET,
211
        locking.LEVEL_INSTANCE: ['instance1.example.com'],
212
      }
213
      # Acquire just two nodes
214
      self.needed_locks = {
215
        locking.LEVEL_NODE: ['node1.example.com', 'node2.example.com'],
216
      }
217
      # Acquire no locks
218
      self.needed_locks = {} # No, you can't leave it to the default value None
219

220
    """
221
    # The implementation of this method is mandatory only if the new LU is
222
    # concurrent, so that old LUs don't need to be changed all at the same
223
    # time.
224
    if self.REQ_BGL:
225
      self.needed_locks = {} # Exclusive LUs don't need locks.
226
    else:
227
      raise NotImplementedError
228

    
229
  def DeclareLocks(self, level):
230
    """Declare LU locking needs for a level
231

232
    While most LUs can just declare their locking needs at ExpandNames time,
233
    sometimes there's the need to calculate some locks after having acquired
234
    the ones before. This function is called just before acquiring locks at a
235
    particular level, but after acquiring the ones at lower levels, and permits
236
    such calculations. It can be used to modify self.needed_locks, and by
237
    default it does nothing.
238

239
    This function is only called if you have something already set in
240
    self.needed_locks for the level.
241

242
    @param level: Locking level which is going to be locked
243
    @type level: member of ganeti.locking.LEVELS
244

245
    """
246

    
247
  def CheckPrereq(self):
248
    """Check prerequisites for this LU.
249

250
    This method should check that the prerequisites for the execution
251
    of this LU are fulfilled. It can do internode communication, but
252
    it should be idempotent - no cluster or system changes are
253
    allowed.
254

255
    The method should raise errors.OpPrereqError in case something is
256
    not fulfilled. Its return value is ignored.
257

258
    This method should also update all the parameters of the opcode to
259
    their canonical form if it hasn't been done by ExpandNames before.
260

261
    """
262
    if self.tasklets is not None:
263
      for (idx, tl) in enumerate(self.tasklets):
264
        logging.debug("Checking prerequisites for tasklet %s/%s",
265
                      idx + 1, len(self.tasklets))
266
        tl.CheckPrereq()
267
    else:
268
      pass
269

    
270
  def Exec(self, feedback_fn):
271
    """Execute the LU.
272

273
    This method should implement the actual work. It should raise
274
    errors.OpExecError for failures that are somewhat dealt with in
275
    code, or expected.
276

277
    """
278
    if self.tasklets is not None:
279
      for (idx, tl) in enumerate(self.tasklets):
280
        logging.debug("Executing tasklet %s/%s", idx + 1, len(self.tasklets))
281
        tl.Exec(feedback_fn)
282
    else:
283
      raise NotImplementedError
284

    
285
  def BuildHooksEnv(self):
286
    """Build hooks environment for this LU.
287

288
    @rtype: dict
289
    @return: Dictionary containing the environment that will be used for
290
      running the hooks for this LU. The keys of the dict must not be prefixed
291
      with "GANETI_"--that'll be added by the hooks runner. The hooks runner
292
      will extend the environment with additional variables. If no environment
293
      should be defined, an empty dictionary should be returned (not C{None}).
294
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
295
      will not be called.
296

297
    """
298
    raise NotImplementedError
299

    
300
  def BuildHooksNodes(self):
301
    """Build list of nodes to run LU's hooks.
302

303
    @rtype: tuple; (list, list)
304
    @return: Tuple containing a list of node names on which the hook
305
      should run before the execution and a list of node names on which the
306
      hook should run after the execution. No nodes should be returned as an
307
      empty list (and not None).
308
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
309
      will not be called.
310

311
    """
312
    raise NotImplementedError
313

    
314
  def HooksCallBack(self, phase, hook_results, feedback_fn, lu_result):
315
    """Notify the LU about the results of its hooks.
316

317
    This method is called every time a hooks phase is executed, and notifies
318
    the Logical Unit about the hooks' result. The LU can then use it to alter
319
    its result based on the hooks.  By default the method does nothing and the
320
    previous result is passed back unchanged but any LU can define it if it
321
    wants to use the local cluster hook-scripts somehow.
322

323
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
324
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
325
    @param hook_results: the results of the multi-node hooks rpc call
326
    @param feedback_fn: function used send feedback back to the caller
327
    @param lu_result: the previous Exec result this LU had, or None
328
        in the PRE phase
329
    @return: the new Exec result, based on the previous result
330
        and hook results
331

332
    """
333
    # API must be kept, thus we ignore the unused argument and could
334
    # be a function warnings
335
    # pylint: disable-msg=W0613,R0201
336
    return lu_result
337

    
338
  def _ExpandAndLockInstance(self):
339
    """Helper function to expand and lock an instance.
340

341
    Many LUs that work on an instance take its name in self.op.instance_name
342
    and need to expand it and then declare the expanded name for locking. This
343
    function does it, and then updates self.op.instance_name to the expanded
344
    name. It also initializes needed_locks as a dict, if this hasn't been done
345
    before.
346

347
    """
348
    if self.needed_locks is None:
349
      self.needed_locks = {}
350
    else:
351
      assert locking.LEVEL_INSTANCE not in self.needed_locks, \
352
        "_ExpandAndLockInstance called with instance-level locks set"
353
    self.op.instance_name = _ExpandInstanceName(self.cfg,
354
                                                self.op.instance_name)
355
    self.needed_locks[locking.LEVEL_INSTANCE] = self.op.instance_name
356

    
357
  def _LockInstancesNodes(self, primary_only=False):
358
    """Helper function to declare instances' nodes for locking.
359

360
    This function should be called after locking one or more instances to lock
361
    their nodes. Its effect is populating self.needed_locks[locking.LEVEL_NODE]
362
    with all primary or secondary nodes for instances already locked and
363
    present in self.needed_locks[locking.LEVEL_INSTANCE].
364

365
    It should be called from DeclareLocks, and for safety only works if
366
    self.recalculate_locks[locking.LEVEL_NODE] is set.
367

368
    In the future it may grow parameters to just lock some instance's nodes, or
369
    to just lock primaries or secondary nodes, if needed.
370

371
    If should be called in DeclareLocks in a way similar to::
372

373
      if level == locking.LEVEL_NODE:
374
        self._LockInstancesNodes()
375

376
    @type primary_only: boolean
377
    @param primary_only: only lock primary nodes of locked instances
378

379
    """
380
    assert locking.LEVEL_NODE in self.recalculate_locks, \
381
      "_LockInstancesNodes helper function called with no nodes to recalculate"
382

    
383
    # TODO: check if we're really been called with the instance locks held
384

    
385
    # For now we'll replace self.needed_locks[locking.LEVEL_NODE], but in the
386
    # future we might want to have different behaviors depending on the value
387
    # of self.recalculate_locks[locking.LEVEL_NODE]
388
    wanted_nodes = []
389
    for instance_name in self.glm.list_owned(locking.LEVEL_INSTANCE):
390
      instance = self.context.cfg.GetInstanceInfo(instance_name)
391
      wanted_nodes.append(instance.primary_node)
392
      if not primary_only:
393
        wanted_nodes.extend(instance.secondary_nodes)
394

    
395
    if self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_REPLACE:
396
      self.needed_locks[locking.LEVEL_NODE] = wanted_nodes
397
    elif self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_APPEND:
398
      self.needed_locks[locking.LEVEL_NODE].extend(wanted_nodes)
399

    
400
    del self.recalculate_locks[locking.LEVEL_NODE]
401

    
402

    
403
class NoHooksLU(LogicalUnit): # pylint: disable-msg=W0223
404
  """Simple LU which runs no hooks.
405

406
  This LU is intended as a parent for other LogicalUnits which will
407
  run no hooks, in order to reduce duplicate code.
408

409
  """
410
  HPATH = None
411
  HTYPE = None
412

    
413
  def BuildHooksEnv(self):
414
    """Empty BuildHooksEnv for NoHooksLu.
415

416
    This just raises an error.
417

418
    """
419
    raise AssertionError("BuildHooksEnv called for NoHooksLUs")
420

    
421
  def BuildHooksNodes(self):
422
    """Empty BuildHooksNodes for NoHooksLU.
423

424
    """
425
    raise AssertionError("BuildHooksNodes called for NoHooksLU")
426

    
427

    
428
class Tasklet:
429
  """Tasklet base class.
430

431
  Tasklets are subcomponents for LUs. LUs can consist entirely of tasklets or
432
  they can mix legacy code with tasklets. Locking needs to be done in the LU,
433
  tasklets know nothing about locks.
434

435
  Subclasses must follow these rules:
436
    - Implement CheckPrereq
437
    - Implement Exec
438

439
  """
440
  def __init__(self, lu):
441
    self.lu = lu
442

    
443
    # Shortcuts
444
    self.cfg = lu.cfg
445
    self.rpc = lu.rpc
446

    
447
  def CheckPrereq(self):
448
    """Check prerequisites for this tasklets.
449

450
    This method should check whether the prerequisites for the execution of
451
    this tasklet are fulfilled. It can do internode communication, but it
452
    should be idempotent - no cluster or system changes are allowed.
453

454
    The method should raise errors.OpPrereqError in case something is not
455
    fulfilled. Its return value is ignored.
456

457
    This method should also update all parameters to their canonical form if it
458
    hasn't been done before.
459

460
    """
461
    pass
462

    
463
  def Exec(self, feedback_fn):
464
    """Execute the tasklet.
465

466
    This method should implement the actual work. It should raise
467
    errors.OpExecError for failures that are somewhat dealt with in code, or
468
    expected.
469

470
    """
471
    raise NotImplementedError
472

    
473

    
474
class _QueryBase:
475
  """Base for query utility classes.
476

477
  """
478
  #: Attribute holding field definitions
479
  FIELDS = None
480

    
481
  def __init__(self, filter_, fields, use_locking):
482
    """Initializes this class.
483

484
    """
485
    self.use_locking = use_locking
486

    
487
    self.query = query.Query(self.FIELDS, fields, filter_=filter_,
488
                             namefield="name")
489
    self.requested_data = self.query.RequestedData()
490
    self.names = self.query.RequestedNames()
491

    
492
    # Sort only if no names were requested
493
    self.sort_by_name = not self.names
494

    
495
    self.do_locking = None
496
    self.wanted = None
497

    
498
  def _GetNames(self, lu, all_names, lock_level):
499
    """Helper function to determine names asked for in the query.
500

501
    """
502
    if self.do_locking:
503
      names = lu.glm.list_owned(lock_level)
504
    else:
505
      names = all_names
506

    
507
    if self.wanted == locking.ALL_SET:
508
      assert not self.names
509
      # caller didn't specify names, so ordering is not important
510
      return utils.NiceSort(names)
511

    
512
    # caller specified names and we must keep the same order
513
    assert self.names
514
    assert not self.do_locking or lu.glm.is_owned(lock_level)
515

    
516
    missing = set(self.wanted).difference(names)
517
    if missing:
518
      raise errors.OpExecError("Some items were removed before retrieving"
519
                               " their data: %s" % missing)
520

    
521
    # Return expanded names
522
    return self.wanted
523

    
524
  def ExpandNames(self, lu):
525
    """Expand names for this query.
526

527
    See L{LogicalUnit.ExpandNames}.
528

529
    """
530
    raise NotImplementedError()
531

    
532
  def DeclareLocks(self, lu, level):
533
    """Declare locks for this query.
534

535
    See L{LogicalUnit.DeclareLocks}.
536

537
    """
538
    raise NotImplementedError()
539

    
540
  def _GetQueryData(self, lu):
541
    """Collects all data for this query.
542

543
    @return: Query data object
544

545
    """
546
    raise NotImplementedError()
547

    
548
  def NewStyleQuery(self, lu):
549
    """Collect data and execute query.
550

551
    """
552
    return query.GetQueryResponse(self.query, self._GetQueryData(lu),
553
                                  sort_by_name=self.sort_by_name)
554

    
555
  def OldStyleQuery(self, lu):
556
    """Collect data and execute query.
557

558
    """
559
    return self.query.OldStyleQuery(self._GetQueryData(lu),
560
                                    sort_by_name=self.sort_by_name)
561

    
562

    
563
def _GetWantedNodes(lu, nodes):
564
  """Returns list of checked and expanded node names.
565

566
  @type lu: L{LogicalUnit}
567
  @param lu: the logical unit on whose behalf we execute
568
  @type nodes: list
569
  @param nodes: list of node names or None for all nodes
570
  @rtype: list
571
  @return: the list of nodes, sorted
572
  @raise errors.ProgrammerError: if the nodes parameter is wrong type
573

574
  """
575
  if nodes:
576
    return [_ExpandNodeName(lu.cfg, name) for name in nodes]
577

    
578
  return utils.NiceSort(lu.cfg.GetNodeList())
579

    
580

    
581
def _GetWantedInstances(lu, instances):
582
  """Returns list of checked and expanded instance names.
583

584
  @type lu: L{LogicalUnit}
585
  @param lu: the logical unit on whose behalf we execute
586
  @type instances: list
587
  @param instances: list of instance names or None for all instances
588
  @rtype: list
589
  @return: the list of instances, sorted
590
  @raise errors.OpPrereqError: if the instances parameter is wrong type
591
  @raise errors.OpPrereqError: if any of the passed instances is not found
592

593
  """
594
  if instances:
595
    wanted = [_ExpandInstanceName(lu.cfg, name) for name in instances]
596
  else:
597
    wanted = utils.NiceSort(lu.cfg.GetInstanceList())
598
  return wanted
599

    
600

    
601
def _GetUpdatedParams(old_params, update_dict,
602
                      use_default=True, use_none=False):
603
  """Return the new version of a parameter dictionary.
604

605
  @type old_params: dict
606
  @param old_params: old parameters
607
  @type update_dict: dict
608
  @param update_dict: dict containing new parameter values, or
609
      constants.VALUE_DEFAULT to reset the parameter to its default
610
      value
611
  @param use_default: boolean
612
  @type use_default: whether to recognise L{constants.VALUE_DEFAULT}
613
      values as 'to be deleted' values
614
  @param use_none: boolean
615
  @type use_none: whether to recognise C{None} values as 'to be
616
      deleted' values
617
  @rtype: dict
618
  @return: the new parameter dictionary
619

620
  """
621
  params_copy = copy.deepcopy(old_params)
622
  for key, val in update_dict.iteritems():
623
    if ((use_default and val == constants.VALUE_DEFAULT) or
624
        (use_none and val is None)):
625
      try:
626
        del params_copy[key]
627
      except KeyError:
628
        pass
629
    else:
630
      params_copy[key] = val
631
  return params_copy
632

    
633

    
634
def _ReleaseLocks(lu, level, names=None, keep=None):
635
  """Releases locks owned by an LU.
636

637
  @type lu: L{LogicalUnit}
638
  @param level: Lock level
639
  @type names: list or None
640
  @param names: Names of locks to release
641
  @type keep: list or None
642
  @param keep: Names of locks to retain
643

644
  """
645
  assert not (keep is not None and names is not None), \
646
         "Only one of the 'names' and the 'keep' parameters can be given"
647

    
648
  if names is not None:
649
    should_release = names.__contains__
650
  elif keep:
651
    should_release = lambda name: name not in keep
652
  else:
653
    should_release = None
654

    
655
  if should_release:
656
    retain = []
657
    release = []
658

    
659
    # Determine which locks to release
660
    for name in lu.glm.list_owned(level):
661
      if should_release(name):
662
        release.append(name)
663
      else:
664
        retain.append(name)
665

    
666
    assert len(lu.glm.list_owned(level)) == (len(retain) + len(release))
667

    
668
    # Release just some locks
669
    lu.glm.release(level, names=release)
670

    
671
    assert frozenset(lu.glm.list_owned(level)) == frozenset(retain)
672
  else:
673
    # Release everything
674
    lu.glm.release(level)
675

    
676
    assert not lu.glm.is_owned(level), "No locks should be owned"
677

    
678

    
679
def _RunPostHook(lu, node_name):
680
  """Runs the post-hook for an opcode on a single node.
681

682
  """
683
  hm = lu.proc.hmclass(lu.rpc.call_hooks_runner, lu)
684
  try:
685
    hm.RunPhase(constants.HOOKS_PHASE_POST, nodes=[node_name])
686
  except:
687
    # pylint: disable-msg=W0702
688
    lu.LogWarning("Errors occurred running hooks on %s" % node_name)
689

    
690

    
691
def _CheckOutputFields(static, dynamic, selected):
692
  """Checks whether all selected fields are valid.
693

694
  @type static: L{utils.FieldSet}
695
  @param static: static fields set
696
  @type dynamic: L{utils.FieldSet}
697
  @param dynamic: dynamic fields set
698

699
  """
700
  f = utils.FieldSet()
701
  f.Extend(static)
702
  f.Extend(dynamic)
703

    
704
  delta = f.NonMatching(selected)
705
  if delta:
706
    raise errors.OpPrereqError("Unknown output fields selected: %s"
707
                               % ",".join(delta), errors.ECODE_INVAL)
708

    
709

    
710
def _CheckGlobalHvParams(params):
711
  """Validates that given hypervisor params are not global ones.
712

713
  This will ensure that instances don't get customised versions of
714
  global params.
715

716
  """
717
  used_globals = constants.HVC_GLOBALS.intersection(params)
718
  if used_globals:
719
    msg = ("The following hypervisor parameters are global and cannot"
720
           " be customized at instance level, please modify them at"
721
           " cluster level: %s" % utils.CommaJoin(used_globals))
722
    raise errors.OpPrereqError(msg, errors.ECODE_INVAL)
723

    
724

    
725
def _CheckNodeOnline(lu, node, msg=None):
726
  """Ensure that a given node is online.
727

728
  @param lu: the LU on behalf of which we make the check
729
  @param node: the node to check
730
  @param msg: if passed, should be a message to replace the default one
731
  @raise errors.OpPrereqError: if the node is offline
732

733
  """
734
  if msg is None:
735
    msg = "Can't use offline node"
736
  if lu.cfg.GetNodeInfo(node).offline:
737
    raise errors.OpPrereqError("%s: %s" % (msg, node), errors.ECODE_STATE)
738

    
739

    
740
def _CheckNodeNotDrained(lu, node):
741
  """Ensure that a given node is not drained.
742

743
  @param lu: the LU on behalf of which we make the check
744
  @param node: the node to check
745
  @raise errors.OpPrereqError: if the node is drained
746

747
  """
748
  if lu.cfg.GetNodeInfo(node).drained:
749
    raise errors.OpPrereqError("Can't use drained node %s" % node,
750
                               errors.ECODE_STATE)
751

    
752

    
753
def _CheckNodeVmCapable(lu, node):
754
  """Ensure that a given node is vm capable.
755

756
  @param lu: the LU on behalf of which we make the check
757
  @param node: the node to check
758
  @raise errors.OpPrereqError: if the node is not vm capable
759

760
  """
761
  if not lu.cfg.GetNodeInfo(node).vm_capable:
762
    raise errors.OpPrereqError("Can't use non-vm_capable node %s" % node,
763
                               errors.ECODE_STATE)
764

    
765

    
766
def _CheckNodeHasOS(lu, node, os_name, force_variant):
767
  """Ensure that a node supports a given OS.
768

769
  @param lu: the LU on behalf of which we make the check
770
  @param node: the node to check
771
  @param os_name: the OS to query about
772
  @param force_variant: whether to ignore variant errors
773
  @raise errors.OpPrereqError: if the node is not supporting the OS
774

775
  """
776
  result = lu.rpc.call_os_get(node, os_name)
777
  result.Raise("OS '%s' not in supported OS list for node %s" %
778
               (os_name, node),
779
               prereq=True, ecode=errors.ECODE_INVAL)
780
  if not force_variant:
781
    _CheckOSVariant(result.payload, os_name)
782

    
783

    
784
def _CheckNodeHasSecondaryIP(lu, node, secondary_ip, prereq):
785
  """Ensure that a node has the given secondary ip.
786

787
  @type lu: L{LogicalUnit}
788
  @param lu: the LU on behalf of which we make the check
789
  @type node: string
790
  @param node: the node to check
791
  @type secondary_ip: string
792
  @param secondary_ip: the ip to check
793
  @type prereq: boolean
794
  @param prereq: whether to throw a prerequisite or an execute error
795
  @raise errors.OpPrereqError: if the node doesn't have the ip, and prereq=True
796
  @raise errors.OpExecError: if the node doesn't have the ip, and prereq=False
797

798
  """
799
  result = lu.rpc.call_node_has_ip_address(node, secondary_ip)
800
  result.Raise("Failure checking secondary ip on node %s" % node,
801
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
802
  if not result.payload:
803
    msg = ("Node claims it doesn't have the secondary ip you gave (%s),"
804
           " please fix and re-run this command" % secondary_ip)
805
    if prereq:
806
      raise errors.OpPrereqError(msg, errors.ECODE_ENVIRON)
807
    else:
808
      raise errors.OpExecError(msg)
809

    
810

    
811
def _GetClusterDomainSecret():
812
  """Reads the cluster domain secret.
813

814
  """
815
  return utils.ReadOneLineFile(constants.CLUSTER_DOMAIN_SECRET_FILE,
816
                               strict=True)
817

    
818

    
819
def _CheckInstanceDown(lu, instance, reason):
820
  """Ensure that an instance is not running."""
821
  if instance.admin_up:
822
    raise errors.OpPrereqError("Instance %s is marked to be up, %s" %
823
                               (instance.name, reason), errors.ECODE_STATE)
824

    
825
  pnode = instance.primary_node
826
  ins_l = lu.rpc.call_instance_list([pnode], [instance.hypervisor])[pnode]
827
  ins_l.Raise("Can't contact node %s for instance information" % pnode,
828
              prereq=True, ecode=errors.ECODE_ENVIRON)
829

    
830
  if instance.name in ins_l.payload:
831
    raise errors.OpPrereqError("Instance %s is running, %s" %
832
                               (instance.name, reason), errors.ECODE_STATE)
833

    
834

    
835
def _ExpandItemName(fn, name, kind):
836
  """Expand an item name.
837

838
  @param fn: the function to use for expansion
839
  @param name: requested item name
840
  @param kind: text description ('Node' or 'Instance')
841
  @return: the resolved (full) name
842
  @raise errors.OpPrereqError: if the item is not found
843

844
  """
845
  full_name = fn(name)
846
  if full_name is None:
847
    raise errors.OpPrereqError("%s '%s' not known" % (kind, name),
848
                               errors.ECODE_NOENT)
849
  return full_name
850

    
851

    
852
def _ExpandNodeName(cfg, name):
853
  """Wrapper over L{_ExpandItemName} for nodes."""
854
  return _ExpandItemName(cfg.ExpandNodeName, name, "Node")
855

    
856

    
857
def _ExpandInstanceName(cfg, name):
858
  """Wrapper over L{_ExpandItemName} for instance."""
859
  return _ExpandItemName(cfg.ExpandInstanceName, name, "Instance")
860

    
861

    
862
def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status,
863
                          memory, vcpus, nics, disk_template, disks,
864
                          bep, hvp, hypervisor_name):
865
  """Builds instance related env variables for hooks
866

867
  This builds the hook environment from individual variables.
868

869
  @type name: string
870
  @param name: the name of the instance
871
  @type primary_node: string
872
  @param primary_node: the name of the instance's primary node
873
  @type secondary_nodes: list
874
  @param secondary_nodes: list of secondary nodes as strings
875
  @type os_type: string
876
  @param os_type: the name of the instance's OS
877
  @type status: boolean
878
  @param status: the should_run status of the instance
879
  @type memory: string
880
  @param memory: the memory size of the instance
881
  @type vcpus: string
882
  @param vcpus: the count of VCPUs the instance has
883
  @type nics: list
884
  @param nics: list of tuples (ip, mac, mode, link) representing
885
      the NICs the instance has
886
  @type disk_template: string
887
  @param disk_template: the disk template of the instance
888
  @type disks: list
889
  @param disks: the list of (size, mode) pairs
890
  @type bep: dict
891
  @param bep: the backend parameters for the instance
892
  @type hvp: dict
893
  @param hvp: the hypervisor parameters for the instance
894
  @type hypervisor_name: string
895
  @param hypervisor_name: the hypervisor for the instance
896
  @rtype: dict
897
  @return: the hook environment for this instance
898

899
  """
900
  if status:
901
    str_status = "up"
902
  else:
903
    str_status = "down"
904
  env = {
905
    "OP_TARGET": name,
906
    "INSTANCE_NAME": name,
907
    "INSTANCE_PRIMARY": primary_node,
908
    "INSTANCE_SECONDARIES": " ".join(secondary_nodes),
909
    "INSTANCE_OS_TYPE": os_type,
910
    "INSTANCE_STATUS": str_status,
911
    "INSTANCE_MEMORY": memory,
912
    "INSTANCE_VCPUS": vcpus,
913
    "INSTANCE_DISK_TEMPLATE": disk_template,
914
    "INSTANCE_HYPERVISOR": hypervisor_name,
915
  }
916

    
917
  if nics:
918
    nic_count = len(nics)
919
    for idx, (ip, mac, mode, link) in enumerate(nics):
920
      if ip is None:
921
        ip = ""
922
      env["INSTANCE_NIC%d_IP" % idx] = ip
923
      env["INSTANCE_NIC%d_MAC" % idx] = mac
924
      env["INSTANCE_NIC%d_MODE" % idx] = mode
925
      env["INSTANCE_NIC%d_LINK" % idx] = link
926
      if mode == constants.NIC_MODE_BRIDGED:
927
        env["INSTANCE_NIC%d_BRIDGE" % idx] = link
928
  else:
929
    nic_count = 0
930

    
931
  env["INSTANCE_NIC_COUNT"] = nic_count
932

    
933
  if disks:
934
    disk_count = len(disks)
935
    for idx, (size, mode) in enumerate(disks):
936
      env["INSTANCE_DISK%d_SIZE" % idx] = size
937
      env["INSTANCE_DISK%d_MODE" % idx] = mode
938
  else:
939
    disk_count = 0
940

    
941
  env["INSTANCE_DISK_COUNT"] = disk_count
942

    
943
  for source, kind in [(bep, "BE"), (hvp, "HV")]:
944
    for key, value in source.items():
945
      env["INSTANCE_%s_%s" % (kind, key)] = value
946

    
947
  return env
948

    
949

    
950
def _NICListToTuple(lu, nics):
951
  """Build a list of nic information tuples.
952

953
  This list is suitable to be passed to _BuildInstanceHookEnv or as a return
954
  value in LUInstanceQueryData.
955

956
  @type lu:  L{LogicalUnit}
957
  @param lu: the logical unit on whose behalf we execute
958
  @type nics: list of L{objects.NIC}
959
  @param nics: list of nics to convert to hooks tuples
960

961
  """
962
  hooks_nics = []
963
  cluster = lu.cfg.GetClusterInfo()
964
  for nic in nics:
965
    ip = nic.ip
966
    mac = nic.mac
967
    filled_params = cluster.SimpleFillNIC(nic.nicparams)
968
    mode = filled_params[constants.NIC_MODE]
969
    link = filled_params[constants.NIC_LINK]
970
    hooks_nics.append((ip, mac, mode, link))
971
  return hooks_nics
972

    
973

    
974
def _BuildInstanceHookEnvByObject(lu, instance, override=None):
975
  """Builds instance related env variables for hooks from an object.
976

977
  @type lu: L{LogicalUnit}
978
  @param lu: the logical unit on whose behalf we execute
979
  @type instance: L{objects.Instance}
980
  @param instance: the instance for which we should build the
981
      environment
982
  @type override: dict
983
  @param override: dictionary with key/values that will override
984
      our values
985
  @rtype: dict
986
  @return: the hook environment dictionary
987

988
  """
989
  cluster = lu.cfg.GetClusterInfo()
990
  bep = cluster.FillBE(instance)
991
  hvp = cluster.FillHV(instance)
992
  args = {
993
    'name': instance.name,
994
    'primary_node': instance.primary_node,
995
    'secondary_nodes': instance.secondary_nodes,
996
    'os_type': instance.os,
997
    'status': instance.admin_up,
998
    'memory': bep[constants.BE_MEMORY],
999
    'vcpus': bep[constants.BE_VCPUS],
1000
    'nics': _NICListToTuple(lu, instance.nics),
1001
    'disk_template': instance.disk_template,
1002
    'disks': [(disk.size, disk.mode) for disk in instance.disks],
1003
    'bep': bep,
1004
    'hvp': hvp,
1005
    'hypervisor_name': instance.hypervisor,
1006
  }
1007
  if override:
1008
    args.update(override)
1009
  return _BuildInstanceHookEnv(**args) # pylint: disable-msg=W0142
1010

    
1011

    
1012
def _AdjustCandidatePool(lu, exceptions):
1013
  """Adjust the candidate pool after node operations.
1014

1015
  """
1016
  mod_list = lu.cfg.MaintainCandidatePool(exceptions)
1017
  if mod_list:
1018
    lu.LogInfo("Promoted nodes to master candidate role: %s",
1019
               utils.CommaJoin(node.name for node in mod_list))
1020
    for name in mod_list:
1021
      lu.context.ReaddNode(name)
1022
  mc_now, mc_max, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1023
  if mc_now > mc_max:
1024
    lu.LogInfo("Note: more nodes are candidates (%d) than desired (%d)" %
1025
               (mc_now, mc_max))
1026

    
1027

    
1028
def _DecideSelfPromotion(lu, exceptions=None):
1029
  """Decide whether I should promote myself as a master candidate.
1030

1031
  """
1032
  cp_size = lu.cfg.GetClusterInfo().candidate_pool_size
1033
  mc_now, mc_should, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1034
  # the new node will increase mc_max with one, so:
1035
  mc_should = min(mc_should + 1, cp_size)
1036
  return mc_now < mc_should
1037

    
1038

    
1039
def _CheckNicsBridgesExist(lu, target_nics, target_node):
1040
  """Check that the brigdes needed by a list of nics exist.
1041

1042
  """
1043
  cluster = lu.cfg.GetClusterInfo()
1044
  paramslist = [cluster.SimpleFillNIC(nic.nicparams) for nic in target_nics]
1045
  brlist = [params[constants.NIC_LINK] for params in paramslist
1046
            if params[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED]
1047
  if brlist:
1048
    result = lu.rpc.call_bridges_exist(target_node, brlist)
1049
    result.Raise("Error checking bridges on destination node '%s'" %
1050
                 target_node, prereq=True, ecode=errors.ECODE_ENVIRON)
1051

    
1052

    
1053
def _CheckInstanceBridgesExist(lu, instance, node=None):
1054
  """Check that the brigdes needed by an instance exist.
1055

1056
  """
1057
  if node is None:
1058
    node = instance.primary_node
1059
  _CheckNicsBridgesExist(lu, instance.nics, node)
1060

    
1061

    
1062
def _CheckOSVariant(os_obj, name):
1063
  """Check whether an OS name conforms to the os variants specification.
1064

1065
  @type os_obj: L{objects.OS}
1066
  @param os_obj: OS object to check
1067
  @type name: string
1068
  @param name: OS name passed by the user, to check for validity
1069

1070
  """
1071
  if not os_obj.supported_variants:
1072
    return
1073
  variant = objects.OS.GetVariant(name)
1074
  if not variant:
1075
    raise errors.OpPrereqError("OS name must include a variant",
1076
                               errors.ECODE_INVAL)
1077

    
1078
  if variant not in os_obj.supported_variants:
1079
    raise errors.OpPrereqError("Unsupported OS variant", errors.ECODE_INVAL)
1080

    
1081

    
1082
def _GetNodeInstancesInner(cfg, fn):
1083
  return [i for i in cfg.GetAllInstancesInfo().values() if fn(i)]
1084

    
1085

    
1086
def _GetNodeInstances(cfg, node_name):
1087
  """Returns a list of all primary and secondary instances on a node.
1088

1089
  """
1090

    
1091
  return _GetNodeInstancesInner(cfg, lambda inst: node_name in inst.all_nodes)
1092

    
1093

    
1094
def _GetNodePrimaryInstances(cfg, node_name):
1095
  """Returns primary instances on a node.
1096

1097
  """
1098
  return _GetNodeInstancesInner(cfg,
1099
                                lambda inst: node_name == inst.primary_node)
1100

    
1101

    
1102
def _GetNodeSecondaryInstances(cfg, node_name):
1103
  """Returns secondary instances on a node.
1104

1105
  """
1106
  return _GetNodeInstancesInner(cfg,
1107
                                lambda inst: node_name in inst.secondary_nodes)
1108

    
1109

    
1110
def _GetStorageTypeArgs(cfg, storage_type):
1111
  """Returns the arguments for a storage type.
1112

1113
  """
1114
  # Special case for file storage
1115
  if storage_type == constants.ST_FILE:
1116
    # storage.FileStorage wants a list of storage directories
1117
    return [[cfg.GetFileStorageDir(), cfg.GetSharedFileStorageDir()]]
1118

    
1119
  return []
1120

    
1121

    
1122
def _FindFaultyInstanceDisks(cfg, rpc, instance, node_name, prereq):
1123
  faulty = []
1124

    
1125
  for dev in instance.disks:
1126
    cfg.SetDiskID(dev, node_name)
1127

    
1128
  result = rpc.call_blockdev_getmirrorstatus(node_name, instance.disks)
1129
  result.Raise("Failed to get disk status from node %s" % node_name,
1130
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
1131

    
1132
  for idx, bdev_status in enumerate(result.payload):
1133
    if bdev_status and bdev_status.ldisk_status == constants.LDS_FAULTY:
1134
      faulty.append(idx)
1135

    
1136
  return faulty
1137

    
1138

    
1139
def _CheckIAllocatorOrNode(lu, iallocator_slot, node_slot):
1140
  """Check the sanity of iallocator and node arguments and use the
1141
  cluster-wide iallocator if appropriate.
1142

1143
  Check that at most one of (iallocator, node) is specified. If none is
1144
  specified, then the LU's opcode's iallocator slot is filled with the
1145
  cluster-wide default iallocator.
1146

1147
  @type iallocator_slot: string
1148
  @param iallocator_slot: the name of the opcode iallocator slot
1149
  @type node_slot: string
1150
  @param node_slot: the name of the opcode target node slot
1151

1152
  """
1153
  node = getattr(lu.op, node_slot, None)
1154
  iallocator = getattr(lu.op, iallocator_slot, None)
1155

    
1156
  if node is not None and iallocator is not None:
1157
    raise errors.OpPrereqError("Do not specify both, iallocator and node",
1158
                               errors.ECODE_INVAL)
1159
  elif node is None and iallocator is None:
1160
    default_iallocator = lu.cfg.GetDefaultIAllocator()
1161
    if default_iallocator:
1162
      setattr(lu.op, iallocator_slot, default_iallocator)
1163
    else:
1164
      raise errors.OpPrereqError("No iallocator or node given and no"
1165
                                 " cluster-wide default iallocator found;"
1166
                                 " please specify either an iallocator or a"
1167
                                 " node, or set a cluster-wide default"
1168
                                 " iallocator")
1169

    
1170

    
1171
class LUClusterPostInit(LogicalUnit):
1172
  """Logical unit for running hooks after cluster initialization.
1173

1174
  """
1175
  HPATH = "cluster-init"
1176
  HTYPE = constants.HTYPE_CLUSTER
1177

    
1178
  def BuildHooksEnv(self):
1179
    """Build hooks env.
1180

1181
    """
1182
    return {
1183
      "OP_TARGET": self.cfg.GetClusterName(),
1184
      }
1185

    
1186
  def BuildHooksNodes(self):
1187
    """Build hooks nodes.
1188

1189
    """
1190
    return ([], [self.cfg.GetMasterNode()])
1191

    
1192
  def Exec(self, feedback_fn):
1193
    """Nothing to do.
1194

1195
    """
1196
    return True
1197

    
1198

    
1199
class LUClusterDestroy(LogicalUnit):
1200
  """Logical unit for destroying the cluster.
1201

1202
  """
1203
  HPATH = "cluster-destroy"
1204
  HTYPE = constants.HTYPE_CLUSTER
1205

    
1206
  def BuildHooksEnv(self):
1207
    """Build hooks env.
1208

1209
    """
1210
    return {
1211
      "OP_TARGET": self.cfg.GetClusterName(),
1212
      }
1213

    
1214
  def BuildHooksNodes(self):
1215
    """Build hooks nodes.
1216

1217
    """
1218
    return ([], [])
1219

    
1220
  def CheckPrereq(self):
1221
    """Check prerequisites.
1222

1223
    This checks whether the cluster is empty.
1224

1225
    Any errors are signaled by raising errors.OpPrereqError.
1226

1227
    """
1228
    master = self.cfg.GetMasterNode()
1229

    
1230
    nodelist = self.cfg.GetNodeList()
1231
    if len(nodelist) != 1 or nodelist[0] != master:
1232
      raise errors.OpPrereqError("There are still %d node(s) in"
1233
                                 " this cluster." % (len(nodelist) - 1),
1234
                                 errors.ECODE_INVAL)
1235
    instancelist = self.cfg.GetInstanceList()
1236
    if instancelist:
1237
      raise errors.OpPrereqError("There are still %d instance(s) in"
1238
                                 " this cluster." % len(instancelist),
1239
                                 errors.ECODE_INVAL)
1240

    
1241
  def Exec(self, feedback_fn):
1242
    """Destroys the cluster.
1243

1244
    """
1245
    master = self.cfg.GetMasterNode()
1246

    
1247
    # Run post hooks on master node before it's removed
1248
    _RunPostHook(self, master)
1249

    
1250
    result = self.rpc.call_node_stop_master(master, False)
1251
    result.Raise("Could not disable the master role")
1252

    
1253
    return master
1254

    
1255

    
1256
def _VerifyCertificate(filename):
1257
  """Verifies a certificate for LUClusterVerify.
1258

1259
  @type filename: string
1260
  @param filename: Path to PEM file
1261

1262
  """
1263
  try:
1264
    cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
1265
                                           utils.ReadFile(filename))
1266
  except Exception, err: # pylint: disable-msg=W0703
1267
    return (LUClusterVerify.ETYPE_ERROR,
1268
            "Failed to load X509 certificate %s: %s" % (filename, err))
1269

    
1270
  (errcode, msg) = \
1271
    utils.VerifyX509Certificate(cert, constants.SSL_CERT_EXPIRATION_WARN,
1272
                                constants.SSL_CERT_EXPIRATION_ERROR)
1273

    
1274
  if msg:
1275
    fnamemsg = "While verifying %s: %s" % (filename, msg)
1276
  else:
1277
    fnamemsg = None
1278

    
1279
  if errcode is None:
1280
    return (None, fnamemsg)
1281
  elif errcode == utils.CERT_WARNING:
1282
    return (LUClusterVerify.ETYPE_WARNING, fnamemsg)
1283
  elif errcode == utils.CERT_ERROR:
1284
    return (LUClusterVerify.ETYPE_ERROR, fnamemsg)
1285

    
1286
  raise errors.ProgrammerError("Unhandled certificate error code %r" % errcode)
1287

    
1288

    
1289
class LUClusterVerify(LogicalUnit):
1290
  """Verifies the cluster status.
1291

1292
  """
1293
  HPATH = "cluster-verify"
1294
  HTYPE = constants.HTYPE_CLUSTER
1295
  REQ_BGL = False
1296

    
1297
  TCLUSTER = "cluster"
1298
  TNODE = "node"
1299
  TINSTANCE = "instance"
1300

    
1301
  ECLUSTERCFG = (TCLUSTER, "ECLUSTERCFG")
1302
  ECLUSTERCERT = (TCLUSTER, "ECLUSTERCERT")
1303
  ECLUSTERFILECHECK = (TCLUSTER, "ECLUSTERFILECHECK")
1304
  EINSTANCEBADNODE = (TINSTANCE, "EINSTANCEBADNODE")
1305
  EINSTANCEDOWN = (TINSTANCE, "EINSTANCEDOWN")
1306
  EINSTANCELAYOUT = (TINSTANCE, "EINSTANCELAYOUT")
1307
  EINSTANCEMISSINGDISK = (TINSTANCE, "EINSTANCEMISSINGDISK")
1308
  EINSTANCEFAULTYDISK = (TINSTANCE, "EINSTANCEFAULTYDISK")
1309
  EINSTANCEWRONGNODE = (TINSTANCE, "EINSTANCEWRONGNODE")
1310
  EINSTANCESPLITGROUPS = (TINSTANCE, "EINSTANCESPLITGROUPS")
1311
  ENODEDRBD = (TNODE, "ENODEDRBD")
1312
  ENODEDRBDHELPER = (TNODE, "ENODEDRBDHELPER")
1313
  ENODEFILECHECK = (TNODE, "ENODEFILECHECK")
1314
  ENODEHOOKS = (TNODE, "ENODEHOOKS")
1315
  ENODEHV = (TNODE, "ENODEHV")
1316
  ENODELVM = (TNODE, "ENODELVM")
1317
  ENODEN1 = (TNODE, "ENODEN1")
1318
  ENODENET = (TNODE, "ENODENET")
1319
  ENODEOS = (TNODE, "ENODEOS")
1320
  ENODEORPHANINSTANCE = (TNODE, "ENODEORPHANINSTANCE")
1321
  ENODEORPHANLV = (TNODE, "ENODEORPHANLV")
1322
  ENODERPC = (TNODE, "ENODERPC")
1323
  ENODESSH = (TNODE, "ENODESSH")
1324
  ENODEVERSION = (TNODE, "ENODEVERSION")
1325
  ENODESETUP = (TNODE, "ENODESETUP")
1326
  ENODETIME = (TNODE, "ENODETIME")
1327
  ENODEOOBPATH = (TNODE, "ENODEOOBPATH")
1328

    
1329
  ETYPE_FIELD = "code"
1330
  ETYPE_ERROR = "ERROR"
1331
  ETYPE_WARNING = "WARNING"
1332

    
1333
  _HOOKS_INDENT_RE = re.compile("^", re.M)
1334

    
1335
  class NodeImage(object):
1336
    """A class representing the logical and physical status of a node.
1337

1338
    @type name: string
1339
    @ivar name: the node name to which this object refers
1340
    @ivar volumes: a structure as returned from
1341
        L{ganeti.backend.GetVolumeList} (runtime)
1342
    @ivar instances: a list of running instances (runtime)
1343
    @ivar pinst: list of configured primary instances (config)
1344
    @ivar sinst: list of configured secondary instances (config)
1345
    @ivar sbp: dictionary of {primary-node: list of instances} for all
1346
        instances for which this node is secondary (config)
1347
    @ivar mfree: free memory, as reported by hypervisor (runtime)
1348
    @ivar dfree: free disk, as reported by the node (runtime)
1349
    @ivar offline: the offline status (config)
1350
    @type rpc_fail: boolean
1351
    @ivar rpc_fail: whether the RPC verify call was successfull (overall,
1352
        not whether the individual keys were correct) (runtime)
1353
    @type lvm_fail: boolean
1354
    @ivar lvm_fail: whether the RPC call didn't return valid LVM data
1355
    @type hyp_fail: boolean
1356
    @ivar hyp_fail: whether the RPC call didn't return the instance list
1357
    @type ghost: boolean
1358
    @ivar ghost: whether this is a known node or not (config)
1359
    @type os_fail: boolean
1360
    @ivar os_fail: whether the RPC call didn't return valid OS data
1361
    @type oslist: list
1362
    @ivar oslist: list of OSes as diagnosed by DiagnoseOS
1363
    @type vm_capable: boolean
1364
    @ivar vm_capable: whether the node can host instances
1365

1366
    """
1367
    def __init__(self, offline=False, name=None, vm_capable=True):
1368
      self.name = name
1369
      self.volumes = {}
1370
      self.instances = []
1371
      self.pinst = []
1372
      self.sinst = []
1373
      self.sbp = {}
1374
      self.mfree = 0
1375
      self.dfree = 0
1376
      self.offline = offline
1377
      self.vm_capable = vm_capable
1378
      self.rpc_fail = False
1379
      self.lvm_fail = False
1380
      self.hyp_fail = False
1381
      self.ghost = False
1382
      self.os_fail = False
1383
      self.oslist = {}
1384

    
1385
  def ExpandNames(self):
1386
    self.needed_locks = {
1387
      locking.LEVEL_NODE: locking.ALL_SET,
1388
      locking.LEVEL_INSTANCE: locking.ALL_SET,
1389
    }
1390
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
1391

    
1392
  def CheckPrereq(self):
1393
    self.all_node_info = self.cfg.GetAllNodesInfo()
1394
    self.all_inst_info = self.cfg.GetAllInstancesInfo()
1395
    self.my_node_names = utils.NiceSort(list(self.all_node_info))
1396
    self.my_node_info = self.all_node_info
1397
    self.my_inst_names = utils.NiceSort(list(self.all_inst_info))
1398
    self.my_inst_info = self.all_inst_info
1399

    
1400
  def _Error(self, ecode, item, msg, *args, **kwargs):
1401
    """Format an error message.
1402

1403
    Based on the opcode's error_codes parameter, either format a
1404
    parseable error code, or a simpler error string.
1405

1406
    This must be called only from Exec and functions called from Exec.
1407

1408
    """
1409
    ltype = kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR)
1410
    itype, etxt = ecode
1411
    # first complete the msg
1412
    if args:
1413
      msg = msg % args
1414
    # then format the whole message
1415
    if self.op.error_codes:
1416
      msg = "%s:%s:%s:%s:%s" % (ltype, etxt, itype, item, msg)
1417
    else:
1418
      if item:
1419
        item = " " + item
1420
      else:
1421
        item = ""
1422
      msg = "%s: %s%s: %s" % (ltype, itype, item, msg)
1423
    # and finally report it via the feedback_fn
1424
    self._feedback_fn("  - %s" % msg)
1425

    
1426
  def _ErrorIf(self, cond, *args, **kwargs):
1427
    """Log an error message if the passed condition is True.
1428

1429
    """
1430
    cond = bool(cond) or self.op.debug_simulate_errors
1431
    if cond:
1432
      self._Error(*args, **kwargs)
1433
    # do not mark the operation as failed for WARN cases only
1434
    if kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR) == self.ETYPE_ERROR:
1435
      self.bad = self.bad or cond
1436

    
1437
  def _VerifyNode(self, ninfo, nresult):
1438
    """Perform some basic validation on data returned from a node.
1439

1440
      - check the result data structure is well formed and has all the
1441
        mandatory fields
1442
      - check ganeti version
1443

1444
    @type ninfo: L{objects.Node}
1445
    @param ninfo: the node to check
1446
    @param nresult: the results from the node
1447
    @rtype: boolean
1448
    @return: whether overall this call was successful (and we can expect
1449
         reasonable values in the respose)
1450

1451
    """
1452
    node = ninfo.name
1453
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1454

    
1455
    # main result, nresult should be a non-empty dict
1456
    test = not nresult or not isinstance(nresult, dict)
1457
    _ErrorIf(test, self.ENODERPC, node,
1458
                  "unable to verify node: no data returned")
1459
    if test:
1460
      return False
1461

    
1462
    # compares ganeti version
1463
    local_version = constants.PROTOCOL_VERSION
1464
    remote_version = nresult.get("version", None)
1465
    test = not (remote_version and
1466
                isinstance(remote_version, (list, tuple)) and
1467
                len(remote_version) == 2)
1468
    _ErrorIf(test, self.ENODERPC, node,
1469
             "connection to node returned invalid data")
1470
    if test:
1471
      return False
1472

    
1473
    test = local_version != remote_version[0]
1474
    _ErrorIf(test, self.ENODEVERSION, node,
1475
             "incompatible protocol versions: master %s,"
1476
             " node %s", local_version, remote_version[0])
1477
    if test:
1478
      return False
1479

    
1480
    # node seems compatible, we can actually try to look into its results
1481

    
1482
    # full package version
1483
    self._ErrorIf(constants.RELEASE_VERSION != remote_version[1],
1484
                  self.ENODEVERSION, node,
1485
                  "software version mismatch: master %s, node %s",
1486
                  constants.RELEASE_VERSION, remote_version[1],
1487
                  code=self.ETYPE_WARNING)
1488

    
1489
    hyp_result = nresult.get(constants.NV_HYPERVISOR, None)
1490
    if ninfo.vm_capable and isinstance(hyp_result, dict):
1491
      for hv_name, hv_result in hyp_result.iteritems():
1492
        test = hv_result is not None
1493
        _ErrorIf(test, self.ENODEHV, node,
1494
                 "hypervisor %s verify failure: '%s'", hv_name, hv_result)
1495

    
1496
    hvp_result = nresult.get(constants.NV_HVPARAMS, None)
1497
    if ninfo.vm_capable and isinstance(hvp_result, list):
1498
      for item, hv_name, hv_result in hvp_result:
1499
        _ErrorIf(True, self.ENODEHV, node,
1500
                 "hypervisor %s parameter verify failure (source %s): %s",
1501
                 hv_name, item, hv_result)
1502

    
1503
    test = nresult.get(constants.NV_NODESETUP,
1504
                       ["Missing NODESETUP results"])
1505
    _ErrorIf(test, self.ENODESETUP, node, "node setup error: %s",
1506
             "; ".join(test))
1507

    
1508
    return True
1509

    
1510
  def _VerifyNodeTime(self, ninfo, nresult,
1511
                      nvinfo_starttime, nvinfo_endtime):
1512
    """Check the node time.
1513

1514
    @type ninfo: L{objects.Node}
1515
    @param ninfo: the node to check
1516
    @param nresult: the remote results for the node
1517
    @param nvinfo_starttime: the start time of the RPC call
1518
    @param nvinfo_endtime: the end time of the RPC call
1519

1520
    """
1521
    node = ninfo.name
1522
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1523

    
1524
    ntime = nresult.get(constants.NV_TIME, None)
1525
    try:
1526
      ntime_merged = utils.MergeTime(ntime)
1527
    except (ValueError, TypeError):
1528
      _ErrorIf(True, self.ENODETIME, node, "Node returned invalid time")
1529
      return
1530

    
1531
    if ntime_merged < (nvinfo_starttime - constants.NODE_MAX_CLOCK_SKEW):
1532
      ntime_diff = "%.01fs" % abs(nvinfo_starttime - ntime_merged)
1533
    elif ntime_merged > (nvinfo_endtime + constants.NODE_MAX_CLOCK_SKEW):
1534
      ntime_diff = "%.01fs" % abs(ntime_merged - nvinfo_endtime)
1535
    else:
1536
      ntime_diff = None
1537

    
1538
    _ErrorIf(ntime_diff is not None, self.ENODETIME, node,
1539
             "Node time diverges by at least %s from master node time",
1540
             ntime_diff)
1541

    
1542
  def _VerifyNodeLVM(self, ninfo, nresult, vg_name):
1543
    """Check the node LVM results.
1544

1545
    @type ninfo: L{objects.Node}
1546
    @param ninfo: the node to check
1547
    @param nresult: the remote results for the node
1548
    @param vg_name: the configured VG name
1549

1550
    """
1551
    if vg_name is None:
1552
      return
1553

    
1554
    node = ninfo.name
1555
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1556

    
1557
    # checks vg existence and size > 20G
1558
    vglist = nresult.get(constants.NV_VGLIST, None)
1559
    test = not vglist
1560
    _ErrorIf(test, self.ENODELVM, node, "unable to check volume groups")
1561
    if not test:
1562
      vgstatus = utils.CheckVolumeGroupSize(vglist, vg_name,
1563
                                            constants.MIN_VG_SIZE)
1564
      _ErrorIf(vgstatus, self.ENODELVM, node, vgstatus)
1565

    
1566
    # check pv names
1567
    pvlist = nresult.get(constants.NV_PVLIST, None)
1568
    test = pvlist is None
1569
    _ErrorIf(test, self.ENODELVM, node, "Can't get PV list from node")
1570
    if not test:
1571
      # check that ':' is not present in PV names, since it's a
1572
      # special character for lvcreate (denotes the range of PEs to
1573
      # use on the PV)
1574
      for _, pvname, owner_vg in pvlist:
1575
        test = ":" in pvname
1576
        _ErrorIf(test, self.ENODELVM, node, "Invalid character ':' in PV"
1577
                 " '%s' of VG '%s'", pvname, owner_vg)
1578

    
1579
  def _VerifyNodeBridges(self, ninfo, nresult, bridges):
1580
    """Check the node bridges.
1581

1582
    @type ninfo: L{objects.Node}
1583
    @param ninfo: the node to check
1584
    @param nresult: the remote results for the node
1585
    @param bridges: the expected list of bridges
1586

1587
    """
1588
    if not bridges:
1589
      return
1590

    
1591
    node = ninfo.name
1592
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1593

    
1594
    missing = nresult.get(constants.NV_BRIDGES, None)
1595
    test = not isinstance(missing, list)
1596
    _ErrorIf(test, self.ENODENET, node,
1597
             "did not return valid bridge information")
1598
    if not test:
1599
      _ErrorIf(bool(missing), self.ENODENET, node, "missing bridges: %s" %
1600
               utils.CommaJoin(sorted(missing)))
1601

    
1602
  def _VerifyNodeNetwork(self, ninfo, nresult):
1603
    """Check the node network connectivity results.
1604

1605
    @type ninfo: L{objects.Node}
1606
    @param ninfo: the node to check
1607
    @param nresult: the remote results for the node
1608

1609
    """
1610
    node = ninfo.name
1611
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1612

    
1613
    test = constants.NV_NODELIST not in nresult
1614
    _ErrorIf(test, self.ENODESSH, node,
1615
             "node hasn't returned node ssh connectivity data")
1616
    if not test:
1617
      if nresult[constants.NV_NODELIST]:
1618
        for a_node, a_msg in nresult[constants.NV_NODELIST].items():
1619
          _ErrorIf(True, self.ENODESSH, node,
1620
                   "ssh communication with node '%s': %s", a_node, a_msg)
1621

    
1622
    test = constants.NV_NODENETTEST not in nresult
1623
    _ErrorIf(test, self.ENODENET, node,
1624
             "node hasn't returned node tcp connectivity data")
1625
    if not test:
1626
      if nresult[constants.NV_NODENETTEST]:
1627
        nlist = utils.NiceSort(nresult[constants.NV_NODENETTEST].keys())
1628
        for anode in nlist:
1629
          _ErrorIf(True, self.ENODENET, node,
1630
                   "tcp communication with node '%s': %s",
1631
                   anode, nresult[constants.NV_NODENETTEST][anode])
1632

    
1633
    test = constants.NV_MASTERIP not in nresult
1634
    _ErrorIf(test, self.ENODENET, node,
1635
             "node hasn't returned node master IP reachability data")
1636
    if not test:
1637
      if not nresult[constants.NV_MASTERIP]:
1638
        if node == self.master_node:
1639
          msg = "the master node cannot reach the master IP (not configured?)"
1640
        else:
1641
          msg = "cannot reach the master IP"
1642
        _ErrorIf(True, self.ENODENET, node, msg)
1643

    
1644
  def _VerifyInstance(self, instance, instanceconfig, node_image,
1645
                      diskstatus):
1646
    """Verify an instance.
1647

1648
    This function checks to see if the required block devices are
1649
    available on the instance's node.
1650

1651
    """
1652
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1653
    node_current = instanceconfig.primary_node
1654

    
1655
    node_vol_should = {}
1656
    instanceconfig.MapLVsByNode(node_vol_should)
1657

    
1658
    for node in node_vol_should:
1659
      n_img = node_image[node]
1660
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1661
        # ignore missing volumes on offline or broken nodes
1662
        continue
1663
      for volume in node_vol_should[node]:
1664
        test = volume not in n_img.volumes
1665
        _ErrorIf(test, self.EINSTANCEMISSINGDISK, instance,
1666
                 "volume %s missing on node %s", volume, node)
1667

    
1668
    if instanceconfig.admin_up:
1669
      pri_img = node_image[node_current]
1670
      test = instance not in pri_img.instances and not pri_img.offline
1671
      _ErrorIf(test, self.EINSTANCEDOWN, instance,
1672
               "instance not running on its primary node %s",
1673
               node_current)
1674

    
1675
    diskdata = [(nname, success, status, idx)
1676
                for (nname, disks) in diskstatus.items()
1677
                for idx, (success, status) in enumerate(disks)]
1678

    
1679
    for nname, success, bdev_status, idx in diskdata:
1680
      # the 'ghost node' construction in Exec() ensures that we have a
1681
      # node here
1682
      snode = node_image[nname]
1683
      bad_snode = snode.ghost or snode.offline
1684
      _ErrorIf(instanceconfig.admin_up and not success and not bad_snode,
1685
               self.EINSTANCEFAULTYDISK, instance,
1686
               "couldn't retrieve status for disk/%s on %s: %s",
1687
               idx, nname, bdev_status)
1688
      _ErrorIf((instanceconfig.admin_up and success and
1689
                bdev_status.ldisk_status == constants.LDS_FAULTY),
1690
               self.EINSTANCEFAULTYDISK, instance,
1691
               "disk/%s on %s is faulty", idx, nname)
1692

    
1693
  def _VerifyOrphanVolumes(self, node_vol_should, node_image, reserved):
1694
    """Verify if there are any unknown volumes in the cluster.
1695

1696
    The .os, .swap and backup volumes are ignored. All other volumes are
1697
    reported as unknown.
1698

1699
    @type reserved: L{ganeti.utils.FieldSet}
1700
    @param reserved: a FieldSet of reserved volume names
1701

1702
    """
1703
    for node, n_img in node_image.items():
1704
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1705
        # skip non-healthy nodes
1706
        continue
1707
      for volume in n_img.volumes:
1708
        test = ((node not in node_vol_should or
1709
                volume not in node_vol_should[node]) and
1710
                not reserved.Matches(volume))
1711
        self._ErrorIf(test, self.ENODEORPHANLV, node,
1712
                      "volume %s is unknown", volume)
1713

    
1714
  def _VerifyNPlusOneMemory(self, node_image, instance_cfg):
1715
    """Verify N+1 Memory Resilience.
1716

1717
    Check that if one single node dies we can still start all the
1718
    instances it was primary for.
1719

1720
    """
1721
    cluster_info = self.cfg.GetClusterInfo()
1722
    for node, n_img in node_image.items():
1723
      # This code checks that every node which is now listed as
1724
      # secondary has enough memory to host all instances it is
1725
      # supposed to should a single other node in the cluster fail.
1726
      # FIXME: not ready for failover to an arbitrary node
1727
      # FIXME: does not support file-backed instances
1728
      # WARNING: we currently take into account down instances as well
1729
      # as up ones, considering that even if they're down someone
1730
      # might want to start them even in the event of a node failure.
1731
      if n_img.offline:
1732
        # we're skipping offline nodes from the N+1 warning, since
1733
        # most likely we don't have good memory infromation from them;
1734
        # we already list instances living on such nodes, and that's
1735
        # enough warning
1736
        continue
1737
      for prinode, instances in n_img.sbp.items():
1738
        needed_mem = 0
1739
        for instance in instances:
1740
          bep = cluster_info.FillBE(instance_cfg[instance])
1741
          if bep[constants.BE_AUTO_BALANCE]:
1742
            needed_mem += bep[constants.BE_MEMORY]
1743
        test = n_img.mfree < needed_mem
1744
        self._ErrorIf(test, self.ENODEN1, node,
1745
                      "not enough memory to accomodate instance failovers"
1746
                      " should node %s fail (%dMiB needed, %dMiB available)",
1747
                      prinode, needed_mem, n_img.mfree)
1748

    
1749
  @classmethod
1750
  def _VerifyFiles(cls, errorif, nodeinfo, master_node, all_nvinfo,
1751
                   (files_all, files_all_opt, files_mc, files_vm)):
1752
    """Verifies file checksums collected from all nodes.
1753

1754
    @param errorif: Callback for reporting errors
1755
    @param nodeinfo: List of L{objects.Node} objects
1756
    @param master_node: Name of master node
1757
    @param all_nvinfo: RPC results
1758

1759
    """
1760
    node_names = frozenset(node.name for node in nodeinfo)
1761

    
1762
    assert master_node in node_names
1763
    assert (len(files_all | files_all_opt | files_mc | files_vm) ==
1764
            sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
1765
           "Found file listed in more than one file list"
1766

    
1767
    # Define functions determining which nodes to consider for a file
1768
    file2nodefn = dict([(filename, fn)
1769
      for (files, fn) in [(files_all, None),
1770
                          (files_all_opt, None),
1771
                          (files_mc, lambda node: (node.master_candidate or
1772
                                                   node.name == master_node)),
1773
                          (files_vm, lambda node: node.vm_capable)]
1774
      for filename in files])
1775

    
1776
    fileinfo = dict((filename, {}) for filename in file2nodefn.keys())
1777

    
1778
    for node in nodeinfo:
1779
      nresult = all_nvinfo[node.name]
1780

    
1781
      if nresult.fail_msg or not nresult.payload:
1782
        node_files = None
1783
      else:
1784
        node_files = nresult.payload.get(constants.NV_FILELIST, None)
1785

    
1786
      test = not (node_files and isinstance(node_files, dict))
1787
      errorif(test, cls.ENODEFILECHECK, node.name,
1788
              "Node did not return file checksum data")
1789
      if test:
1790
        continue
1791

    
1792
      for (filename, checksum) in node_files.items():
1793
        # Check if the file should be considered for a node
1794
        fn = file2nodefn[filename]
1795
        if fn is None or fn(node):
1796
          fileinfo[filename].setdefault(checksum, set()).add(node.name)
1797

    
1798
    for (filename, checksums) in fileinfo.items():
1799
      assert compat.all(len(i) > 10 for i in checksums), "Invalid checksum"
1800

    
1801
      # Nodes having the file
1802
      with_file = frozenset(node_name
1803
                            for nodes in fileinfo[filename].values()
1804
                            for node_name in nodes)
1805

    
1806
      # Nodes missing file
1807
      missing_file = node_names - with_file
1808

    
1809
      if filename in files_all_opt:
1810
        # All or no nodes
1811
        errorif(missing_file and missing_file != node_names,
1812
                cls.ECLUSTERFILECHECK, None,
1813
                "File %s is optional, but it must exist on all or no nodes (not"
1814
                " found on %s)",
1815
                filename, utils.CommaJoin(utils.NiceSort(missing_file)))
1816
      else:
1817
        errorif(missing_file, cls.ECLUSTERFILECHECK, None,
1818
                "File %s is missing from node(s) %s", filename,
1819
                utils.CommaJoin(utils.NiceSort(missing_file)))
1820

    
1821
      # See if there are multiple versions of the file
1822
      test = len(checksums) > 1
1823
      if test:
1824
        variants = ["variant %s on %s" %
1825
                    (idx + 1, utils.CommaJoin(utils.NiceSort(nodes)))
1826
                    for (idx, (checksum, nodes)) in
1827
                      enumerate(sorted(checksums.items()))]
1828
      else:
1829
        variants = []
1830

    
1831
      errorif(test, cls.ECLUSTERFILECHECK, None,
1832
              "File %s found with %s different checksums (%s)",
1833
              filename, len(checksums), "; ".join(variants))
1834

    
1835
  def _VerifyNodeDrbd(self, ninfo, nresult, instanceinfo, drbd_helper,
1836
                      drbd_map):
1837
    """Verifies and the node DRBD status.
1838

1839
    @type ninfo: L{objects.Node}
1840
    @param ninfo: the node to check
1841
    @param nresult: the remote results for the node
1842
    @param instanceinfo: the dict of instances
1843
    @param drbd_helper: the configured DRBD usermode helper
1844
    @param drbd_map: the DRBD map as returned by
1845
        L{ganeti.config.ConfigWriter.ComputeDRBDMap}
1846

1847
    """
1848
    node = ninfo.name
1849
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1850

    
1851
    if drbd_helper:
1852
      helper_result = nresult.get(constants.NV_DRBDHELPER, None)
1853
      test = (helper_result == None)
1854
      _ErrorIf(test, self.ENODEDRBDHELPER, node,
1855
               "no drbd usermode helper returned")
1856
      if helper_result:
1857
        status, payload = helper_result
1858
        test = not status
1859
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
1860
                 "drbd usermode helper check unsuccessful: %s", payload)
1861
        test = status and (payload != drbd_helper)
1862
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
1863
                 "wrong drbd usermode helper: %s", payload)
1864

    
1865
    # compute the DRBD minors
1866
    node_drbd = {}
1867
    for minor, instance in drbd_map[node].items():
1868
      test = instance not in instanceinfo
1869
      _ErrorIf(test, self.ECLUSTERCFG, None,
1870
               "ghost instance '%s' in temporary DRBD map", instance)
1871
        # ghost instance should not be running, but otherwise we
1872
        # don't give double warnings (both ghost instance and
1873
        # unallocated minor in use)
1874
      if test:
1875
        node_drbd[minor] = (instance, False)
1876
      else:
1877
        instance = instanceinfo[instance]
1878
        node_drbd[minor] = (instance.name, instance.admin_up)
1879

    
1880
    # and now check them
1881
    used_minors = nresult.get(constants.NV_DRBDLIST, [])
1882
    test = not isinstance(used_minors, (tuple, list))
1883
    _ErrorIf(test, self.ENODEDRBD, node,
1884
             "cannot parse drbd status file: %s", str(used_minors))
1885
    if test:
1886
      # we cannot check drbd status
1887
      return
1888

    
1889
    for minor, (iname, must_exist) in node_drbd.items():
1890
      test = minor not in used_minors and must_exist
1891
      _ErrorIf(test, self.ENODEDRBD, node,
1892
               "drbd minor %d of instance %s is not active", minor, iname)
1893
    for minor in used_minors:
1894
      test = minor not in node_drbd
1895
      _ErrorIf(test, self.ENODEDRBD, node,
1896
               "unallocated drbd minor %d is in use", minor)
1897

    
1898
  def _UpdateNodeOS(self, ninfo, nresult, nimg):
1899
    """Builds the node OS structures.
1900

1901
    @type ninfo: L{objects.Node}
1902
    @param ninfo: the node to check
1903
    @param nresult: the remote results for the node
1904
    @param nimg: the node image object
1905

1906
    """
1907
    node = ninfo.name
1908
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1909

    
1910
    remote_os = nresult.get(constants.NV_OSLIST, None)
1911
    test = (not isinstance(remote_os, list) or
1912
            not compat.all(isinstance(v, list) and len(v) == 7
1913
                           for v in remote_os))
1914

    
1915
    _ErrorIf(test, self.ENODEOS, node,
1916
             "node hasn't returned valid OS data")
1917

    
1918
    nimg.os_fail = test
1919

    
1920
    if test:
1921
      return
1922

    
1923
    os_dict = {}
1924

    
1925
    for (name, os_path, status, diagnose,
1926
         variants, parameters, api_ver) in nresult[constants.NV_OSLIST]:
1927

    
1928
      if name not in os_dict:
1929
        os_dict[name] = []
1930

    
1931
      # parameters is a list of lists instead of list of tuples due to
1932
      # JSON lacking a real tuple type, fix it:
1933
      parameters = [tuple(v) for v in parameters]
1934
      os_dict[name].append((os_path, status, diagnose,
1935
                            set(variants), set(parameters), set(api_ver)))
1936

    
1937
    nimg.oslist = os_dict
1938

    
1939
  def _VerifyNodeOS(self, ninfo, nimg, base):
1940
    """Verifies the node OS list.
1941

1942
    @type ninfo: L{objects.Node}
1943
    @param ninfo: the node to check
1944
    @param nimg: the node image object
1945
    @param base: the 'template' node we match against (e.g. from the master)
1946

1947
    """
1948
    node = ninfo.name
1949
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1950

    
1951
    assert not nimg.os_fail, "Entered _VerifyNodeOS with failed OS rpc?"
1952

    
1953
    beautify_params = lambda l: ["%s: %s" % (k, v) for (k, v) in l]
1954
    for os_name, os_data in nimg.oslist.items():
1955
      assert os_data, "Empty OS status for OS %s?!" % os_name
1956
      f_path, f_status, f_diag, f_var, f_param, f_api = os_data[0]
1957
      _ErrorIf(not f_status, self.ENODEOS, node,
1958
               "Invalid OS %s (located at %s): %s", os_name, f_path, f_diag)
1959
      _ErrorIf(len(os_data) > 1, self.ENODEOS, node,
1960
               "OS '%s' has multiple entries (first one shadows the rest): %s",
1961
               os_name, utils.CommaJoin([v[0] for v in os_data]))
1962
      # this will catched in backend too
1963
      _ErrorIf(compat.any(v >= constants.OS_API_V15 for v in f_api)
1964
               and not f_var, self.ENODEOS, node,
1965
               "OS %s with API at least %d does not declare any variant",
1966
               os_name, constants.OS_API_V15)
1967
      # comparisons with the 'base' image
1968
      test = os_name not in base.oslist
1969
      _ErrorIf(test, self.ENODEOS, node,
1970
               "Extra OS %s not present on reference node (%s)",
1971
               os_name, base.name)
1972
      if test:
1973
        continue
1974
      assert base.oslist[os_name], "Base node has empty OS status?"
1975
      _, b_status, _, b_var, b_param, b_api = base.oslist[os_name][0]
1976
      if not b_status:
1977
        # base OS is invalid, skipping
1978
        continue
1979
      for kind, a, b in [("API version", f_api, b_api),
1980
                         ("variants list", f_var, b_var),
1981
                         ("parameters", beautify_params(f_param),
1982
                          beautify_params(b_param))]:
1983
        _ErrorIf(a != b, self.ENODEOS, node,
1984
                 "OS %s for %s differs from reference node %s: [%s] vs. [%s]",
1985
                 kind, os_name, base.name,
1986
                 utils.CommaJoin(sorted(a)), utils.CommaJoin(sorted(b)))
1987

    
1988
    # check any missing OSes
1989
    missing = set(base.oslist.keys()).difference(nimg.oslist.keys())
1990
    _ErrorIf(missing, self.ENODEOS, node,
1991
             "OSes present on reference node %s but missing on this node: %s",
1992
             base.name, utils.CommaJoin(missing))
1993

    
1994
  def _VerifyOob(self, ninfo, nresult):
1995
    """Verifies out of band functionality of a node.
1996

1997
    @type ninfo: L{objects.Node}
1998
    @param ninfo: the node to check
1999
    @param nresult: the remote results for the node
2000

2001
    """
2002
    node = ninfo.name
2003
    # We just have to verify the paths on master and/or master candidates
2004
    # as the oob helper is invoked on the master
2005
    if ((ninfo.master_candidate or ninfo.master_capable) and
2006
        constants.NV_OOB_PATHS in nresult):
2007
      for path_result in nresult[constants.NV_OOB_PATHS]:
2008
        self._ErrorIf(path_result, self.ENODEOOBPATH, node, path_result)
2009

    
2010
  def _UpdateNodeVolumes(self, ninfo, nresult, nimg, vg_name):
2011
    """Verifies and updates the node volume data.
2012

2013
    This function will update a L{NodeImage}'s internal structures
2014
    with data from the remote call.
2015

2016
    @type ninfo: L{objects.Node}
2017
    @param ninfo: the node to check
2018
    @param nresult: the remote results for the node
2019
    @param nimg: the node image object
2020
    @param vg_name: the configured VG name
2021

2022
    """
2023
    node = ninfo.name
2024
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2025

    
2026
    nimg.lvm_fail = True
2027
    lvdata = nresult.get(constants.NV_LVLIST, "Missing LV data")
2028
    if vg_name is None:
2029
      pass
2030
    elif isinstance(lvdata, basestring):
2031
      _ErrorIf(True, self.ENODELVM, node, "LVM problem on node: %s",
2032
               utils.SafeEncode(lvdata))
2033
    elif not isinstance(lvdata, dict):
2034
      _ErrorIf(True, self.ENODELVM, node, "rpc call to node failed (lvlist)")
2035
    else:
2036
      nimg.volumes = lvdata
2037
      nimg.lvm_fail = False
2038

    
2039
  def _UpdateNodeInstances(self, ninfo, nresult, nimg):
2040
    """Verifies and updates the node instance list.
2041

2042
    If the listing was successful, then updates this node's instance
2043
    list. Otherwise, it marks the RPC call as failed for the instance
2044
    list key.
2045

2046
    @type ninfo: L{objects.Node}
2047
    @param ninfo: the node to check
2048
    @param nresult: the remote results for the node
2049
    @param nimg: the node image object
2050

2051
    """
2052
    idata = nresult.get(constants.NV_INSTANCELIST, None)
2053
    test = not isinstance(idata, list)
2054
    self._ErrorIf(test, self.ENODEHV, ninfo.name, "rpc call to node failed"
2055
                  " (instancelist): %s", utils.SafeEncode(str(idata)))
2056
    if test:
2057
      nimg.hyp_fail = True
2058
    else:
2059
      nimg.instances = idata
2060

    
2061
  def _UpdateNodeInfo(self, ninfo, nresult, nimg, vg_name):
2062
    """Verifies and computes a node information map
2063

2064
    @type ninfo: L{objects.Node}
2065
    @param ninfo: the node to check
2066
    @param nresult: the remote results for the node
2067
    @param nimg: the node image object
2068
    @param vg_name: the configured VG name
2069

2070
    """
2071
    node = ninfo.name
2072
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2073

    
2074
    # try to read free memory (from the hypervisor)
2075
    hv_info = nresult.get(constants.NV_HVINFO, None)
2076
    test = not isinstance(hv_info, dict) or "memory_free" not in hv_info
2077
    _ErrorIf(test, self.ENODEHV, node, "rpc call to node failed (hvinfo)")
2078
    if not test:
2079
      try:
2080
        nimg.mfree = int(hv_info["memory_free"])
2081
      except (ValueError, TypeError):
2082
        _ErrorIf(True, self.ENODERPC, node,
2083
                 "node returned invalid nodeinfo, check hypervisor")
2084

    
2085
    # FIXME: devise a free space model for file based instances as well
2086
    if vg_name is not None:
2087
      test = (constants.NV_VGLIST not in nresult or
2088
              vg_name not in nresult[constants.NV_VGLIST])
2089
      _ErrorIf(test, self.ENODELVM, node,
2090
               "node didn't return data for the volume group '%s'"
2091
               " - it is either missing or broken", vg_name)
2092
      if not test:
2093
        try:
2094
          nimg.dfree = int(nresult[constants.NV_VGLIST][vg_name])
2095
        except (ValueError, TypeError):
2096
          _ErrorIf(True, self.ENODERPC, node,
2097
                   "node returned invalid LVM info, check LVM status")
2098

    
2099
  def _CollectDiskInfo(self, nodelist, node_image, instanceinfo):
2100
    """Gets per-disk status information for all instances.
2101

2102
    @type nodelist: list of strings
2103
    @param nodelist: Node names
2104
    @type node_image: dict of (name, L{objects.Node})
2105
    @param node_image: Node objects
2106
    @type instanceinfo: dict of (name, L{objects.Instance})
2107
    @param instanceinfo: Instance objects
2108
    @rtype: {instance: {node: [(succes, payload)]}}
2109
    @return: a dictionary of per-instance dictionaries with nodes as
2110
        keys and disk information as values; the disk information is a
2111
        list of tuples (success, payload)
2112

2113
    """
2114
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2115

    
2116
    node_disks = {}
2117
    node_disks_devonly = {}
2118
    diskless_instances = set()
2119
    diskless = constants.DT_DISKLESS
2120

    
2121
    for nname in nodelist:
2122
      node_instances = list(itertools.chain(node_image[nname].pinst,
2123
                                            node_image[nname].sinst))
2124
      diskless_instances.update(inst for inst in node_instances
2125
                                if instanceinfo[inst].disk_template == diskless)
2126
      disks = [(inst, disk)
2127
               for inst in node_instances
2128
               for disk in instanceinfo[inst].disks]
2129

    
2130
      if not disks:
2131
        # No need to collect data
2132
        continue
2133

    
2134
      node_disks[nname] = disks
2135

    
2136
      # Creating copies as SetDiskID below will modify the objects and that can
2137
      # lead to incorrect data returned from nodes
2138
      devonly = [dev.Copy() for (_, dev) in disks]
2139

    
2140
      for dev in devonly:
2141
        self.cfg.SetDiskID(dev, nname)
2142

    
2143
      node_disks_devonly[nname] = devonly
2144

    
2145
    assert len(node_disks) == len(node_disks_devonly)
2146

    
2147
    # Collect data from all nodes with disks
2148
    result = self.rpc.call_blockdev_getmirrorstatus_multi(node_disks.keys(),
2149
                                                          node_disks_devonly)
2150

    
2151
    assert len(result) == len(node_disks)
2152

    
2153
    instdisk = {}
2154

    
2155
    for (nname, nres) in result.items():
2156
      disks = node_disks[nname]
2157

    
2158
      if nres.offline:
2159
        # No data from this node
2160
        data = len(disks) * [(False, "node offline")]
2161
      else:
2162
        msg = nres.fail_msg
2163
        _ErrorIf(msg, self.ENODERPC, nname,
2164
                 "while getting disk information: %s", msg)
2165
        if msg:
2166
          # No data from this node
2167
          data = len(disks) * [(False, msg)]
2168
        else:
2169
          data = []
2170
          for idx, i in enumerate(nres.payload):
2171
            if isinstance(i, (tuple, list)) and len(i) == 2:
2172
              data.append(i)
2173
            else:
2174
              logging.warning("Invalid result from node %s, entry %d: %s",
2175
                              nname, idx, i)
2176
              data.append((False, "Invalid result from the remote node"))
2177

    
2178
      for ((inst, _), status) in zip(disks, data):
2179
        instdisk.setdefault(inst, {}).setdefault(nname, []).append(status)
2180

    
2181
    # Add empty entries for diskless instances.
2182
    for inst in diskless_instances:
2183
      assert inst not in instdisk
2184
      instdisk[inst] = {}
2185

    
2186
    assert compat.all(len(statuses) == len(instanceinfo[inst].disks) and
2187
                      len(nnames) <= len(instanceinfo[inst].all_nodes) and
2188
                      compat.all(isinstance(s, (tuple, list)) and
2189
                                 len(s) == 2 for s in statuses)
2190
                      for inst, nnames in instdisk.items()
2191
                      for nname, statuses in nnames.items())
2192
    assert set(instdisk) == set(instanceinfo), "instdisk consistency failure"
2193

    
2194
    return instdisk
2195

    
2196
  def _VerifyHVP(self, hvp_data):
2197
    """Verifies locally the syntax of the hypervisor parameters.
2198

2199
    """
2200
    for item, hv_name, hv_params in hvp_data:
2201
      msg = ("hypervisor %s parameters syntax check (source %s): %%s" %
2202
             (item, hv_name))
2203
      try:
2204
        hv_class = hypervisor.GetHypervisor(hv_name)
2205
        utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
2206
        hv_class.CheckParameterSyntax(hv_params)
2207
      except errors.GenericError, err:
2208
        self._ErrorIf(True, self.ECLUSTERCFG, None, msg % str(err))
2209

    
2210
  def BuildHooksEnv(self):
2211
    """Build hooks env.
2212

2213
    Cluster-Verify hooks just ran in the post phase and their failure makes
2214
    the output be logged in the verify output and the verification to fail.
2215

2216
    """
2217
    env = {
2218
      "CLUSTER_TAGS": " ".join(self.cfg.GetClusterInfo().GetTags())
2219
      }
2220

    
2221
    env.update(("NODE_TAGS_%s" % node.name, " ".join(node.GetTags()))
2222
               for node in self.my_node_info.values())
2223

    
2224
    return env
2225

    
2226
  def BuildHooksNodes(self):
2227
    """Build hooks nodes.
2228

2229
    """
2230
    assert self.my_node_names, ("Node list not gathered,"
2231
      " has CheckPrereq been executed?")
2232
    return ([], self.my_node_names)
2233

    
2234
  def Exec(self, feedback_fn):
2235
    """Verify integrity of cluster, performing various test on nodes.
2236

2237
    """
2238
    # This method has too many local variables. pylint: disable-msg=R0914
2239
    self.bad = False
2240
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2241
    verbose = self.op.verbose
2242
    self._feedback_fn = feedback_fn
2243
    feedback_fn("* Verifying global settings")
2244
    for msg in self.cfg.VerifyConfig():
2245
      _ErrorIf(True, self.ECLUSTERCFG, None, msg)
2246

    
2247
    # Check the cluster certificates
2248
    for cert_filename in constants.ALL_CERT_FILES:
2249
      (errcode, msg) = _VerifyCertificate(cert_filename)
2250
      _ErrorIf(errcode, self.ECLUSTERCERT, None, msg, code=errcode)
2251

    
2252
    vg_name = self.cfg.GetVGName()
2253
    drbd_helper = self.cfg.GetDRBDHelper()
2254
    hypervisors = self.cfg.GetClusterInfo().enabled_hypervisors
2255
    cluster = self.cfg.GetClusterInfo()
2256
    groupinfo = self.cfg.GetAllNodeGroupsInfo()
2257
    node_data_list = [self.my_node_info[name] for name in self.my_node_names]
2258

    
2259
    i_non_redundant = [] # Non redundant instances
2260
    i_non_a_balanced = [] # Non auto-balanced instances
2261
    n_offline = 0 # Count of offline nodes
2262
    n_drained = 0 # Count of nodes being drained
2263
    node_vol_should = {}
2264

    
2265
    # FIXME: verify OS list
2266

    
2267
    # File verification
2268
    filemap = _ComputeAncillaryFiles(cluster, False)
2269

    
2270
    # do local checksums
2271
    master_node = self.master_node = self.cfg.GetMasterNode()
2272
    master_ip = self.cfg.GetMasterIP()
2273

    
2274
    # Compute the set of hypervisor parameters
2275
    hvp_data = []
2276
    for hv_name in hypervisors:
2277
      hvp_data.append(("cluster", hv_name, cluster.GetHVDefaults(hv_name)))
2278
    for os_name, os_hvp in cluster.os_hvp.items():
2279
      for hv_name, hv_params in os_hvp.items():
2280
        if not hv_params:
2281
          continue
2282
        full_params = cluster.GetHVDefaults(hv_name, os_name=os_name)
2283
        hvp_data.append(("os %s" % os_name, hv_name, full_params))
2284
    # TODO: collapse identical parameter values in a single one
2285
    for instance in self.all_inst_info.values():
2286
      if not instance.hvparams:
2287
        continue
2288
      hvp_data.append(("instance %s" % instance.name, instance.hypervisor,
2289
                       cluster.FillHV(instance)))
2290
    # and verify them locally
2291
    self._VerifyHVP(hvp_data)
2292

    
2293
    feedback_fn("* Gathering data (%d nodes)" % len(self.my_node_names))
2294
    node_verify_param = {
2295
      constants.NV_FILELIST:
2296
        utils.UniqueSequence(filename
2297
                             for files in filemap
2298
                             for filename in files),
2299
      constants.NV_NODELIST: [node.name for node in self.all_node_info.values()
2300
                              if not node.offline],
2301
      constants.NV_HYPERVISOR: hypervisors,
2302
      constants.NV_HVPARAMS: hvp_data,
2303
      constants.NV_NODENETTEST: [(node.name, node.primary_ip, node.secondary_ip)
2304
                                 for node in node_data_list
2305
                                 if not node.offline],
2306
      constants.NV_INSTANCELIST: hypervisors,
2307
      constants.NV_VERSION: None,
2308
      constants.NV_HVINFO: self.cfg.GetHypervisorType(),
2309
      constants.NV_NODESETUP: None,
2310
      constants.NV_TIME: None,
2311
      constants.NV_MASTERIP: (master_node, master_ip),
2312
      constants.NV_OSLIST: None,
2313
      constants.NV_VMNODES: self.cfg.GetNonVmCapableNodeList(),
2314
      }
2315

    
2316
    if vg_name is not None:
2317
      node_verify_param[constants.NV_VGLIST] = None
2318
      node_verify_param[constants.NV_LVLIST] = vg_name
2319
      node_verify_param[constants.NV_PVLIST] = [vg_name]
2320
      node_verify_param[constants.NV_DRBDLIST] = None
2321

    
2322
    if drbd_helper:
2323
      node_verify_param[constants.NV_DRBDHELPER] = drbd_helper
2324

    
2325
    # bridge checks
2326
    # FIXME: this needs to be changed per node-group, not cluster-wide
2327
    bridges = set()
2328
    default_nicpp = cluster.nicparams[constants.PP_DEFAULT]
2329
    if default_nicpp[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2330
      bridges.add(default_nicpp[constants.NIC_LINK])
2331
    for instance in self.my_inst_info.values():
2332
      for nic in instance.nics:
2333
        full_nic = cluster.SimpleFillNIC(nic.nicparams)
2334
        if full_nic[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2335
          bridges.add(full_nic[constants.NIC_LINK])
2336

    
2337
    if bridges:
2338
      node_verify_param[constants.NV_BRIDGES] = list(bridges)
2339

    
2340
    # Build our expected cluster state
2341
    node_image = dict((node.name, self.NodeImage(offline=node.offline,
2342
                                                 name=node.name,
2343
                                                 vm_capable=node.vm_capable))
2344
                      for node in node_data_list)
2345

    
2346
    # Gather OOB paths
2347
    oob_paths = []
2348
    for node in self.all_node_info.values():
2349
      path = _SupportsOob(self.cfg, node)
2350
      if path and path not in oob_paths:
2351
        oob_paths.append(path)
2352

    
2353
    if oob_paths:
2354
      node_verify_param[constants.NV_OOB_PATHS] = oob_paths
2355

    
2356
    for instance in self.my_inst_names:
2357
      inst_config = self.my_inst_info[instance]
2358

    
2359
      for nname in inst_config.all_nodes:
2360
        if nname not in node_image:
2361
          # ghost node
2362
          gnode = self.NodeImage(name=nname)
2363
          gnode.ghost = True
2364
          node_image[nname] = gnode
2365

    
2366
      inst_config.MapLVsByNode(node_vol_should)
2367

    
2368
      pnode = inst_config.primary_node
2369
      node_image[pnode].pinst.append(instance)
2370

    
2371
      for snode in inst_config.secondary_nodes:
2372
        nimg = node_image[snode]
2373
        nimg.sinst.append(instance)
2374
        if pnode not in nimg.sbp:
2375
          nimg.sbp[pnode] = []
2376
        nimg.sbp[pnode].append(instance)
2377

    
2378
    # At this point, we have the in-memory data structures complete,
2379
    # except for the runtime information, which we'll gather next
2380

    
2381
    # Due to the way our RPC system works, exact response times cannot be
2382
    # guaranteed (e.g. a broken node could run into a timeout). By keeping the
2383
    # time before and after executing the request, we can at least have a time
2384
    # window.
2385
    nvinfo_starttime = time.time()
2386
    all_nvinfo = self.rpc.call_node_verify(self.my_node_names,
2387
                                           node_verify_param,
2388
                                           self.cfg.GetClusterName())
2389
    nvinfo_endtime = time.time()
2390

    
2391
    all_drbd_map = self.cfg.ComputeDRBDMap()
2392

    
2393
    feedback_fn("* Gathering disk information (%s nodes)" %
2394
                len(self.my_node_names))
2395
    instdisk = self._CollectDiskInfo(self.my_node_names, node_image,
2396
                                     self.my_inst_info)
2397

    
2398
    feedback_fn("* Verifying configuration file consistency")
2399

    
2400
    # If not all nodes are being checked, we need to make sure the master node
2401
    # and a non-checked vm_capable node are in the list.
2402
    absent_nodes = set(self.all_node_info).difference(self.my_node_info)
2403
    if absent_nodes:
2404
      vf_nvinfo = all_nvinfo.copy()
2405
      vf_node_info = list(self.my_node_info.values())
2406
      additional_nodes = []
2407
      if master_node not in self.my_node_info:
2408
        additional_nodes.append(master_node)
2409
        vf_node_info.append(self.all_node_info[master_node])
2410
      # Add the first vm_capable node we find which is not included
2411
      for node in absent_nodes:
2412
        nodeinfo = self.all_node_info[node]
2413
        if nodeinfo.vm_capable and not nodeinfo.offline:
2414
          additional_nodes.append(node)
2415
          vf_node_info.append(self.all_node_info[node])
2416
          break
2417
      key = constants.NV_FILELIST
2418
      vf_nvinfo.update(self.rpc.call_node_verify(additional_nodes,
2419
                                                 {key: node_verify_param[key]},
2420
                                                 self.cfg.GetClusterName()))
2421
    else:
2422
      vf_nvinfo = all_nvinfo
2423
      vf_node_info = self.my_node_info.values()
2424

    
2425
    self._VerifyFiles(_ErrorIf, vf_node_info, master_node, vf_nvinfo, filemap)
2426

    
2427
    feedback_fn("* Verifying node status")
2428

    
2429
    refos_img = None
2430

    
2431
    for node_i in node_data_list:
2432
      node = node_i.name
2433
      nimg = node_image[node]
2434

    
2435
      if node_i.offline:
2436
        if verbose:
2437
          feedback_fn("* Skipping offline node %s" % (node,))
2438
        n_offline += 1
2439
        continue
2440

    
2441
      if node == master_node:
2442
        ntype = "master"
2443
      elif node_i.master_candidate:
2444
        ntype = "master candidate"
2445
      elif node_i.drained:
2446
        ntype = "drained"
2447
        n_drained += 1
2448
      else:
2449
        ntype = "regular"
2450
      if verbose:
2451
        feedback_fn("* Verifying node %s (%s)" % (node, ntype))
2452

    
2453
      msg = all_nvinfo[node].fail_msg
2454
      _ErrorIf(msg, self.ENODERPC, node, "while contacting node: %s", msg)
2455
      if msg:
2456
        nimg.rpc_fail = True
2457
        continue
2458

    
2459
      nresult = all_nvinfo[node].payload
2460

    
2461
      nimg.call_ok = self._VerifyNode(node_i, nresult)
2462
      self._VerifyNodeTime(node_i, nresult, nvinfo_starttime, nvinfo_endtime)
2463
      self._VerifyNodeNetwork(node_i, nresult)
2464
      self._VerifyOob(node_i, nresult)
2465

    
2466
      if nimg.vm_capable:
2467
        self._VerifyNodeLVM(node_i, nresult, vg_name)
2468
        self._VerifyNodeDrbd(node_i, nresult, self.all_inst_info, drbd_helper,
2469
                             all_drbd_map)
2470

    
2471
        self._UpdateNodeVolumes(node_i, nresult, nimg, vg_name)
2472
        self._UpdateNodeInstances(node_i, nresult, nimg)
2473
        self._UpdateNodeInfo(node_i, nresult, nimg, vg_name)
2474
        self._UpdateNodeOS(node_i, nresult, nimg)
2475

    
2476
        if not nimg.os_fail:
2477
          if refos_img is None:
2478
            refos_img = nimg
2479
          self._VerifyNodeOS(node_i, nimg, refos_img)
2480
        self._VerifyNodeBridges(node_i, nresult, bridges)
2481

    
2482
        # Check whether all running instancies are primary for the node. (This
2483
        # can no longer be done from _VerifyInstance below, since some of the
2484
        # wrong instances could be from other node groups.)
2485
        non_primary_inst = set(nimg.instances).difference(nimg.pinst)
2486

    
2487
        for inst in non_primary_inst:
2488
          test = inst in self.all_inst_info
2489
          _ErrorIf(test, self.EINSTANCEWRONGNODE, inst,
2490
                   "instance should not run on node %s", node_i.name)
2491
          _ErrorIf(not test, self.ENODEORPHANINSTANCE, node_i.name,
2492
                   "node is running unknown instance %s", inst)
2493

    
2494
    feedback_fn("* Verifying instance status")
2495
    for instance in self.my_inst_names:
2496
      if verbose:
2497
        feedback_fn("* Verifying instance %s" % instance)
2498
      inst_config = self.my_inst_info[instance]
2499
      self._VerifyInstance(instance, inst_config, node_image,
2500
                           instdisk[instance])
2501
      inst_nodes_offline = []
2502

    
2503
      pnode = inst_config.primary_node
2504
      pnode_img = node_image[pnode]
2505
      _ErrorIf(pnode_img.rpc_fail and not pnode_img.offline,
2506
               self.ENODERPC, pnode, "instance %s, connection to"
2507
               " primary node failed", instance)
2508

    
2509
      _ErrorIf(inst_config.admin_up and pnode_img.offline,
2510
               self.EINSTANCEBADNODE, instance,
2511
               "instance is marked as running and lives on offline node %s",
2512
               inst_config.primary_node)
2513

    
2514
      # If the instance is non-redundant we cannot survive losing its primary
2515
      # node, so we are not N+1 compliant. On the other hand we have no disk
2516
      # templates with more than one secondary so that situation is not well
2517
      # supported either.
2518
      # FIXME: does not support file-backed instances
2519
      if not inst_config.secondary_nodes:
2520
        i_non_redundant.append(instance)
2521

    
2522
      _ErrorIf(len(inst_config.secondary_nodes) > 1, self.EINSTANCELAYOUT,
2523
               instance, "instance has multiple secondary nodes: %s",
2524
               utils.CommaJoin(inst_config.secondary_nodes),
2525
               code=self.ETYPE_WARNING)
2526

    
2527
      if inst_config.disk_template in constants.DTS_INT_MIRROR:
2528
        pnode = inst_config.primary_node
2529
        instance_nodes = utils.NiceSort(inst_config.all_nodes)
2530
        instance_groups = {}
2531

    
2532
        for node in instance_nodes:
2533
          instance_groups.setdefault(self.all_node_info[node].group,
2534
                                     []).append(node)
2535

    
2536
        pretty_list = [
2537
          "%s (group %s)" % (utils.CommaJoin(nodes), groupinfo[group].name)
2538
          # Sort so that we always list the primary node first.
2539
          for group, nodes in sorted(instance_groups.items(),
2540
                                     key=lambda (_, nodes): pnode in nodes,
2541
                                     reverse=True)]
2542

    
2543
        self._ErrorIf(len(instance_groups) > 1, self.EINSTANCESPLITGROUPS,
2544
                      instance, "instance has primary and secondary nodes in"
2545
                      " different groups: %s", utils.CommaJoin(pretty_list),
2546
                      code=self.ETYPE_WARNING)
2547

    
2548
      if not cluster.FillBE(inst_config)[constants.BE_AUTO_BALANCE]:
2549
        i_non_a_balanced.append(instance)
2550

    
2551
      for snode in inst_config.secondary_nodes:
2552
        s_img = node_image[snode]
2553
        _ErrorIf(s_img.rpc_fail and not s_img.offline, self.ENODERPC, snode,
2554
                 "instance %s, connection to secondary node failed", instance)
2555

    
2556
        if s_img.offline:
2557
          inst_nodes_offline.append(snode)
2558

    
2559
      # warn that the instance lives on offline nodes
2560
      _ErrorIf(inst_nodes_offline, self.EINSTANCEBADNODE, instance,
2561
               "instance has offline secondary node(s) %s",
2562
               utils.CommaJoin(inst_nodes_offline))
2563
      # ... or ghost/non-vm_capable nodes
2564
      for node in inst_config.all_nodes:
2565
        _ErrorIf(node_image[node].ghost, self.EINSTANCEBADNODE, instance,
2566
                 "instance lives on ghost node %s", node)
2567
        _ErrorIf(not node_image[node].vm_capable, self.EINSTANCEBADNODE,
2568
                 instance, "instance lives on non-vm_capable node %s", node)
2569

    
2570
    feedback_fn("* Verifying orphan volumes")
2571
    reserved = utils.FieldSet(*cluster.reserved_lvs)
2572
    self._VerifyOrphanVolumes(node_vol_should, node_image, reserved)
2573

    
2574
    if constants.VERIFY_NPLUSONE_MEM not in self.op.skip_checks:
2575
      feedback_fn("* Verifying N+1 Memory redundancy")
2576
      self._VerifyNPlusOneMemory(node_image, self.my_inst_info)
2577

    
2578
    feedback_fn("* Other Notes")
2579
    if i_non_redundant:
2580
      feedback_fn("  - NOTICE: %d non-redundant instance(s) found."
2581
                  % len(i_non_redundant))
2582

    
2583
    if i_non_a_balanced:
2584
      feedback_fn("  - NOTICE: %d non-auto-balanced instance(s) found."
2585
                  % len(i_non_a_balanced))
2586

    
2587
    if n_offline:
2588
      feedback_fn("  - NOTICE: %d offline node(s) found." % n_offline)
2589

    
2590
    if n_drained:
2591
      feedback_fn("  - NOTICE: %d drained node(s) found." % n_drained)
2592

    
2593
    return not self.bad
2594

    
2595
  def HooksCallBack(self, phase, hooks_results, feedback_fn, lu_result):
2596
    """Analyze the post-hooks' result
2597

2598
    This method analyses the hook result, handles it, and sends some
2599
    nicely-formatted feedback back to the user.
2600

2601
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
2602
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
2603
    @param hooks_results: the results of the multi-node hooks rpc call
2604
    @param feedback_fn: function used send feedback back to the caller
2605
    @param lu_result: previous Exec result
2606
    @return: the new Exec result, based on the previous result
2607
        and hook results
2608

2609
    """
2610
    # We only really run POST phase hooks, and are only interested in
2611
    # their results
2612
    if phase == constants.HOOKS_PHASE_POST:
2613
      # Used to change hooks' output to proper indentation
2614
      feedback_fn("* Hooks Results")
2615
      assert hooks_results, "invalid result from hooks"
2616

    
2617
      for node_name in hooks_results:
2618
        res = hooks_results[node_name]
2619
        msg = res.fail_msg
2620
        test = msg and not res.offline
2621
        self._ErrorIf(test, self.ENODEHOOKS, node_name,
2622
                      "Communication failure in hooks execution: %s", msg)
2623
        if res.offline or msg:
2624
          # No need to investigate payload if node is offline or gave an error.
2625
          # override manually lu_result here as _ErrorIf only
2626
          # overrides self.bad
2627
          lu_result = 1
2628
          continue
2629
        for script, hkr, output in res.payload:
2630
          test = hkr == constants.HKR_FAIL
2631
          self._ErrorIf(test, self.ENODEHOOKS, node_name,
2632
                        "Script %s failed, output:", script)
2633
          if test:
2634
            output = self._HOOKS_INDENT_RE.sub('      ', output)
2635
            feedback_fn("%s" % output)
2636
            lu_result = 0
2637

    
2638
      return lu_result
2639

    
2640

    
2641
class LUClusterVerifyDisks(NoHooksLU):
2642
  """Verifies the cluster disks status.
2643

2644
  """
2645
  REQ_BGL = False
2646

    
2647
  def ExpandNames(self):
2648
    self.needed_locks = {
2649
      locking.LEVEL_NODE: locking.ALL_SET,
2650
      locking.LEVEL_INSTANCE: locking.ALL_SET,
2651
    }
2652
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2653

    
2654
  def Exec(self, feedback_fn):
2655
    """Verify integrity of cluster disks.
2656

2657
    @rtype: tuple of three items
2658
    @return: a tuple of (dict of node-to-node_error, list of instances
2659
        which need activate-disks, dict of instance: (node, volume) for
2660
        missing volumes
2661

2662
    """
2663
    result = res_nodes, res_instances, res_missing = {}, [], {}
2664

    
2665
    nodes = utils.NiceSort(self.cfg.GetVmCapableNodeList())
2666
    instances = self.cfg.GetAllInstancesInfo().values()
2667

    
2668
    nv_dict = {}
2669
    for inst in instances:
2670
      inst_lvs = {}
2671
      if not inst.admin_up:
2672
        continue
2673
      inst.MapLVsByNode(inst_lvs)
2674
      # transform { iname: {node: [vol,],},} to {(node, vol): iname}
2675
      for node, vol_list in inst_lvs.iteritems():
2676
        for vol in vol_list:
2677
          nv_dict[(node, vol)] = inst
2678

    
2679
    if not nv_dict:
2680
      return result
2681

    
2682
    node_lvs = self.rpc.call_lv_list(nodes, [])
2683
    for node, node_res in node_lvs.items():
2684
      if node_res.offline:
2685
        continue
2686
      msg = node_res.fail_msg
2687
      if msg:
2688
        logging.warning("Error enumerating LVs on node %s: %s", node, msg)
2689
        res_nodes[node] = msg
2690
        continue
2691

    
2692
      lvs = node_res.payload
2693
      for lv_name, (_, _, lv_online) in lvs.items():
2694
        inst = nv_dict.pop((node, lv_name), None)
2695
        if (not lv_online and inst is not None
2696
            and inst.name not in res_instances):
2697
          res_instances.append(inst.name)
2698

    
2699
    # any leftover items in nv_dict are missing LVs, let's arrange the
2700
    # data better
2701
    for key, inst in nv_dict.iteritems():
2702
      if inst.name not in res_missing:
2703
        res_missing[inst.name] = []
2704
      res_missing[inst.name].append(key)
2705

    
2706
    return result
2707

    
2708

    
2709
class LUClusterRepairDiskSizes(NoHooksLU):
2710
  """Verifies the cluster disks sizes.
2711

2712
  """
2713
  REQ_BGL = False
2714

    
2715
  def ExpandNames(self):
2716
    if self.op.instances:
2717
      self.wanted_names = _GetWantedInstances(self, self.op.instances)
2718
      self.needed_locks = {
2719
        locking.LEVEL_NODE: [],
2720
        locking.LEVEL_INSTANCE: self.wanted_names,
2721
        }
2722
      self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE
2723
    else:
2724
      self.wanted_names = None
2725
      self.needed_locks = {
2726
        locking.LEVEL_NODE: locking.ALL_SET,
2727
        locking.LEVEL_INSTANCE: locking.ALL_SET,
2728
        }
2729
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2730

    
2731
  def DeclareLocks(self, level):
2732
    if level == locking.LEVEL_NODE and self.wanted_names is not None:
2733
      self._LockInstancesNodes(primary_only=True)
2734

    
2735
  def CheckPrereq(self):
2736
    """Check prerequisites.
2737

2738
    This only checks the optional instance list against the existing names.
2739

2740
    """
2741
    if self.wanted_names is None:
2742
      self.wanted_names = self.glm.list_owned(locking.LEVEL_INSTANCE)
2743

    
2744
    self.wanted_instances = [self.cfg.GetInstanceInfo(name) for name
2745
                             in self.wanted_names]
2746

    
2747
  def _EnsureChildSizes(self, disk):
2748
    """Ensure children of the disk have the needed disk size.
2749

2750
    This is valid mainly for DRBD8 and fixes an issue where the
2751
    children have smaller disk size.
2752

2753
    @param disk: an L{ganeti.objects.Disk} object
2754

2755
    """
2756
    if disk.dev_type == constants.LD_DRBD8:
2757
      assert disk.children, "Empty children for DRBD8?"
2758
      fchild = disk.children[0]
2759
      mismatch = fchild.size < disk.size
2760
      if mismatch:
2761
        self.LogInfo("Child disk has size %d, parent %d, fixing",
2762
                     fchild.size, disk.size)
2763
        fchild.size = disk.size
2764

    
2765
      # and we recurse on this child only, not on the metadev
2766
      return self._EnsureChildSizes(fchild) or mismatch
2767
    else:
2768
      return False
2769

    
2770
  def Exec(self, feedback_fn):
2771
    """Verify the size of cluster disks.
2772

2773
    """
2774
    # TODO: check child disks too
2775
    # TODO: check differences in size between primary/secondary nodes
2776
    per_node_disks = {}
2777
    for instance in self.wanted_instances:
2778
      pnode = instance.primary_node
2779
      if pnode not in per_node_disks:
2780
        per_node_disks[pnode] = []
2781
      for idx, disk in enumerate(instance.disks):
2782
        per_node_disks[pnode].append((instance, idx, disk))
2783

    
2784
    changed = []
2785
    for node, dskl in per_node_disks.items():
2786
      newl = [v[2].Copy() for v in dskl]
2787
      for dsk in newl:
2788
        self.cfg.SetDiskID(dsk, node)
2789
      result = self.rpc.call_blockdev_getsize(node, newl)
2790
      if result.fail_msg:
2791
        self.LogWarning("Failure in blockdev_getsize call to node"
2792
                        " %s, ignoring", node)
2793
        continue
2794
      if len(result.payload) != len(dskl):
2795
        logging.warning("Invalid result from node %s: len(dksl)=%d,"
2796
                        " result.payload=%s", node, len(dskl), result.payload)
2797
        self.LogWarning("Invalid result from node %s, ignoring node results",
2798
                        node)
2799
        continue
2800
      for ((instance, idx, disk), size) in zip(dskl, result.payload):
2801
        if size is None:
2802
          self.LogWarning("Disk %d of instance %s did not return size"
2803
                          " information, ignoring", idx, instance.name)
2804
          continue
2805
        if not isinstance(size, (int, long)):
2806
          self.LogWarning("Disk %d of instance %s did not return valid"
2807
                          " size information, ignoring", idx, instance.name)
2808
          continue
2809
        size = size >> 20
2810
        if size != disk.size:
2811
          self.LogInfo("Disk %d of instance %s has mismatched size,"
2812
                       " correcting: recorded %d, actual %d", idx,
2813
                       instance.name, disk.size, size)
2814
          disk.size = size
2815
          self.cfg.Update(instance, feedback_fn)
2816
          changed.append((instance.name, idx, size))
2817
        if self._EnsureChildSizes(disk):
2818
          self.cfg.Update(instance, feedback_fn)
2819
          changed.append((instance.name, idx, disk.size))
2820
    return changed
2821

    
2822

    
2823
class LUClusterRename(LogicalUnit):
2824
  """Rename the cluster.
2825

2826
  """
2827
  HPATH = "cluster-rename"
2828
  HTYPE = constants.HTYPE_CLUSTER
2829

    
2830
  def BuildHooksEnv(self):
2831
    """Build hooks env.
2832

2833
    """
2834
    return {
2835
      "OP_TARGET": self.cfg.GetClusterName(),
2836
      "NEW_NAME": self.op.name,
2837
      }
2838

    
2839
  def BuildHooksNodes(self):
2840
    """Build hooks nodes.
2841

2842
    """
2843
    return ([self.cfg.GetMasterNode()], self.cfg.GetNodeList())
2844

    
2845
  def CheckPrereq(self):
2846
    """Verify that the passed name is a valid one.
2847

2848
    """
2849
    hostname = netutils.GetHostname(name=self.op.name,
2850
                                    family=self.cfg.GetPrimaryIPFamily())
2851

    
2852
    new_name = hostname.name
2853
    self.ip = new_ip = hostname.ip
2854
    old_name = self.cfg.GetClusterName()
2855
    old_ip = self.cfg.GetMasterIP()
2856
    if new_name == old_name and new_ip == old_ip:
2857
      raise errors.OpPrereqError("Neither the name nor the IP address of the"
2858
                                 " cluster has changed",
2859
                                 errors.ECODE_INVAL)
2860
    if new_ip != old_ip:
2861
      if netutils.TcpPing(new_ip, constants.DEFAULT_NODED_PORT):
2862
        raise errors.OpPrereqError("The given cluster IP address (%s) is"
2863
                                   " reachable on the network" %
2864
                                   new_ip, errors.ECODE_NOTUNIQUE)
2865

    
2866
    self.op.name = new_name
2867

    
2868
  def Exec(self, feedback_fn):
2869
    """Rename the cluster.
2870

2871
    """
2872
    clustername = self.op.name
2873
    ip = self.ip
2874

    
2875
    # shutdown the master IP
2876
    master = self.cfg.GetMasterNode()
2877
    result = self.rpc.call_node_stop_master(master, False)
2878
    result.Raise("Could not disable the master role")
2879

    
2880
    try:
2881
      cluster = self.cfg.GetClusterInfo()
2882
      cluster.cluster_name = clustername
2883
      cluster.master_ip = ip
2884
      self.cfg.Update(cluster, feedback_fn)
2885

    
2886
      # update the known hosts file
2887
      ssh.WriteKnownHostsFile(self.cfg, constants.SSH_KNOWN_HOSTS_FILE)
2888
      node_list = self.cfg.GetOnlineNodeList()
2889
      try:
2890
        node_list.remove(master)
2891
      except ValueError:
2892
        pass
2893
      _UploadHelper(self, node_list, constants.SSH_KNOWN_HOSTS_FILE)
2894
    finally:
2895
      result = self.rpc.call_node_start_master(master, False, False)
2896
      msg = result.fail_msg
2897
      if msg:
2898
        self.LogWarning("Could not re-enable the master role on"
2899
                        " the master, please restart manually: %s", msg)
2900

    
2901
    return clustername
2902

    
2903

    
2904
class LUClusterSetParams(LogicalUnit):
2905
  """Change the parameters of the cluster.
2906

2907
  """
2908
  HPATH = "cluster-modify"
2909
  HTYPE = constants.HTYPE_CLUSTER
2910
  REQ_BGL = False
2911

    
2912
  def CheckArguments(self):
2913
    """Check parameters
2914

2915
    """
2916
    if self.op.uid_pool:
2917
      uidpool.CheckUidPool(self.op.uid_pool)
2918

    
2919
    if self.op.add_uids:
2920
      uidpool.CheckUidPool(self.op.add_uids)
2921

    
2922
    if self.op.remove_uids:
2923
      uidpool.CheckUidPool(self.op.remove_uids)
2924

    
2925
  def ExpandNames(self):
2926
    # FIXME: in the future maybe other cluster params won't require checking on
2927
    # all nodes to be modified.
2928
    self.needed_locks = {
2929
      locking.LEVEL_NODE: locking.ALL_SET,
2930
    }
2931
    self.share_locks[locking.LEVEL_NODE] = 1
2932

    
2933
  def BuildHooksEnv(self):
2934
    """Build hooks env.
2935

2936
    """
2937
    return {
2938
      "OP_TARGET": self.cfg.GetClusterName(),
2939
      "NEW_VG_NAME": self.op.vg_name,
2940
      }
2941

    
2942
  def BuildHooksNodes(self):
2943
    """Build hooks nodes.
2944

2945
    """
2946
    mn = self.cfg.GetMasterNode()
2947
    return ([mn], [mn])
2948

    
2949
  def CheckPrereq(self):
2950
    """Check prerequisites.
2951

2952
    This checks whether the given params don't conflict and
2953
    if the given volume group is valid.
2954

2955
    """
2956
    if self.op.vg_name is not None and not self.op.vg_name:
2957
      if self.cfg.HasAnyDiskOfType(constants.LD_LV):
2958
        raise errors.OpPrereqError("Cannot disable lvm storage while lvm-based"
2959
                                   " instances exist", errors.ECODE_INVAL)
2960

    
2961
    if self.op.drbd_helper is not None and not self.op.drbd_helper:
2962
      if self.cfg.HasAnyDiskOfType(constants.LD_DRBD8):
2963
        raise errors.OpPrereqError("Cannot disable drbd helper while"
2964
                                   " drbd-based instances exist",
2965
                                   errors.ECODE_INVAL)
2966

    
2967
    node_list = self.glm.list_owned(locking.LEVEL_NODE)
2968

    
2969
    # if vg_name not None, checks given volume group on all nodes
2970
    if self.op.vg_name:
2971
      vglist = self.rpc.call_vg_list(node_list)
2972
      for node in node_list:
2973
        msg = vglist[node].fail_msg
2974
        if msg:
2975
          # ignoring down node
2976
          self.LogWarning("Error while gathering data on node %s"
2977
                          " (ignoring node): %s", node, msg)
2978
          continue
2979
        vgstatus = utils.CheckVolumeGroupSize(vglist[node].payload,
2980
                                              self.op.vg_name,
2981
                                              constants.MIN_VG_SIZE)
2982
        if vgstatus:
2983
          raise errors.OpPrereqError("Error on node '%s': %s" %
2984
                                     (node, vgstatus), errors.ECODE_ENVIRON)
2985

    
2986
    if self.op.drbd_helper:
2987
      # checks given drbd helper on all nodes
2988
      helpers = self.rpc.call_drbd_helper(node_list)
2989
      for node in node_list:
2990
        ninfo = self.cfg.GetNodeInfo(node)
2991
        if ninfo.offline:
2992
          self.LogInfo("Not checking drbd helper on offline node %s", node)
2993
          continue
2994
        msg = helpers[node].fail_msg
2995
        if msg:
2996
          raise errors.OpPrereqError("Error checking drbd helper on node"
2997
                                     " '%s': %s" % (node, msg),
2998
                                     errors.ECODE_ENVIRON)
2999
        node_helper = helpers[node].payload
3000
        if node_helper != self.op.drbd_helper:
3001
          raise errors.OpPrereqError("Error on node '%s': drbd helper is %s" %
3002
                                     (node, node_helper), errors.ECODE_ENVIRON)
3003

    
3004
    self.cluster = cluster = self.cfg.GetClusterInfo()
3005
    # validate params changes
3006
    if self.op.beparams:
3007
      utils.ForceDictType(self.op.beparams, constants.BES_PARAMETER_TYPES)
3008
      self.new_beparams = cluster.SimpleFillBE(self.op.beparams)
3009

    
3010
    if self.op.ndparams:
3011
      utils.ForceDictType(self.op.ndparams, constants.NDS_PARAMETER_TYPES)
3012
      self.new_ndparams = cluster.SimpleFillND(self.op.ndparams)
3013

    
3014
      # TODO: we need a more general way to handle resetting
3015
      # cluster-level parameters to default values
3016
      if self.new_ndparams["oob_program"] == "":
3017
        self.new_ndparams["oob_program"] = \
3018
            constants.NDC_DEFAULTS[constants.ND_OOB_PROGRAM]
3019

    
3020
    if self.op.nicparams:
3021
      utils.ForceDictType(self.op.nicparams, constants.NICS_PARAMETER_TYPES)
3022
      self.new_nicparams = cluster.SimpleFillNIC(self.op.nicparams)
3023
      objects.NIC.CheckParameterSyntax(self.new_nicparams)
3024
      nic_errors = []
3025

    
3026
      # check all instances for consistency
3027
      for instance in self.cfg.GetAllInstancesInfo().values():
3028
        for nic_idx, nic in enumerate(instance.nics):
3029
          params_copy = copy.deepcopy(nic.nicparams)
3030
          params_filled = objects.FillDict(self.new_nicparams, params_copy)
3031

    
3032
          # check parameter syntax
3033
          try:
3034
            objects.NIC.CheckParameterSyntax(params_filled)
3035
          except errors.ConfigurationError, err:
3036
            nic_errors.append("Instance %s, nic/%d: %s" %
3037
                              (instance.name, nic_idx, err))
3038

    
3039
          # if we're moving instances to routed, check that they have an ip
3040
          target_mode = params_filled[constants.NIC_MODE]
3041
          if target_mode == constants.NIC_MODE_ROUTED and not nic.ip:
3042
            nic_errors.append("Instance %s, nic/%d: routed NIC with no ip"
3043
                              " address" % (instance.name, nic_idx))
3044
      if nic_errors:
3045
        raise errors.OpPrereqError("Cannot apply the change, errors:\n%s" %
3046
                                   "\n".join(nic_errors))
3047

    
3048
    # hypervisor list/parameters
3049
    self.new_hvparams = new_hvp = objects.FillDict(cluster.hvparams, {})
3050
    if self.op.hvparams:
3051
      for hv_name, hv_dict in self.op.hvparams.items():
3052
        if hv_name not in self.new_hvparams:
3053
          self.new_hvparams[hv_name] = hv_dict
3054
        else:
3055
          self.new_hvparams[hv_name].update(hv_dict)
3056

    
3057
    # os hypervisor parameters
3058
    self.new_os_hvp = objects.FillDict(cluster.os_hvp, {})
3059
    if self.op.os_hvp:
3060
      for os_name, hvs in self.op.os_hvp.items():
3061
        if os_name not in self.new_os_hvp:
3062
          self.new_os_hvp[os_name] = hvs
3063
        else:
3064
          for hv_name, hv_dict in hvs.items():
3065
            if hv_name not in self.new_os_hvp[os_name]:
3066
              self.new_os_hvp[os_name][hv_name] = hv_dict
3067
            else:
3068
              self.new_os_hvp[os_name][hv_name].update(hv_dict)
3069

    
3070
    # os parameters
3071
    self.new_osp = objects.FillDict(cluster.osparams, {})
3072
    if self.op.osparams:
3073
      for os_name, osp in self.op.osparams.items():
3074
        if os_name not in self.new_osp:
3075
          self.new_osp[os_name] = {}
3076

    
3077
        self.new_osp[os_name] = _GetUpdatedParams(self.new_osp[os_name], osp,
3078
                                                  use_none=True)
3079

    
3080
        if not self.new_osp[os_name]:
3081
          # we removed all parameters
3082
          del self.new_osp[os_name]
3083
        else:
3084
          # check the parameter validity (remote check)
3085
          _CheckOSParams(self, False, [self.cfg.GetMasterNode()],
3086
                         os_name, self.new_osp[os_name])
3087

    
3088
    # changes to the hypervisor list
3089
    if self.op.enabled_hypervisors is not None:
3090
      self.hv_list = self.op.enabled_hypervisors
3091
      for hv in self.hv_list:
3092
        # if the hypervisor doesn't already exist in the cluster
3093
        # hvparams, we initialize it to empty, and then (in both
3094
        # cases) we make sure to fill the defaults, as we might not
3095
        # have a complete defaults list if the hypervisor wasn't
3096
        # enabled before
3097
        if hv not in new_hvp:
3098
          new_hvp[hv] = {}
3099
        new_hvp[hv] = objects.FillDict(constants.HVC_DEFAULTS[hv], new_hvp[hv])
3100
        utils.ForceDictType(new_hvp[hv], constants.HVS_PARAMETER_TYPES)
3101
    else:
3102
      self.hv_list = cluster.enabled_hypervisors
3103

    
3104
    if self.op.hvparams or self.op.enabled_hypervisors is not None:
3105
      # either the enabled list has changed, or the parameters have, validate
3106
      for hv_name, hv_params in self.new_hvparams.items():
3107
        if ((self.op.hvparams and hv_name in self.op.hvparams) or
3108
            (self.op.enabled_hypervisors and
3109
             hv_name in self.op.enabled_hypervisors)):
3110
          # either this is a new hypervisor, or its parameters have changed
3111
          hv_class = hypervisor.GetHypervisor(hv_name)
3112
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3113
          hv_class.CheckParameterSyntax(hv_params)
3114
          _CheckHVParams(self, node_list, hv_name, hv_params)
3115

    
3116
    if self.op.os_hvp:
3117
      # no need to check any newly-enabled hypervisors, since the
3118
      # defaults have already been checked in the above code-block
3119
      for os_name, os_hvp in self.new_os_hvp.items():
3120
        for hv_name, hv_params in os_hvp.items():
3121
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3122
          # we need to fill in the new os_hvp on top of the actual hv_p
3123
          cluster_defaults = self.new_hvparams.get(hv_name, {})
3124
          new_osp = objects.FillDict(cluster_defaults, hv_params)
3125
          hv_class = hypervisor.GetHypervisor(hv_name)
3126
          hv_class.CheckParameterSyntax(new_osp)
3127
          _CheckHVParams(self, node_list, hv_name, new_osp)
3128

    
3129
    if self.op.default_iallocator:
3130
      alloc_script = utils.FindFile(self.op.default_iallocator,
3131
                                    constants.IALLOCATOR_SEARCH_PATH,
3132
                                    os.path.isfile)
3133
      if alloc_script is None:
3134
        raise errors.OpPrereqError("Invalid default iallocator script '%s'"
3135
                                   " specified" % self.op.default_iallocator,
3136
                                   errors.ECODE_INVAL)
3137

    
3138
  def Exec(self, feedback_fn):
3139
    """Change the parameters of the cluster.
3140

3141
    """
3142
    if self.op.vg_name is not None:
3143
      new_volume = self.op.vg_name
3144
      if not new_volume:
3145
        new_volume = None
3146
      if new_volume != self.cfg.GetVGName():
3147
        self.cfg.SetVGName(new_volume)
3148
      else:
3149
        feedback_fn("Cluster LVM configuration already in desired"
3150
                    " state, not changing")
3151
    if self.op.drbd_helper is not None:
3152
      new_helper = self.op.drbd_helper
3153
      if not new_helper:
3154
        new_helper = None
3155
      if new_helper != self.cfg.GetDRBDHelper():
3156
        self.cfg.SetDRBDHelper(new_helper)
3157
      else:
3158
        feedback_fn("Cluster DRBD helper already in desired state,"
3159
                    " not changing")
3160
    if self.op.hvparams:
3161
      self.cluster.hvparams = self.new_hvparams
3162
    if self.op.os_hvp:
3163
      self.cluster.os_hvp = self.new_os_hvp
3164
    if self.op.enabled_hypervisors is not None:
3165
      self.cluster.hvparams = self.new_hvparams
3166
      self.cluster.enabled_hypervisors = self.op.enabled_hypervisors
3167
    if self.op.beparams:
3168
      self.cluster.beparams[constants.PP_DEFAULT] = self.new_beparams
3169
    if self.op.nicparams:
3170
      self.cluster.nicparams[constants.PP_DEFAULT] = self.new_nicparams
3171
    if self.op.osparams:
3172
      self.cluster.osparams = self.new_osp
3173
    if self.op.ndparams:
3174
      self.cluster.ndparams = self.new_ndparams
3175

    
3176
    if self.op.candidate_pool_size is not None:
3177
      self.cluster.candidate_pool_size = self.op.candidate_pool_size
3178
      # we need to update the pool size here, otherwise the save will fail
3179
      _AdjustCandidatePool(self, [])
3180

    
3181
    if self.op.maintain_node_health is not None:
3182
      self.cluster.maintain_node_health = self.op.maintain_node_health
3183

    
3184
    if self.op.prealloc_wipe_disks is not None:
3185
      self.cluster.prealloc_wipe_disks = self.op.prealloc_wipe_disks
3186

    
3187
    if self.op.add_uids is not None:
3188
      uidpool.AddToUidPool(self.cluster.uid_pool, self.op.add_uids)
3189

    
3190
    if self.op.remove_uids is not None:
3191
      uidpool.RemoveFromUidPool(self.cluster.uid_pool, self.op.remove_uids)
3192

    
3193
    if self.op.uid_pool is not None:
3194
      self.cluster.uid_pool = self.op.uid_pool
3195

    
3196
    if self.op.default_iallocator is not None:
3197
      self.cluster.default_iallocator = self.op.default_iallocator
3198

    
3199
    if self.op.reserved_lvs is not None:
3200
      self.cluster.reserved_lvs = self.op.reserved_lvs
3201

    
3202
    def helper_os(aname, mods, desc):
3203
      desc += " OS list"
3204
      lst = getattr(self.cluster, aname)
3205
      for key, val in mods:
3206
        if key == constants.DDM_ADD:
3207
          if val in lst:
3208
            feedback_fn("OS %s already in %s, ignoring" % (val, desc))
3209
          else:
3210
            lst.append(val)
3211
        elif key == constants.DDM_REMOVE:
3212
          if val in lst:
3213
            lst.remove(val)
3214
          else:
3215
            feedback_fn("OS %s not found in %s, ignoring" % (val, desc))
3216
        else:
3217
          raise errors.ProgrammerError("Invalid modification '%s'" % key)
3218

    
3219
    if self.op.hidden_os:
3220
      helper_os("hidden_os", self.op.hidden_os, "hidden")
3221

    
3222
    if self.op.blacklisted_os:
3223
      helper_os("blacklisted_os", self.op.blacklisted_os, "blacklisted")
3224

    
3225
    if self.op.master_netdev:
3226
      master = self.cfg.GetMasterNode()
3227
      feedback_fn("Shutting down master ip on the current netdev (%s)" %
3228
                  self.cluster.master_netdev)
3229
      result = self.rpc.call_node_stop_master(master, False)
3230
      result.Raise("Could not disable the master ip")
3231
      feedback_fn("Changing master_netdev from %s to %s" %
3232
                  (self.cluster.master_netdev, self.op.master_netdev))
3233
      self.cluster.master_netdev = self.op.master_netdev
3234

    
3235
    self.cfg.Update(self.cluster, feedback_fn)
3236

    
3237
    if self.op.master_netdev:
3238
      feedback_fn("Starting the master ip on the new master netdev (%s)" %
3239
                  self.op.master_netdev)
3240
      result = self.rpc.call_node_start_master(master, False, False)
3241
      if result.fail_msg:
3242
        self.LogWarning("Could not re-enable the master ip on"
3243
                        " the master, please restart manually: %s",
3244
                        result.fail_msg)
3245

    
3246

    
3247
def _UploadHelper(lu, nodes, fname):
3248
  """Helper for uploading a file and showing warnings.
3249

3250
  """
3251
  if os.path.exists(fname):
3252
    result = lu.rpc.call_upload_file(nodes, fname)
3253
    for to_node, to_result in result.items():
3254
      msg = to_result.fail_msg
3255
      if msg:
3256
        msg = ("Copy of file %s to node %s failed: %s" %
3257
               (fname, to_node, msg))
3258
        lu.proc.LogWarning(msg)
3259

    
3260

    
3261
def _ComputeAncillaryFiles(cluster, redist):
3262
  """Compute files external to Ganeti which need to be consistent.
3263

3264
  @type redist: boolean
3265
  @param redist: Whether to include files which need to be redistributed
3266

3267
  """
3268
  # Compute files for all nodes
3269
  files_all = set([
3270
    constants.SSH_KNOWN_HOSTS_FILE,
3271
    constants.CONFD_HMAC_KEY,
3272
    constants.CLUSTER_DOMAIN_SECRET_FILE,
3273
    ])
3274

    
3275
  if not redist:
3276
    files_all.update(constants.ALL_CERT_FILES)
3277
    files_all.update(ssconf.SimpleStore().GetFileList())
3278

    
3279
  if cluster.modify_etc_hosts:
3280
    files_all.add(constants.ETC_HOSTS)
3281

    
3282
  # Files which must either exist on all nodes or on none
3283
  files_all_opt = set([
3284
    constants.RAPI_USERS_FILE,
3285
    ])
3286

    
3287
  # Files which should only be on master candidates
3288
  files_mc = set()
3289
  if not redist:
3290
    files_mc.add(constants.CLUSTER_CONF_FILE)
3291

    
3292
  # Files which should only be on VM-capable nodes
3293
  files_vm = set(filename
3294
    for hv_name in cluster.enabled_hypervisors
3295
    for filename in hypervisor.GetHypervisor(hv_name).GetAncillaryFiles())
3296

    
3297
  # Filenames must be unique
3298
  assert (len(files_all | files_all_opt | files_mc | files_vm) ==
3299
          sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
3300
         "Found file listed in more than one file list"
3301

    
3302
  return (files_all, files_all_opt, files_mc, files_vm)
3303

    
3304

    
3305
def _RedistributeAncillaryFiles(lu, additional_nodes=None, additional_vm=True):
3306
  """Distribute additional files which are part of the cluster configuration.
3307

3308
  ConfigWriter takes care of distributing the config and ssconf files, but
3309
  there are more files which should be distributed to all nodes. This function
3310
  makes sure those are copied.
3311

3312
  @param lu: calling logical unit
3313
  @param additional_nodes: list of nodes not in the config to distribute to
3314
  @type additional_vm: boolean
3315
  @param additional_vm: whether the additional nodes are vm-capable or not
3316

3317
  """
3318
  # Gather target nodes
3319
  cluster = lu.cfg.GetClusterInfo()
3320
  master_info = lu.cfg.GetNodeInfo(lu.cfg.GetMasterNode())
3321

    
3322
  online_nodes = lu.cfg.GetOnlineNodeList()
3323
  vm_nodes = lu.cfg.GetVmCapableNodeList()
3324

    
3325
  if additional_nodes is not None:
3326
    online_nodes.extend(additional_nodes)
3327
    if additional_vm:
3328
      vm_nodes.extend(additional_nodes)
3329

    
3330
  # Never distribute to master node
3331
  for nodelist in [online_nodes, vm_nodes]:
3332
    if master_info.name in nodelist:
3333
      nodelist.remove(master_info.name)
3334

    
3335
  # Gather file lists
3336
  (files_all, files_all_opt, files_mc, files_vm) = \
3337
    _ComputeAncillaryFiles(cluster, True)
3338

    
3339
  # Never re-distribute configuration file from here
3340
  assert not (constants.CLUSTER_CONF_FILE in files_all or
3341
              constants.CLUSTER_CONF_FILE in files_vm)
3342
  assert not files_mc, "Master candidates not handled in this function"
3343

    
3344
  filemap = [
3345
    (online_nodes, files_all),
3346
    (online_nodes, files_all_opt),
3347
    (vm_nodes, files_vm),
3348
    ]
3349

    
3350
  # Upload the files
3351
  for (node_list, files) in filemap:
3352
    for fname in files:
3353
      _UploadHelper(lu, node_list, fname)
3354

    
3355

    
3356
class LUClusterRedistConf(NoHooksLU):
3357
  """Force the redistribution of cluster configuration.
3358

3359
  This is a very simple LU.
3360

3361
  """
3362
  REQ_BGL = False
3363

    
3364
  def ExpandNames(self):
3365
    self.needed_locks = {
3366
      locking.LEVEL_NODE: locking.ALL_SET,
3367
    }
3368
    self.share_locks[locking.LEVEL_NODE] = 1
3369

    
3370
  def Exec(self, feedback_fn):
3371
    """Redistribute the configuration.
3372

3373
    """
3374
    self.cfg.Update(self.cfg.GetClusterInfo(), feedback_fn)
3375
    _RedistributeAncillaryFiles(self)
3376

    
3377

    
3378
def _WaitForSync(lu, instance, disks=None, oneshot=False):
3379
  """Sleep and poll for an instance's disk to sync.
3380

3381
  """
3382
  if not instance.disks or disks is not None and not disks:
3383
    return True
3384

    
3385
  disks = _ExpandCheckDisks(instance, disks)
3386

    
3387
  if not oneshot:
3388
    lu.proc.LogInfo("Waiting for instance %s to sync disks." % instance.name)
3389

    
3390
  node = instance.primary_node
3391

    
3392
  for dev in disks:
3393
    lu.cfg.SetDiskID(dev, node)
3394

    
3395
  # TODO: Convert to utils.Retry
3396

    
3397
  retries = 0
3398
  degr_retries = 10 # in seconds, as we sleep 1 second each time
3399
  while True:
3400
    max_time = 0
3401
    done = True
3402
    cumul_degraded = False
3403
    rstats = lu.rpc.call_blockdev_getmirrorstatus(node, disks)
3404
    msg = rstats.fail_msg
3405
    if msg:
3406
      lu.LogWarning("Can't get any data from node %s: %s", node, msg)
3407
      retries += 1
3408
      if retries >= 10:
3409
        raise errors.RemoteError("Can't contact node %s for mirror data,"
3410
                                 " aborting." % node)
3411
      time.sleep(6)
3412
      continue
3413
    rstats = rstats.payload
3414
    retries = 0
3415
    for i, mstat in enumerate(rstats):
3416
      if mstat is None:
3417
        lu.LogWarning("Can't compute data for node %s/%s",
3418
                           node, disks[i].iv_name)
3419
        continue
3420

    
3421
      cumul_degraded = (cumul_degraded or
3422
                        (mstat.is_degraded and mstat.sync_percent is None))
3423
      if mstat.sync_percent is not None:
3424
        done = False
3425
        if mstat.estimated_time is not None:
3426
          rem_time = ("%s remaining (estimated)" %
3427
                      utils.FormatSeconds(mstat.estimated_time))
3428
          max_time = mstat.estimated_time
3429
        else:
3430
          rem_time = "no time estimate"
3431
        lu.proc.LogInfo("- device %s: %5.2f%% done, %s" %
3432
                        (disks[i].iv_name, mstat.sync_percent, rem_time))
3433

    
3434
    # if we're done but degraded, let's do a few small retries, to
3435
    # make sure we see a stable and not transient situation; therefore
3436
    # we force restart of the loop
3437
    if (done or oneshot) and cumul_degraded and degr_retries > 0:
3438
      logging.info("Degraded disks found, %d retries left", degr_retries)
3439
      degr_retries -= 1
3440
      time.sleep(1)
3441
      continue
3442

    
3443
    if done or oneshot:
3444
      break
3445

    
3446
    time.sleep(min(60, max_time))
3447

    
3448
  if done:
3449
    lu.proc.LogInfo("Instance %s's disks are in sync." % instance.name)
3450
  return not cumul_degraded
3451

    
3452

    
3453
def _CheckDiskConsistency(lu, dev, node, on_primary, ldisk=False):
3454
  """Check that mirrors are not degraded.
3455

3456
  The ldisk parameter, if True, will change the test from the
3457
  is_degraded attribute (which represents overall non-ok status for
3458
  the device(s)) to the ldisk (representing the local storage status).
3459

3460
  """
3461
  lu.cfg.SetDiskID(dev, node)
3462

    
3463
  result = True
3464

    
3465
  if on_primary or dev.AssembleOnSecondary():
3466
    rstats = lu.rpc.call_blockdev_find(node, dev)
3467
    msg = rstats.fail_msg
3468
    if msg:
3469
      lu.LogWarning("Can't find disk on node %s: %s", node, msg)
3470
      result = False
3471
    elif not rstats.payload:
3472
      lu.LogWarning("Can't find disk on node %s", node)
3473
      result = False
3474
    else:
3475
      if ldisk:
3476
        result = result and rstats.payload.ldisk_status == constants.LDS_OKAY
3477
      else:
3478
        result = result and not rstats.payload.is_degraded
3479

    
3480
  if dev.children:
3481
    for child in dev.children:
3482
      result = result and _CheckDiskConsistency(lu, child, node, on_primary)
3483

    
3484
  return result
3485

    
3486

    
3487
class LUOobCommand(NoHooksLU):
3488
  """Logical unit for OOB handling.
3489

3490
  """
3491
  REG_BGL = False
3492
  _SKIP_MASTER = (constants.OOB_POWER_OFF, constants.OOB_POWER_CYCLE)
3493

    
3494
  def ExpandNames(self):
3495
    """Gather locks we need.
3496

3497
    """
3498
    if self.op.node_names:
3499
      self.op.node_names = _GetWantedNodes(self, self.op.node_names)
3500
      lock_names = self.op.node_names
3501
    else:
3502
      lock_names = locking.ALL_SET
3503

    
3504
    self.needed_locks = {
3505
      locking.LEVEL_NODE: lock_names,
3506
      }
3507

    
3508
  def CheckPrereq(self):
3509
    """Check prerequisites.
3510

3511
    This checks:
3512
     - the node exists in the configuration
3513
     - OOB is supported
3514

3515
    Any errors are signaled by raising errors.OpPrereqError.
3516

3517
    """
3518
    self.nodes = []
3519
    self.master_node = self.cfg.GetMasterNode()
3520

    
3521
    assert self.op.power_delay >= 0.0
3522

    
3523
    if self.op.node_names:
3524
      if (self.op.command in self._SKIP_MASTER and
3525
          self.master_node in self.op.node_names):
3526
        master_node_obj = self.cfg.GetNodeInfo(self.master_node)
3527
        master_oob_handler = _SupportsOob(self.cfg, master_node_obj)
3528

    
3529
        if master_oob_handler:
3530
          additional_text = ("run '%s %s %s' if you want to operate on the"
3531
                             " master regardless") % (master_oob_handler,
3532
                                                      self.op.command,
3533
                                                      self.master_node)
3534
        else:
3535
          additional_text = "it does not support out-of-band operations"
3536

    
3537
        raise errors.OpPrereqError(("Operating on the master node %s is not"
3538
                                    " allowed for %s; %s") %
3539
                                   (self.master_node, self.op.command,
3540
                                    additional_text), errors.ECODE_INVAL)
3541
    else:
3542
      self.op.node_names = self.cfg.GetNodeList()
3543
      if self.op.command in self._SKIP_MASTER:
3544
        self.op.node_names.remove(self.master_node)
3545

    
3546
    if self.op.command in self._SKIP_MASTER:
3547
      assert self.master_node not in self.op.node_names
3548

    
3549
    for node_name in self.op.node_names:
3550
      node = self.cfg.GetNodeInfo(node_name)
3551

    
3552
      if node is None:
3553
        raise errors.OpPrereqError("Node %s not found" % node_name,
3554
                                   errors.ECODE_NOENT)
3555
      else:
3556
        self.nodes.append(node)
3557

    
3558
      if (not self.op.ignore_status and
3559
          (self.op.command == constants.OOB_POWER_OFF and not node.offline)):
3560
        raise errors.OpPrereqError(("Cannot power off node %s because it is"
3561
                                    " not marked offline") % node_name,
3562
                                   errors.ECODE_STATE)
3563

    
3564
  def Exec(self, feedback_fn):
3565
    """Execute OOB and return result if we expect any.
3566

3567
    """
3568
    master_node = self.master_node
3569
    ret = []
3570

    
3571
    for idx, node in enumerate(utils.NiceSort(self.nodes,
3572
                                              key=lambda node: node.name)):
3573
      node_entry = [(constants.RS_NORMAL, node.name)]
3574
      ret.append(node_entry)
3575

    
3576
      oob_program = _SupportsOob(self.cfg, node)
3577

    
3578
      if not oob_program:
3579
        node_entry.append((constants.RS_UNAVAIL, None))
3580
        continue
3581

    
3582
      logging.info("Executing out-of-band command '%s' using '%s' on %s",
3583
                   self.op.command, oob_program, node.name)
3584
      result = self.rpc.call_run_oob(master_node, oob_program,
3585
                                     self.op.command, node.name,
3586
                                     self.op.timeout)
3587

    
3588
      if result.fail_msg:
3589
        self.LogWarning("Out-of-band RPC failed on node '%s': %s",
3590
                        node.name, result.fail_msg)
3591
        node_entry.append((constants.RS_NODATA, None))
3592
      else:
3593
        try:
3594
          self._CheckPayload(result)
3595
        except errors.OpExecError, err:
3596
          self.LogWarning("Payload returned by node '%s' is not valid: %s",
3597
                          node.name, err)
3598
          node_entry.append((constants.RS_NODATA, None))
3599
        else:
3600
          if self.op.command == constants.OOB_HEALTH:
3601
            # For health we should log important events
3602
            for item, status in result.payload:
3603
              if status in [constants.OOB_STATUS_WARNING,
3604
                            constants.OOB_STATUS_CRITICAL]:
3605
                self.LogWarning("Item '%s' on node '%s' has status '%s'",
3606
                                item, node.name, status)
3607

    
3608
          if self.op.command == constants.OOB_POWER_ON:
3609
            node.powered = True
3610
          elif self.op.command == constants.OOB_POWER_OFF:
3611
            node.powered = False
3612
          elif self.op.command == constants.OOB_POWER_STATUS:
3613
            powered = result.payload[constants.OOB_POWER_STATUS_POWERED]
3614
            if powered != node.powered:
3615
              logging.warning(("Recorded power state (%s) of node '%s' does not"
3616
                               " match actual power state (%s)"), node.powered,
3617
                              node.name, powered)
3618

    
3619
          # For configuration changing commands we should update the node
3620
          if self.op.command in (constants.OOB_POWER_ON,
3621
                                 constants.OOB_POWER_OFF):
3622
            self.cfg.Update(node, feedback_fn)
3623

    
3624
          node_entry.append((constants.RS_NORMAL, result.payload))
3625

    
3626
          if (self.op.command == constants.OOB_POWER_ON and
3627
              idx < len(self.nodes) - 1):
3628
            time.sleep(self.op.power_delay)
3629

    
3630
    return ret
3631

    
3632
  def _CheckPayload(self, result):
3633
    """Checks if the payload is valid.
3634

3635
    @param result: RPC result
3636
    @raises errors.OpExecError: If payload is not valid
3637

3638
    """
3639
    errs = []
3640
    if self.op.command == constants.OOB_HEALTH:
3641
      if not isinstance(result.payload, list):
3642
        errs.append("command 'health' is expected to return a list but got %s" %
3643
                    type(result.payload))
3644
      else:
3645
        for item, status in result.payload:
3646
          if status not in constants.OOB_STATUSES:
3647
            errs.append("health item '%s' has invalid status '%s'" %
3648
                        (item, status))
3649

    
3650
    if self.op.command == constants.OOB_POWER_STATUS:
3651
      if not isinstance(result.payload, dict):
3652
        errs.append("power-status is expected to return a dict but got %s" %
3653
                    type(result.payload))
3654

    
3655
    if self.op.command in [
3656
        constants.OOB_POWER_ON,
3657
        constants.OOB_POWER_OFF,
3658
        constants.OOB_POWER_CYCLE,
3659
        ]:
3660
      if result.payload is not None:
3661
        errs.append("%s is expected to not return payload but got '%s'" %
3662
                    (self.op.command, result.payload))
3663

    
3664
    if errs:
3665
      raise errors.OpExecError("Check of out-of-band payload failed due to %s" %
3666
                               utils.CommaJoin(errs))
3667

    
3668
class _OsQuery(_QueryBase):
3669
  FIELDS = query.OS_FIELDS
3670

    
3671
  def ExpandNames(self, lu):
3672
    # Lock all nodes in shared mode
3673
    # Temporary removal of locks, should be reverted later
3674
    # TODO: reintroduce locks when they are lighter-weight
3675
    lu.needed_locks = {}
3676
    #self.share_locks[locking.LEVEL_NODE] = 1
3677
    #self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET
3678

    
3679
    # The following variables interact with _QueryBase._GetNames
3680
    if self.names:
3681
      self.wanted = self.names
3682
    else:
3683
      self.wanted = locking.ALL_SET
3684

    
3685
    self.do_locking = self.use_locking
3686

    
3687
  def DeclareLocks(self, lu, level):
3688
    pass
3689

    
3690
  @staticmethod
3691
  def _DiagnoseByOS(rlist):
3692
    """Remaps a per-node return list into an a per-os per-node dictionary
3693

3694
    @param rlist: a map with node names as keys and OS objects as values
3695

3696
    @rtype: dict
3697
    @return: a dictionary with osnames as keys and as value another
3698
        map, with nodes as keys and tuples of (path, status, diagnose,
3699
        variants, parameters, api_versions) as values, eg::
3700

3701
          {"debian-etch": {"node1": [(/usr/lib/..., True, "", [], []),
3702
                                     (/srv/..., False, "invalid api")],
3703
                           "node2": [(/srv/..., True, "", [], [])]}
3704
          }
3705

3706
    """
3707
    all_os = {}
3708
    # we build here the list of nodes that didn't fail the RPC (at RPC
3709
    # level), so that nodes with a non-responding node daemon don't
3710
    # make all OSes invalid
3711
    good_nodes = [node_name for node_name in rlist
3712
                  if not rlist[node_name].fail_msg]
3713
    for node_name, nr in rlist.items():
3714
      if nr.fail_msg or not nr.payload:
3715
        continue
3716
      for (name, path, status, diagnose, variants,
3717
           params, api_versions) in nr.payload:
3718
        if name not in all_os:
3719
          # build a list of nodes for this os containing empty lists
3720
          # for each node in node_list
3721
          all_os[name] = {}
3722
          for nname in good_nodes:
3723
            all_os[name][nname] = []
3724
        # convert params from [name, help] to (name, help)
3725
        params = [tuple(v) for v in params]
3726
        all_os[name][node_name].append((path, status, diagnose,
3727
                                        variants, params, api_versions))
3728
    return all_os
3729

    
3730
  def _GetQueryData(self, lu):
3731
    """Computes the list of nodes and their attributes.
3732

3733
    """
3734
    # Locking is not used
3735
    assert not (compat.any(lu.glm.is_owned(level)
3736
                           for level in locking.LEVELS
3737
                           if level != locking.LEVEL_CLUSTER) or
3738
                self.do_locking or self.use_locking)
3739

    
3740
    valid_nodes = [node.name
3741
                   for node in lu.cfg.GetAllNodesInfo().values()
3742
                   if not node.offline and node.vm_capable]
3743
    pol = self._DiagnoseByOS(lu.rpc.call_os_diagnose(valid_nodes))
3744
    cluster = lu.cfg.GetClusterInfo()
3745

    
3746
    data = {}
3747

    
3748
    for (os_name, os_data) in pol.items():
3749
      info = query.OsInfo(name=os_name, valid=True, node_status=os_data,
3750
                          hidden=(os_name in cluster.hidden_os),
3751
                          blacklisted=(os_name in cluster.blacklisted_os))
3752

    
3753
      variants = set()
3754
      parameters = set()
3755
      api_versions = set()
3756

    
3757
      for idx, osl in enumerate(os_data.values()):
3758
        info.valid = bool(info.valid and osl and osl[0][1])
3759
        if not info.valid:
3760
          break
3761

    
3762
        (node_variants, node_params, node_api) = osl[0][3:6]
3763
        if idx == 0:
3764
          # First entry
3765
          variants.update(node_variants)
3766
          parameters.update(node_params)
3767
          api_versions.update(node_api)
3768
        else:
3769
          # Filter out inconsistent values
3770
          variants.intersection_update(node_variants)
3771
          parameters.intersection_update(node_params)
3772
          api_versions.intersection_update(node_api)
3773

    
3774
      info.variants = list(variants)
3775
      info.parameters = list(parameters)
3776
      info.api_versions = list(api_versions)
3777

    
3778
      data[os_name] = info
3779

    
3780
    # Prepare data in requested order
3781
    return [data[name] for name in self._GetNames(lu, pol.keys(), None)
3782
            if name in data]
3783

    
3784

    
3785
class LUOsDiagnose(NoHooksLU):
3786
  """Logical unit for OS diagnose/query.
3787

3788
  """
3789
  REQ_BGL = False
3790

    
3791
  @staticmethod
3792
  def _BuildFilter(fields, names):
3793
    """Builds a filter for querying OSes.
3794

3795
    """
3796
    name_filter = qlang.MakeSimpleFilter("name", names)
3797

    
3798
    # Legacy behaviour: Hide hidden, blacklisted or invalid OSes if the
3799
    # respective field is not requested
3800
    status_filter = [[qlang.OP_NOT, [qlang.OP_TRUE, fname]]
3801
                     for fname in ["hidden", "blacklisted"]
3802
                     if fname not in fields]
3803
    if "valid" not in fields:
3804
      status_filter.append([qlang.OP_TRUE, "valid"])
3805

    
3806
    if status_filter:
3807
      status_filter.insert(0, qlang.OP_AND)
3808
    else:
3809
      status_filter = None
3810

    
3811
    if name_filter and status_filter:
3812
      return [qlang.OP_AND, name_filter, status_filter]
3813
    elif name_filter:
3814
      return name_filter
3815
    else:
3816
      return status_filter
3817

    
3818
  def CheckArguments(self):
3819
    self.oq = _OsQuery(self._BuildFilter(self.op.output_fields, self.op.names),
3820
                       self.op.output_fields, False)
3821

    
3822
  def ExpandNames(self):
3823
    self.oq.ExpandNames(self)
3824

    
3825
  def Exec(self, feedback_fn):
3826
    return self.oq.OldStyleQuery(self)
3827

    
3828

    
3829
class LUNodeRemove(LogicalUnit):
3830
  """Logical unit for removing a node.
3831

3832
  """
3833
  HPATH = "node-remove"
3834
  HTYPE = constants.HTYPE_NODE
3835

    
3836
  def BuildHooksEnv(self):
3837
    """Build hooks env.
3838

3839
    This doesn't run on the target node in the pre phase as a failed
3840
    node would then be impossible to remove.
3841

3842
    """
3843
    return {
3844
      "OP_TARGET": self.op.node_name,
3845
      "NODE_NAME": self.op.node_name,
3846
      }
3847

    
3848
  def BuildHooksNodes(self):
3849
    """Build hooks nodes.
3850

3851
    """
3852
    all_nodes = self.cfg.GetNodeList()
3853
    try:
3854
      all_nodes.remove(self.op.node_name)
3855
    except ValueError:
3856
      logging.warning("Node '%s', which is about to be removed, was not found"
3857
                      " in the list of all nodes", self.op.node_name)
3858
    return (all_nodes, all_nodes)
3859

    
3860
  def CheckPrereq(self):
3861
    """Check prerequisites.
3862

3863
    This checks:
3864
     - the node exists in the configuration
3865
     - it does not have primary or secondary instances
3866
     - it's not the master
3867

3868
    Any errors are signaled by raising errors.OpPrereqError.
3869

3870
    """
3871
    self.op.node_name = _ExpandNodeName(self.cfg, self.op.node_name)
3872
    node = self.cfg.GetNodeInfo(self.op.node_name)
3873
    assert node is not None
3874

    
3875
    instance_list = self.cfg.GetInstanceList()
3876

    
3877
    masternode = self.cfg.GetMasterNode()
3878
    if node.name == masternode:
3879
      raise errors.OpPrereqError("Node is the master node, failover to another"
3880
                                 " node is required", errors.ECODE_INVAL)
3881

    
3882
    for instance_name in instance_list:
3883
      instance = self.cfg.GetInstanceInfo(instance_name)
3884
      if node.name in instance.all_nodes:
3885
        raise errors.OpPrereqError("Instance %s is still running on the node,"
3886
                                   " please remove first" % instance_name,
3887
                                   errors.ECODE_INVAL)
3888
    self.op.node_name = node.name
3889
    self.node = node
3890

    
3891
  def Exec(self, feedback_fn):
3892
    """Removes the node from the cluster.
3893

3894
    """
3895
    node = self.node
3896
    logging.info("Stopping the node daemon and removing configs from node %s",
3897
                 node.name)
3898

    
3899
    modify_ssh_setup = self.cfg.GetClusterInfo().modify_ssh_setup
3900

    
3901
    # Promote nodes to master candidate as needed
3902
    _AdjustCandidatePool(self, exceptions=[node.name])
3903
    self.context.RemoveNode(node.name)
3904

    
3905
    # Run post hooks on the node before it's removed
3906
    _RunPostHook(self, node.name)
3907

    
3908
    result = self.rpc.call_node_leave_cluster(node.name, modify_ssh_setup)
3909
    msg = result.fail_msg
3910
    if msg:
3911
      self.LogWarning("Errors encountered on the remote node while leaving"
3912
                      " the cluster: %s", msg)
3913

    
3914
    # Remove node from our /etc/hosts
3915
    if self.cfg.GetClusterInfo().modify_etc_hosts:
3916
      master_node = self.cfg.GetMasterNode()
3917
      result = self.rpc.call_etc_hosts_modify(master_node,
3918
                                              constants.ETC_HOSTS_REMOVE,
3919
                                              node.name, None)
3920
      result.Raise("Can't update hosts file with new host data")
3921
      _RedistributeAncillaryFiles(self)
3922

    
3923

    
3924
class _NodeQuery(_QueryBase):
3925
  FIELDS = query.NODE_FIELDS
3926

    
3927
  def ExpandNames(self, lu):
3928
    lu.needed_locks = {}
3929
    lu.share_locks[locking.LEVEL_NODE] = 1
3930

    
3931
    if self.names:
3932
      self.wanted = _GetWantedNodes(lu, self.names)
3933
    else:
3934
      self.wanted = locking.ALL_SET
3935

    
3936
    self.do_locking = (self.use_locking and
3937
                       query.NQ_LIVE in self.requested_data)
3938

    
3939
    if self.do_locking:
3940
      # if we don't request only static fields, we need to lock the nodes
3941
      lu.needed_locks[locking.LEVEL_NODE] = self.wanted
3942

    
3943
  def DeclareLocks(self, lu, level):
3944
    pass
3945

    
3946
  def _GetQueryData(self, lu):
3947
    """Computes the list of nodes and their attributes.
3948

3949
    """
3950
    all_info = lu.cfg.GetAllNodesInfo()
3951

    
3952
    nodenames = self._GetNames(lu, all_info.keys(), locking.LEVEL_NODE)
3953

    
3954
    # Gather data as requested
3955
    if query.NQ_LIVE in self.requested_data:
3956
      # filter out non-vm_capable nodes
3957
      toquery_nodes = [name for name in nodenames if all_info[name].vm_capable]
3958

    
3959
      node_data = lu.rpc.call_node_info(toquery_nodes, lu.cfg.GetVGName(),
3960
                                        lu.cfg.GetHypervisorType())
3961
      live_data = dict((name, nresult.payload)
3962
                       for (name, nresult) in node_data.items()
3963
                       if not nresult.fail_msg and nresult.payload)
3964
    else:
3965
      live_data = None
3966

    
3967
    if query.NQ_INST in self.requested_data:
3968
      node_to_primary = dict([(name, set()) for name in nodenames])
3969
      node_to_secondary = dict([(name, set()) for name in nodenames])
3970

    
3971
      inst_data = lu.cfg.GetAllInstancesInfo()
3972

    
3973
      for inst in inst_data.values():
3974
        if inst.primary_node in node_to_primary:
3975
          node_to_primary[inst.primary_node].add(inst.name)
3976
        for secnode in inst.secondary_nodes:
3977
          if secnode in node_to_secondary:
3978
            node_to_secondary[secnode].add(inst.name)
3979
    else:
3980
      node_to_primary = None
3981
      node_to_secondary = None
3982

    
3983
    if query.NQ_OOB in self.requested_data:
3984
      oob_support = dict((name, bool(_SupportsOob(lu.cfg, node)))
3985
                         for name, node in all_info.iteritems())
3986
    else:
3987
      oob_support = None
3988

    
3989
    if query.NQ_GROUP in self.requested_data:
3990
      groups = lu.cfg.GetAllNodeGroupsInfo()
3991
    else:
3992
      groups = {}
3993

    
3994
    return query.NodeQueryData([all_info[name] for name in nodenames],
3995
                               live_data, lu.cfg.GetMasterNode(),
3996
                               node_to_primary, node_to_secondary, groups,
3997
                               oob_support, lu.cfg.GetClusterInfo())
3998

    
3999

    
4000
class LUNodeQuery(NoHooksLU):
4001
  """Logical unit for querying nodes.
4002

4003
  """
4004
  # pylint: disable-msg=W0142
4005
  REQ_BGL = False
4006

    
4007
  def CheckArguments(self):
4008
    self.nq = _NodeQuery(qlang.MakeSimpleFilter("name", self.op.names),
4009
                         self.op.output_fields, self.op.use_locking)
4010

    
4011
  def ExpandNames(self):
4012
    self.nq.ExpandNames(self)
4013

    
4014
  def Exec(self, feedback_fn):
4015
    return self.nq.OldStyleQuery(self)
4016

    
4017

    
4018
class LUNodeQueryvols(NoHooksLU):
4019
  """Logical unit for getting volumes on node(s).
4020

4021
  """
4022
  REQ_BGL = False
4023
  _FIELDS_DYNAMIC = utils.FieldSet("phys", "vg", "name", "size", "instance")
4024
  _FIELDS_STATIC = utils.FieldSet("node")
4025

    
4026
  def CheckArguments(self):
4027
    _CheckOutputFields(static=self._FIELDS_STATIC,
4028
                       dynamic=self._FIELDS_DYNAMIC,
4029
                       selected=self.op.output_fields)
4030

    
4031
  def ExpandNames(self):
4032
    self.needed_locks = {}
4033
    self.share_locks[locking.LEVEL_NODE] = 1
4034
    if not self.op.nodes:
4035
      self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET
4036
    else:
4037
      self.needed_locks[locking.LEVEL_NODE] = \
4038
        _GetWantedNodes(self, self.op.nodes)
4039

    
4040
  def Exec(self, feedback_fn):
4041
    """Computes the list of nodes and their attributes.
4042

4043
    """
4044
    nodenames = self.glm.list_owned(locking.LEVEL_NODE)
4045
    volumes = self.rpc.call_node_volumes(nodenames)
4046

    
4047
    ilist = [self.cfg.GetInstanceInfo(iname) for iname
4048
             in self.cfg.GetInstanceList()]
4049

    
4050
    lv_by_node = dict([(inst, inst.MapLVsByNode()) for inst in ilist])
4051

    
4052
    output = []
4053
    for node in nodenames:
4054
      nresult = volumes[node]
4055
      if nresult.offline:
4056
        continue
4057
      msg = nresult.fail_msg
4058
      if msg:
4059
        self.LogWarning("Can't compute volume data on node %s: %s", node, msg)
4060
        continue
4061

    
4062
      node_vols = nresult.payload[:]
4063
      node_vols.sort(key=lambda vol: vol['dev'])
4064

    
4065
      for vol in node_vols:
4066
        node_output = []
4067
        for field in self.op.output_fields:
4068
          if field == "node":
4069
            val = node
4070
          elif field == "phys":
4071
            val = vol['dev']
4072
          elif field == "