Statistics
| Branch: | Tag: | Revision:

root / lib / cmdlib.py @ 73cdf9a3

History | View | Annotate | Download (431.4 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Module implementing the master-side code."""
23

    
24
# pylint: disable-msg=W0201,C0302
25

    
26
# W0201 since most LU attributes are defined in CheckPrereq or similar
27
# functions
28

    
29
# C0302: since we have waaaay to many lines in this module
30

    
31
import os
32
import os.path
33
import time
34
import re
35
import platform
36
import logging
37
import copy
38
import OpenSSL
39
import socket
40
import tempfile
41
import shutil
42
import itertools
43

    
44
from ganeti import ssh
45
from ganeti import utils
46
from ganeti import errors
47
from ganeti import hypervisor
48
from ganeti import locking
49
from ganeti import constants
50
from ganeti import objects
51
from ganeti import serializer
52
from ganeti import ssconf
53
from ganeti import uidpool
54
from ganeti import compat
55
from ganeti import masterd
56
from ganeti import netutils
57
from ganeti import query
58
from ganeti import qlang
59
from ganeti import opcodes
60

    
61
import ganeti.masterd.instance # pylint: disable-msg=W0611
62

    
63

    
64
def _SupportsOob(cfg, node):
65
  """Tells if node supports OOB.
66

67
  @type cfg: L{config.ConfigWriter}
68
  @param cfg: The cluster configuration
69
  @type node: L{objects.Node}
70
  @param node: The node
71
  @return: The OOB script if supported or an empty string otherwise
72

73
  """
74
  return cfg.GetNdParams(node)[constants.ND_OOB_PROGRAM]
75

    
76

    
77
class ResultWithJobs:
78
  """Data container for LU results with jobs.
79

80
  Instances of this class returned from L{LogicalUnit.Exec} will be recognized
81
  by L{mcpu.Processor._ProcessResult}. The latter will then submit the jobs
82
  contained in the C{jobs} attribute and include the job IDs in the opcode
83
  result.
84

85
  """
86
  def __init__(self, jobs, **kwargs):
87
    """Initializes this class.
88

89
    Additional return values can be specified as keyword arguments.
90

91
    @type jobs: list of lists of L{opcode.OpCode}
92
    @param jobs: A list of lists of opcode objects
93

94
    """
95
    self.jobs = jobs
96
    self.other = kwargs
97

    
98

    
99
class LogicalUnit(object):
100
  """Logical Unit base class.
101

102
  Subclasses must follow these rules:
103
    - implement ExpandNames
104
    - implement CheckPrereq (except when tasklets are used)
105
    - implement Exec (except when tasklets are used)
106
    - implement BuildHooksEnv
107
    - implement BuildHooksNodes
108
    - redefine HPATH and HTYPE
109
    - optionally redefine their run requirements:
110
        REQ_BGL: the LU needs to hold the Big Ganeti Lock exclusively
111

112
  Note that all commands require root permissions.
113

114
  @ivar dry_run_result: the value (if any) that will be returned to the caller
115
      in dry-run mode (signalled by opcode dry_run parameter)
116

117
  """
118
  HPATH = None
119
  HTYPE = None
120
  REQ_BGL = True
121

    
122
  def __init__(self, processor, op, context, rpc):
123
    """Constructor for LogicalUnit.
124

125
    This needs to be overridden in derived classes in order to check op
126
    validity.
127

128
    """
129
    self.proc = processor
130
    self.op = op
131
    self.cfg = context.cfg
132
    self.glm = context.glm
133
    self.context = context
134
    self.rpc = rpc
135
    # Dicts used to declare locking needs to mcpu
136
    self.needed_locks = None
137
    self.share_locks = dict.fromkeys(locking.LEVELS, 0)
138
    self.add_locks = {}
139
    self.remove_locks = {}
140
    # Used to force good behavior when calling helper functions
141
    self.recalculate_locks = {}
142
    # logging
143
    self.Log = processor.Log # pylint: disable-msg=C0103
144
    self.LogWarning = processor.LogWarning # pylint: disable-msg=C0103
145
    self.LogInfo = processor.LogInfo # pylint: disable-msg=C0103
146
    self.LogStep = processor.LogStep # pylint: disable-msg=C0103
147
    # support for dry-run
148
    self.dry_run_result = None
149
    # support for generic debug attribute
150
    if (not hasattr(self.op, "debug_level") or
151
        not isinstance(self.op.debug_level, int)):
152
      self.op.debug_level = 0
153

    
154
    # Tasklets
155
    self.tasklets = None
156

    
157
    # Validate opcode parameters and set defaults
158
    self.op.Validate(True)
159

    
160
    self.CheckArguments()
161

    
162
  def CheckArguments(self):
163
    """Check syntactic validity for the opcode arguments.
164

165
    This method is for doing a simple syntactic check and ensure
166
    validity of opcode parameters, without any cluster-related
167
    checks. While the same can be accomplished in ExpandNames and/or
168
    CheckPrereq, doing these separate is better because:
169

170
      - ExpandNames is left as as purely a lock-related function
171
      - CheckPrereq is run after we have acquired locks (and possible
172
        waited for them)
173

174
    The function is allowed to change the self.op attribute so that
175
    later methods can no longer worry about missing parameters.
176

177
    """
178
    pass
179

    
180
  def ExpandNames(self):
181
    """Expand names for this LU.
182

183
    This method is called before starting to execute the opcode, and it should
184
    update all the parameters of the opcode to their canonical form (e.g. a
185
    short node name must be fully expanded after this method has successfully
186
    completed). This way locking, hooks, logging, etc. can work correctly.
187

188
    LUs which implement this method must also populate the self.needed_locks
189
    member, as a dict with lock levels as keys, and a list of needed lock names
190
    as values. Rules:
191

192
      - use an empty dict if you don't need any lock
193
      - if you don't need any lock at a particular level omit that level
194
      - don't put anything for the BGL level
195
      - if you want all locks at a level use locking.ALL_SET as a value
196

197
    If you need to share locks (rather than acquire them exclusively) at one
198
    level you can modify self.share_locks, setting a true value (usually 1) for
199
    that level. By default locks are not shared.
200

201
    This function can also define a list of tasklets, which then will be
202
    executed in order instead of the usual LU-level CheckPrereq and Exec
203
    functions, if those are not defined by the LU.
204

205
    Examples::
206

207
      # Acquire all nodes and one instance
208
      self.needed_locks = {
209
        locking.LEVEL_NODE: locking.ALL_SET,
210
        locking.LEVEL_INSTANCE: ['instance1.example.com'],
211
      }
212
      # Acquire just two nodes
213
      self.needed_locks = {
214
        locking.LEVEL_NODE: ['node1.example.com', 'node2.example.com'],
215
      }
216
      # Acquire no locks
217
      self.needed_locks = {} # No, you can't leave it to the default value None
218

219
    """
220
    # The implementation of this method is mandatory only if the new LU is
221
    # concurrent, so that old LUs don't need to be changed all at the same
222
    # time.
223
    if self.REQ_BGL:
224
      self.needed_locks = {} # Exclusive LUs don't need locks.
225
    else:
226
      raise NotImplementedError
227

    
228
  def DeclareLocks(self, level):
229
    """Declare LU locking needs for a level
230

231
    While most LUs can just declare their locking needs at ExpandNames time,
232
    sometimes there's the need to calculate some locks after having acquired
233
    the ones before. This function is called just before acquiring locks at a
234
    particular level, but after acquiring the ones at lower levels, and permits
235
    such calculations. It can be used to modify self.needed_locks, and by
236
    default it does nothing.
237

238
    This function is only called if you have something already set in
239
    self.needed_locks for the level.
240

241
    @param level: Locking level which is going to be locked
242
    @type level: member of ganeti.locking.LEVELS
243

244
    """
245

    
246
  def CheckPrereq(self):
247
    """Check prerequisites for this LU.
248

249
    This method should check that the prerequisites for the execution
250
    of this LU are fulfilled. It can do internode communication, but
251
    it should be idempotent - no cluster or system changes are
252
    allowed.
253

254
    The method should raise errors.OpPrereqError in case something is
255
    not fulfilled. Its return value is ignored.
256

257
    This method should also update all the parameters of the opcode to
258
    their canonical form if it hasn't been done by ExpandNames before.
259

260
    """
261
    if self.tasklets is not None:
262
      for (idx, tl) in enumerate(self.tasklets):
263
        logging.debug("Checking prerequisites for tasklet %s/%s",
264
                      idx + 1, len(self.tasklets))
265
        tl.CheckPrereq()
266
    else:
267
      pass
268

    
269
  def Exec(self, feedback_fn):
270
    """Execute the LU.
271

272
    This method should implement the actual work. It should raise
273
    errors.OpExecError for failures that are somewhat dealt with in
274
    code, or expected.
275

276
    """
277
    if self.tasklets is not None:
278
      for (idx, tl) in enumerate(self.tasklets):
279
        logging.debug("Executing tasklet %s/%s", idx + 1, len(self.tasklets))
280
        tl.Exec(feedback_fn)
281
    else:
282
      raise NotImplementedError
283

    
284
  def BuildHooksEnv(self):
285
    """Build hooks environment for this LU.
286

287
    @rtype: dict
288
    @return: Dictionary containing the environment that will be used for
289
      running the hooks for this LU. The keys of the dict must not be prefixed
290
      with "GANETI_"--that'll be added by the hooks runner. The hooks runner
291
      will extend the environment with additional variables. If no environment
292
      should be defined, an empty dictionary should be returned (not C{None}).
293
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
294
      will not be called.
295

296
    """
297
    raise NotImplementedError
298

    
299
  def BuildHooksNodes(self):
300
    """Build list of nodes to run LU's hooks.
301

302
    @rtype: tuple; (list, list)
303
    @return: Tuple containing a list of node names on which the hook
304
      should run before the execution and a list of node names on which the
305
      hook should run after the execution. No nodes should be returned as an
306
      empty list (and not None).
307
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
308
      will not be called.
309

310
    """
311
    raise NotImplementedError
312

    
313
  def HooksCallBack(self, phase, hook_results, feedback_fn, lu_result):
314
    """Notify the LU about the results of its hooks.
315

316
    This method is called every time a hooks phase is executed, and notifies
317
    the Logical Unit about the hooks' result. The LU can then use it to alter
318
    its result based on the hooks.  By default the method does nothing and the
319
    previous result is passed back unchanged but any LU can define it if it
320
    wants to use the local cluster hook-scripts somehow.
321

322
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
323
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
324
    @param hook_results: the results of the multi-node hooks rpc call
325
    @param feedback_fn: function used send feedback back to the caller
326
    @param lu_result: the previous Exec result this LU had, or None
327
        in the PRE phase
328
    @return: the new Exec result, based on the previous result
329
        and hook results
330

331
    """
332
    # API must be kept, thus we ignore the unused argument and could
333
    # be a function warnings
334
    # pylint: disable-msg=W0613,R0201
335
    return lu_result
336

    
337
  def _ExpandAndLockInstance(self):
338
    """Helper function to expand and lock an instance.
339

340
    Many LUs that work on an instance take its name in self.op.instance_name
341
    and need to expand it and then declare the expanded name for locking. This
342
    function does it, and then updates self.op.instance_name to the expanded
343
    name. It also initializes needed_locks as a dict, if this hasn't been done
344
    before.
345

346
    """
347
    if self.needed_locks is None:
348
      self.needed_locks = {}
349
    else:
350
      assert locking.LEVEL_INSTANCE not in self.needed_locks, \
351
        "_ExpandAndLockInstance called with instance-level locks set"
352
    self.op.instance_name = _ExpandInstanceName(self.cfg,
353
                                                self.op.instance_name)
354
    self.needed_locks[locking.LEVEL_INSTANCE] = self.op.instance_name
355

    
356
  def _LockInstancesNodes(self, primary_only=False):
357
    """Helper function to declare instances' nodes for locking.
358

359
    This function should be called after locking one or more instances to lock
360
    their nodes. Its effect is populating self.needed_locks[locking.LEVEL_NODE]
361
    with all primary or secondary nodes for instances already locked and
362
    present in self.needed_locks[locking.LEVEL_INSTANCE].
363

364
    It should be called from DeclareLocks, and for safety only works if
365
    self.recalculate_locks[locking.LEVEL_NODE] is set.
366

367
    In the future it may grow parameters to just lock some instance's nodes, or
368
    to just lock primaries or secondary nodes, if needed.
369

370
    If should be called in DeclareLocks in a way similar to::
371

372
      if level == locking.LEVEL_NODE:
373
        self._LockInstancesNodes()
374

375
    @type primary_only: boolean
376
    @param primary_only: only lock primary nodes of locked instances
377

378
    """
379
    assert locking.LEVEL_NODE in self.recalculate_locks, \
380
      "_LockInstancesNodes helper function called with no nodes to recalculate"
381

    
382
    # TODO: check if we're really been called with the instance locks held
383

    
384
    # For now we'll replace self.needed_locks[locking.LEVEL_NODE], but in the
385
    # future we might want to have different behaviors depending on the value
386
    # of self.recalculate_locks[locking.LEVEL_NODE]
387
    wanted_nodes = []
388
    for instance_name in self.glm.list_owned(locking.LEVEL_INSTANCE):
389
      instance = self.context.cfg.GetInstanceInfo(instance_name)
390
      wanted_nodes.append(instance.primary_node)
391
      if not primary_only:
392
        wanted_nodes.extend(instance.secondary_nodes)
393

    
394
    if self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_REPLACE:
395
      self.needed_locks[locking.LEVEL_NODE] = wanted_nodes
396
    elif self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_APPEND:
397
      self.needed_locks[locking.LEVEL_NODE].extend(wanted_nodes)
398

    
399
    del self.recalculate_locks[locking.LEVEL_NODE]
400

    
401

    
402
class NoHooksLU(LogicalUnit): # pylint: disable-msg=W0223
403
  """Simple LU which runs no hooks.
404

405
  This LU is intended as a parent for other LogicalUnits which will
406
  run no hooks, in order to reduce duplicate code.
407

408
  """
409
  HPATH = None
410
  HTYPE = None
411

    
412
  def BuildHooksEnv(self):
413
    """Empty BuildHooksEnv for NoHooksLu.
414

415
    This just raises an error.
416

417
    """
418
    raise AssertionError("BuildHooksEnv called for NoHooksLUs")
419

    
420
  def BuildHooksNodes(self):
421
    """Empty BuildHooksNodes for NoHooksLU.
422

423
    """
424
    raise AssertionError("BuildHooksNodes called for NoHooksLU")
425

    
426

    
427
class Tasklet:
428
  """Tasklet base class.
429

430
  Tasklets are subcomponents for LUs. LUs can consist entirely of tasklets or
431
  they can mix legacy code with tasklets. Locking needs to be done in the LU,
432
  tasklets know nothing about locks.
433

434
  Subclasses must follow these rules:
435
    - Implement CheckPrereq
436
    - Implement Exec
437

438
  """
439
  def __init__(self, lu):
440
    self.lu = lu
441

    
442
    # Shortcuts
443
    self.cfg = lu.cfg
444
    self.rpc = lu.rpc
445

    
446
  def CheckPrereq(self):
447
    """Check prerequisites for this tasklets.
448

449
    This method should check whether the prerequisites for the execution of
450
    this tasklet are fulfilled. It can do internode communication, but it
451
    should be idempotent - no cluster or system changes are allowed.
452

453
    The method should raise errors.OpPrereqError in case something is not
454
    fulfilled. Its return value is ignored.
455

456
    This method should also update all parameters to their canonical form if it
457
    hasn't been done before.
458

459
    """
460
    pass
461

    
462
  def Exec(self, feedback_fn):
463
    """Execute the tasklet.
464

465
    This method should implement the actual work. It should raise
466
    errors.OpExecError for failures that are somewhat dealt with in code, or
467
    expected.
468

469
    """
470
    raise NotImplementedError
471

    
472

    
473
class _QueryBase:
474
  """Base for query utility classes.
475

476
  """
477
  #: Attribute holding field definitions
478
  FIELDS = None
479

    
480
  def __init__(self, filter_, fields, use_locking):
481
    """Initializes this class.
482

483
    """
484
    self.use_locking = use_locking
485

    
486
    self.query = query.Query(self.FIELDS, fields, filter_=filter_,
487
                             namefield="name")
488
    self.requested_data = self.query.RequestedData()
489
    self.names = self.query.RequestedNames()
490

    
491
    # Sort only if no names were requested
492
    self.sort_by_name = not self.names
493

    
494
    self.do_locking = None
495
    self.wanted = None
496

    
497
  def _GetNames(self, lu, all_names, lock_level):
498
    """Helper function to determine names asked for in the query.
499

500
    """
501
    if self.do_locking:
502
      names = lu.glm.list_owned(lock_level)
503
    else:
504
      names = all_names
505

    
506
    if self.wanted == locking.ALL_SET:
507
      assert not self.names
508
      # caller didn't specify names, so ordering is not important
509
      return utils.NiceSort(names)
510

    
511
    # caller specified names and we must keep the same order
512
    assert self.names
513
    assert not self.do_locking or lu.glm.is_owned(lock_level)
514

    
515
    missing = set(self.wanted).difference(names)
516
    if missing:
517
      raise errors.OpExecError("Some items were removed before retrieving"
518
                               " their data: %s" % missing)
519

    
520
    # Return expanded names
521
    return self.wanted
522

    
523
  def ExpandNames(self, lu):
524
    """Expand names for this query.
525

526
    See L{LogicalUnit.ExpandNames}.
527

528
    """
529
    raise NotImplementedError()
530

    
531
  def DeclareLocks(self, lu, level):
532
    """Declare locks for this query.
533

534
    See L{LogicalUnit.DeclareLocks}.
535

536
    """
537
    raise NotImplementedError()
538

    
539
  def _GetQueryData(self, lu):
540
    """Collects all data for this query.
541

542
    @return: Query data object
543

544
    """
545
    raise NotImplementedError()
546

    
547
  def NewStyleQuery(self, lu):
548
    """Collect data and execute query.
549

550
    """
551
    return query.GetQueryResponse(self.query, self._GetQueryData(lu),
552
                                  sort_by_name=self.sort_by_name)
553

    
554
  def OldStyleQuery(self, lu):
555
    """Collect data and execute query.
556

557
    """
558
    return self.query.OldStyleQuery(self._GetQueryData(lu),
559
                                    sort_by_name=self.sort_by_name)
560

    
561

    
562
def _GetWantedNodes(lu, nodes):
563
  """Returns list of checked and expanded node names.
564

565
  @type lu: L{LogicalUnit}
566
  @param lu: the logical unit on whose behalf we execute
567
  @type nodes: list
568
  @param nodes: list of node names or None for all nodes
569
  @rtype: list
570
  @return: the list of nodes, sorted
571
  @raise errors.ProgrammerError: if the nodes parameter is wrong type
572

573
  """
574
  if nodes:
575
    return [_ExpandNodeName(lu.cfg, name) for name in nodes]
576

    
577
  return utils.NiceSort(lu.cfg.GetNodeList())
578

    
579

    
580
def _GetWantedInstances(lu, instances):
581
  """Returns list of checked and expanded instance names.
582

583
  @type lu: L{LogicalUnit}
584
  @param lu: the logical unit on whose behalf we execute
585
  @type instances: list
586
  @param instances: list of instance names or None for all instances
587
  @rtype: list
588
  @return: the list of instances, sorted
589
  @raise errors.OpPrereqError: if the instances parameter is wrong type
590
  @raise errors.OpPrereqError: if any of the passed instances is not found
591

592
  """
593
  if instances:
594
    wanted = [_ExpandInstanceName(lu.cfg, name) for name in instances]
595
  else:
596
    wanted = utils.NiceSort(lu.cfg.GetInstanceList())
597
  return wanted
598

    
599

    
600
def _GetUpdatedParams(old_params, update_dict,
601
                      use_default=True, use_none=False):
602
  """Return the new version of a parameter dictionary.
603

604
  @type old_params: dict
605
  @param old_params: old parameters
606
  @type update_dict: dict
607
  @param update_dict: dict containing new parameter values, or
608
      constants.VALUE_DEFAULT to reset the parameter to its default
609
      value
610
  @param use_default: boolean
611
  @type use_default: whether to recognise L{constants.VALUE_DEFAULT}
612
      values as 'to be deleted' values
613
  @param use_none: boolean
614
  @type use_none: whether to recognise C{None} values as 'to be
615
      deleted' values
616
  @rtype: dict
617
  @return: the new parameter dictionary
618

619
  """
620
  params_copy = copy.deepcopy(old_params)
621
  for key, val in update_dict.iteritems():
622
    if ((use_default and val == constants.VALUE_DEFAULT) or
623
        (use_none and val is None)):
624
      try:
625
        del params_copy[key]
626
      except KeyError:
627
        pass
628
    else:
629
      params_copy[key] = val
630
  return params_copy
631

    
632

    
633
def _ReleaseLocks(lu, level, names=None, keep=None):
634
  """Releases locks owned by an LU.
635

636
  @type lu: L{LogicalUnit}
637
  @param level: Lock level
638
  @type names: list or None
639
  @param names: Names of locks to release
640
  @type keep: list or None
641
  @param keep: Names of locks to retain
642

643
  """
644
  assert not (keep is not None and names is not None), \
645
         "Only one of the 'names' and the 'keep' parameters can be given"
646

    
647
  if names is not None:
648
    should_release = names.__contains__
649
  elif keep:
650
    should_release = lambda name: name not in keep
651
  else:
652
    should_release = None
653

    
654
  if should_release:
655
    retain = []
656
    release = []
657

    
658
    # Determine which locks to release
659
    for name in lu.glm.list_owned(level):
660
      if should_release(name):
661
        release.append(name)
662
      else:
663
        retain.append(name)
664

    
665
    assert len(lu.glm.list_owned(level)) == (len(retain) + len(release))
666

    
667
    # Release just some locks
668
    lu.glm.release(level, names=release)
669

    
670
    assert frozenset(lu.glm.list_owned(level)) == frozenset(retain)
671
  else:
672
    # Release everything
673
    lu.glm.release(level)
674

    
675
    assert not lu.glm.is_owned(level), "No locks should be owned"
676

    
677

    
678
def _RunPostHook(lu, node_name):
679
  """Runs the post-hook for an opcode on a single node.
680

681
  """
682
  hm = lu.proc.hmclass(lu.rpc.call_hooks_runner, lu)
683
  try:
684
    hm.RunPhase(constants.HOOKS_PHASE_POST, nodes=[node_name])
685
  except:
686
    # pylint: disable-msg=W0702
687
    lu.LogWarning("Errors occurred running hooks on %s" % node_name)
688

    
689

    
690
def _CheckOutputFields(static, dynamic, selected):
691
  """Checks whether all selected fields are valid.
692

693
  @type static: L{utils.FieldSet}
694
  @param static: static fields set
695
  @type dynamic: L{utils.FieldSet}
696
  @param dynamic: dynamic fields set
697

698
  """
699
  f = utils.FieldSet()
700
  f.Extend(static)
701
  f.Extend(dynamic)
702

    
703
  delta = f.NonMatching(selected)
704
  if delta:
705
    raise errors.OpPrereqError("Unknown output fields selected: %s"
706
                               % ",".join(delta), errors.ECODE_INVAL)
707

    
708

    
709
def _CheckGlobalHvParams(params):
710
  """Validates that given hypervisor params are not global ones.
711

712
  This will ensure that instances don't get customised versions of
713
  global params.
714

715
  """
716
  used_globals = constants.HVC_GLOBALS.intersection(params)
717
  if used_globals:
718
    msg = ("The following hypervisor parameters are global and cannot"
719
           " be customized at instance level, please modify them at"
720
           " cluster level: %s" % utils.CommaJoin(used_globals))
721
    raise errors.OpPrereqError(msg, errors.ECODE_INVAL)
722

    
723

    
724
def _CheckNodeOnline(lu, node, msg=None):
725
  """Ensure that a given node is online.
726

727
  @param lu: the LU on behalf of which we make the check
728
  @param node: the node to check
729
  @param msg: if passed, should be a message to replace the default one
730
  @raise errors.OpPrereqError: if the node is offline
731

732
  """
733
  if msg is None:
734
    msg = "Can't use offline node"
735
  if lu.cfg.GetNodeInfo(node).offline:
736
    raise errors.OpPrereqError("%s: %s" % (msg, node), errors.ECODE_STATE)
737

    
738

    
739
def _CheckNodeNotDrained(lu, node):
740
  """Ensure that a given node is not drained.
741

742
  @param lu: the LU on behalf of which we make the check
743
  @param node: the node to check
744
  @raise errors.OpPrereqError: if the node is drained
745

746
  """
747
  if lu.cfg.GetNodeInfo(node).drained:
748
    raise errors.OpPrereqError("Can't use drained node %s" % node,
749
                               errors.ECODE_STATE)
750

    
751

    
752
def _CheckNodeVmCapable(lu, node):
753
  """Ensure that a given node is vm capable.
754

755
  @param lu: the LU on behalf of which we make the check
756
  @param node: the node to check
757
  @raise errors.OpPrereqError: if the node is not vm capable
758

759
  """
760
  if not lu.cfg.GetNodeInfo(node).vm_capable:
761
    raise errors.OpPrereqError("Can't use non-vm_capable node %s" % node,
762
                               errors.ECODE_STATE)
763

    
764

    
765
def _CheckNodeHasOS(lu, node, os_name, force_variant):
766
  """Ensure that a node supports a given OS.
767

768
  @param lu: the LU on behalf of which we make the check
769
  @param node: the node to check
770
  @param os_name: the OS to query about
771
  @param force_variant: whether to ignore variant errors
772
  @raise errors.OpPrereqError: if the node is not supporting the OS
773

774
  """
775
  result = lu.rpc.call_os_get(node, os_name)
776
  result.Raise("OS '%s' not in supported OS list for node %s" %
777
               (os_name, node),
778
               prereq=True, ecode=errors.ECODE_INVAL)
779
  if not force_variant:
780
    _CheckOSVariant(result.payload, os_name)
781

    
782

    
783
def _CheckNodeHasSecondaryIP(lu, node, secondary_ip, prereq):
784
  """Ensure that a node has the given secondary ip.
785

786
  @type lu: L{LogicalUnit}
787
  @param lu: the LU on behalf of which we make the check
788
  @type node: string
789
  @param node: the node to check
790
  @type secondary_ip: string
791
  @param secondary_ip: the ip to check
792
  @type prereq: boolean
793
  @param prereq: whether to throw a prerequisite or an execute error
794
  @raise errors.OpPrereqError: if the node doesn't have the ip, and prereq=True
795
  @raise errors.OpExecError: if the node doesn't have the ip, and prereq=False
796

797
  """
798
  result = lu.rpc.call_node_has_ip_address(node, secondary_ip)
799
  result.Raise("Failure checking secondary ip on node %s" % node,
800
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
801
  if not result.payload:
802
    msg = ("Node claims it doesn't have the secondary ip you gave (%s),"
803
           " please fix and re-run this command" % secondary_ip)
804
    if prereq:
805
      raise errors.OpPrereqError(msg, errors.ECODE_ENVIRON)
806
    else:
807
      raise errors.OpExecError(msg)
808

    
809

    
810
def _GetClusterDomainSecret():
811
  """Reads the cluster domain secret.
812

813
  """
814
  return utils.ReadOneLineFile(constants.CLUSTER_DOMAIN_SECRET_FILE,
815
                               strict=True)
816

    
817

    
818
def _CheckInstanceDown(lu, instance, reason):
819
  """Ensure that an instance is not running."""
820
  if instance.admin_up:
821
    raise errors.OpPrereqError("Instance %s is marked to be up, %s" %
822
                               (instance.name, reason), errors.ECODE_STATE)
823

    
824
  pnode = instance.primary_node
825
  ins_l = lu.rpc.call_instance_list([pnode], [instance.hypervisor])[pnode]
826
  ins_l.Raise("Can't contact node %s for instance information" % pnode,
827
              prereq=True, ecode=errors.ECODE_ENVIRON)
828

    
829
  if instance.name in ins_l.payload:
830
    raise errors.OpPrereqError("Instance %s is running, %s" %
831
                               (instance.name, reason), errors.ECODE_STATE)
832

    
833

    
834
def _ExpandItemName(fn, name, kind):
835
  """Expand an item name.
836

837
  @param fn: the function to use for expansion
838
  @param name: requested item name
839
  @param kind: text description ('Node' or 'Instance')
840
  @return: the resolved (full) name
841
  @raise errors.OpPrereqError: if the item is not found
842

843
  """
844
  full_name = fn(name)
845
  if full_name is None:
846
    raise errors.OpPrereqError("%s '%s' not known" % (kind, name),
847
                               errors.ECODE_NOENT)
848
  return full_name
849

    
850

    
851
def _ExpandNodeName(cfg, name):
852
  """Wrapper over L{_ExpandItemName} for nodes."""
853
  return _ExpandItemName(cfg.ExpandNodeName, name, "Node")
854

    
855

    
856
def _ExpandInstanceName(cfg, name):
857
  """Wrapper over L{_ExpandItemName} for instance."""
858
  return _ExpandItemName(cfg.ExpandInstanceName, name, "Instance")
859

    
860

    
861
def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status,
862
                          memory, vcpus, nics, disk_template, disks,
863
                          bep, hvp, hypervisor_name):
864
  """Builds instance related env variables for hooks
865

866
  This builds the hook environment from individual variables.
867

868
  @type name: string
869
  @param name: the name of the instance
870
  @type primary_node: string
871
  @param primary_node: the name of the instance's primary node
872
  @type secondary_nodes: list
873
  @param secondary_nodes: list of secondary nodes as strings
874
  @type os_type: string
875
  @param os_type: the name of the instance's OS
876
  @type status: boolean
877
  @param status: the should_run status of the instance
878
  @type memory: string
879
  @param memory: the memory size of the instance
880
  @type vcpus: string
881
  @param vcpus: the count of VCPUs the instance has
882
  @type nics: list
883
  @param nics: list of tuples (ip, mac, mode, link) representing
884
      the NICs the instance has
885
  @type disk_template: string
886
  @param disk_template: the disk template of the instance
887
  @type disks: list
888
  @param disks: the list of (size, mode) pairs
889
  @type bep: dict
890
  @param bep: the backend parameters for the instance
891
  @type hvp: dict
892
  @param hvp: the hypervisor parameters for the instance
893
  @type hypervisor_name: string
894
  @param hypervisor_name: the hypervisor for the instance
895
  @rtype: dict
896
  @return: the hook environment for this instance
897

898
  """
899
  if status:
900
    str_status = "up"
901
  else:
902
    str_status = "down"
903
  env = {
904
    "OP_TARGET": name,
905
    "INSTANCE_NAME": name,
906
    "INSTANCE_PRIMARY": primary_node,
907
    "INSTANCE_SECONDARIES": " ".join(secondary_nodes),
908
    "INSTANCE_OS_TYPE": os_type,
909
    "INSTANCE_STATUS": str_status,
910
    "INSTANCE_MEMORY": memory,
911
    "INSTANCE_VCPUS": vcpus,
912
    "INSTANCE_DISK_TEMPLATE": disk_template,
913
    "INSTANCE_HYPERVISOR": hypervisor_name,
914
  }
915

    
916
  if nics:
917
    nic_count = len(nics)
918
    for idx, (ip, mac, mode, link) in enumerate(nics):
919
      if ip is None:
920
        ip = ""
921
      env["INSTANCE_NIC%d_IP" % idx] = ip
922
      env["INSTANCE_NIC%d_MAC" % idx] = mac
923
      env["INSTANCE_NIC%d_MODE" % idx] = mode
924
      env["INSTANCE_NIC%d_LINK" % idx] = link
925
      if mode == constants.NIC_MODE_BRIDGED:
926
        env["INSTANCE_NIC%d_BRIDGE" % idx] = link
927
  else:
928
    nic_count = 0
929

    
930
  env["INSTANCE_NIC_COUNT"] = nic_count
931

    
932
  if disks:
933
    disk_count = len(disks)
934
    for idx, (size, mode) in enumerate(disks):
935
      env["INSTANCE_DISK%d_SIZE" % idx] = size
936
      env["INSTANCE_DISK%d_MODE" % idx] = mode
937
  else:
938
    disk_count = 0
939

    
940
  env["INSTANCE_DISK_COUNT"] = disk_count
941

    
942
  for source, kind in [(bep, "BE"), (hvp, "HV")]:
943
    for key, value in source.items():
944
      env["INSTANCE_%s_%s" % (kind, key)] = value
945

    
946
  return env
947

    
948

    
949
def _NICListToTuple(lu, nics):
950
  """Build a list of nic information tuples.
951

952
  This list is suitable to be passed to _BuildInstanceHookEnv or as a return
953
  value in LUInstanceQueryData.
954

955
  @type lu:  L{LogicalUnit}
956
  @param lu: the logical unit on whose behalf we execute
957
  @type nics: list of L{objects.NIC}
958
  @param nics: list of nics to convert to hooks tuples
959

960
  """
961
  hooks_nics = []
962
  cluster = lu.cfg.GetClusterInfo()
963
  for nic in nics:
964
    ip = nic.ip
965
    mac = nic.mac
966
    filled_params = cluster.SimpleFillNIC(nic.nicparams)
967
    mode = filled_params[constants.NIC_MODE]
968
    link = filled_params[constants.NIC_LINK]
969
    hooks_nics.append((ip, mac, mode, link))
970
  return hooks_nics
971

    
972

    
973
def _BuildInstanceHookEnvByObject(lu, instance, override=None):
974
  """Builds instance related env variables for hooks from an object.
975

976
  @type lu: L{LogicalUnit}
977
  @param lu: the logical unit on whose behalf we execute
978
  @type instance: L{objects.Instance}
979
  @param instance: the instance for which we should build the
980
      environment
981
  @type override: dict
982
  @param override: dictionary with key/values that will override
983
      our values
984
  @rtype: dict
985
  @return: the hook environment dictionary
986

987
  """
988
  cluster = lu.cfg.GetClusterInfo()
989
  bep = cluster.FillBE(instance)
990
  hvp = cluster.FillHV(instance)
991
  args = {
992
    'name': instance.name,
993
    'primary_node': instance.primary_node,
994
    'secondary_nodes': instance.secondary_nodes,
995
    'os_type': instance.os,
996
    'status': instance.admin_up,
997
    'memory': bep[constants.BE_MEMORY],
998
    'vcpus': bep[constants.BE_VCPUS],
999
    'nics': _NICListToTuple(lu, instance.nics),
1000
    'disk_template': instance.disk_template,
1001
    'disks': [(disk.size, disk.mode) for disk in instance.disks],
1002
    'bep': bep,
1003
    'hvp': hvp,
1004
    'hypervisor_name': instance.hypervisor,
1005
  }
1006
  if override:
1007
    args.update(override)
1008
  return _BuildInstanceHookEnv(**args) # pylint: disable-msg=W0142
1009

    
1010

    
1011
def _AdjustCandidatePool(lu, exceptions):
1012
  """Adjust the candidate pool after node operations.
1013

1014
  """
1015
  mod_list = lu.cfg.MaintainCandidatePool(exceptions)
1016
  if mod_list:
1017
    lu.LogInfo("Promoted nodes to master candidate role: %s",
1018
               utils.CommaJoin(node.name for node in mod_list))
1019
    for name in mod_list:
1020
      lu.context.ReaddNode(name)
1021
  mc_now, mc_max, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1022
  if mc_now > mc_max:
1023
    lu.LogInfo("Note: more nodes are candidates (%d) than desired (%d)" %
1024
               (mc_now, mc_max))
1025

    
1026

    
1027
def _DecideSelfPromotion(lu, exceptions=None):
1028
  """Decide whether I should promote myself as a master candidate.
1029

1030
  """
1031
  cp_size = lu.cfg.GetClusterInfo().candidate_pool_size
1032
  mc_now, mc_should, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1033
  # the new node will increase mc_max with one, so:
1034
  mc_should = min(mc_should + 1, cp_size)
1035
  return mc_now < mc_should
1036

    
1037

    
1038
def _CheckNicsBridgesExist(lu, target_nics, target_node):
1039
  """Check that the brigdes needed by a list of nics exist.
1040

1041
  """
1042
  cluster = lu.cfg.GetClusterInfo()
1043
  paramslist = [cluster.SimpleFillNIC(nic.nicparams) for nic in target_nics]
1044
  brlist = [params[constants.NIC_LINK] for params in paramslist
1045
            if params[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED]
1046
  if brlist:
1047
    result = lu.rpc.call_bridges_exist(target_node, brlist)
1048
    result.Raise("Error checking bridges on destination node '%s'" %
1049
                 target_node, prereq=True, ecode=errors.ECODE_ENVIRON)
1050

    
1051

    
1052
def _CheckInstanceBridgesExist(lu, instance, node=None):
1053
  """Check that the brigdes needed by an instance exist.
1054

1055
  """
1056
  if node is None:
1057
    node = instance.primary_node
1058
  _CheckNicsBridgesExist(lu, instance.nics, node)
1059

    
1060

    
1061
def _CheckOSVariant(os_obj, name):
1062
  """Check whether an OS name conforms to the os variants specification.
1063

1064
  @type os_obj: L{objects.OS}
1065
  @param os_obj: OS object to check
1066
  @type name: string
1067
  @param name: OS name passed by the user, to check for validity
1068

1069
  """
1070
  if not os_obj.supported_variants:
1071
    return
1072
  variant = objects.OS.GetVariant(name)
1073
  if not variant:
1074
    raise errors.OpPrereqError("OS name must include a variant",
1075
                               errors.ECODE_INVAL)
1076

    
1077
  if variant not in os_obj.supported_variants:
1078
    raise errors.OpPrereqError("Unsupported OS variant", errors.ECODE_INVAL)
1079

    
1080

    
1081
def _GetNodeInstancesInner(cfg, fn):
1082
  return [i for i in cfg.GetAllInstancesInfo().values() if fn(i)]
1083

    
1084

    
1085
def _GetNodeInstances(cfg, node_name):
1086
  """Returns a list of all primary and secondary instances on a node.
1087

1088
  """
1089

    
1090
  return _GetNodeInstancesInner(cfg, lambda inst: node_name in inst.all_nodes)
1091

    
1092

    
1093
def _GetNodePrimaryInstances(cfg, node_name):
1094
  """Returns primary instances on a node.
1095

1096
  """
1097
  return _GetNodeInstancesInner(cfg,
1098
                                lambda inst: node_name == inst.primary_node)
1099

    
1100

    
1101
def _GetNodeSecondaryInstances(cfg, node_name):
1102
  """Returns secondary instances on a node.
1103

1104
  """
1105
  return _GetNodeInstancesInner(cfg,
1106
                                lambda inst: node_name in inst.secondary_nodes)
1107

    
1108

    
1109
def _GetStorageTypeArgs(cfg, storage_type):
1110
  """Returns the arguments for a storage type.
1111

1112
  """
1113
  # Special case for file storage
1114
  if storage_type == constants.ST_FILE:
1115
    # storage.FileStorage wants a list of storage directories
1116
    return [[cfg.GetFileStorageDir(), cfg.GetSharedFileStorageDir()]]
1117

    
1118
  return []
1119

    
1120

    
1121
def _FindFaultyInstanceDisks(cfg, rpc, instance, node_name, prereq):
1122
  faulty = []
1123

    
1124
  for dev in instance.disks:
1125
    cfg.SetDiskID(dev, node_name)
1126

    
1127
  result = rpc.call_blockdev_getmirrorstatus(node_name, instance.disks)
1128
  result.Raise("Failed to get disk status from node %s" % node_name,
1129
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
1130

    
1131
  for idx, bdev_status in enumerate(result.payload):
1132
    if bdev_status and bdev_status.ldisk_status == constants.LDS_FAULTY:
1133
      faulty.append(idx)
1134

    
1135
  return faulty
1136

    
1137

    
1138
def _CheckIAllocatorOrNode(lu, iallocator_slot, node_slot):
1139
  """Check the sanity of iallocator and node arguments and use the
1140
  cluster-wide iallocator if appropriate.
1141

1142
  Check that at most one of (iallocator, node) is specified. If none is
1143
  specified, then the LU's opcode's iallocator slot is filled with the
1144
  cluster-wide default iallocator.
1145

1146
  @type iallocator_slot: string
1147
  @param iallocator_slot: the name of the opcode iallocator slot
1148
  @type node_slot: string
1149
  @param node_slot: the name of the opcode target node slot
1150

1151
  """
1152
  node = getattr(lu.op, node_slot, None)
1153
  iallocator = getattr(lu.op, iallocator_slot, None)
1154

    
1155
  if node is not None and iallocator is not None:
1156
    raise errors.OpPrereqError("Do not specify both, iallocator and node.",
1157
                               errors.ECODE_INVAL)
1158
  elif node is None and iallocator is None:
1159
    default_iallocator = lu.cfg.GetDefaultIAllocator()
1160
    if default_iallocator:
1161
      setattr(lu.op, iallocator_slot, default_iallocator)
1162
    else:
1163
      raise errors.OpPrereqError("No iallocator or node given and no"
1164
                                 " cluster-wide default iallocator found."
1165
                                 " Please specify either an iallocator or a"
1166
                                 " node, or set a cluster-wide default"
1167
                                 " iallocator.")
1168

    
1169

    
1170
class LUClusterPostInit(LogicalUnit):
1171
  """Logical unit for running hooks after cluster initialization.
1172

1173
  """
1174
  HPATH = "cluster-init"
1175
  HTYPE = constants.HTYPE_CLUSTER
1176

    
1177
  def BuildHooksEnv(self):
1178
    """Build hooks env.
1179

1180
    """
1181
    return {
1182
      "OP_TARGET": self.cfg.GetClusterName(),
1183
      }
1184

    
1185
  def BuildHooksNodes(self):
1186
    """Build hooks nodes.
1187

1188
    """
1189
    return ([], [self.cfg.GetMasterNode()])
1190

    
1191
  def Exec(self, feedback_fn):
1192
    """Nothing to do.
1193

1194
    """
1195
    return True
1196

    
1197

    
1198
class LUClusterDestroy(LogicalUnit):
1199
  """Logical unit for destroying the cluster.
1200

1201
  """
1202
  HPATH = "cluster-destroy"
1203
  HTYPE = constants.HTYPE_CLUSTER
1204

    
1205
  def BuildHooksEnv(self):
1206
    """Build hooks env.
1207

1208
    """
1209
    return {
1210
      "OP_TARGET": self.cfg.GetClusterName(),
1211
      }
1212

    
1213
  def BuildHooksNodes(self):
1214
    """Build hooks nodes.
1215

1216
    """
1217
    return ([], [])
1218

    
1219
  def CheckPrereq(self):
1220
    """Check prerequisites.
1221

1222
    This checks whether the cluster is empty.
1223

1224
    Any errors are signaled by raising errors.OpPrereqError.
1225

1226
    """
1227
    master = self.cfg.GetMasterNode()
1228

    
1229
    nodelist = self.cfg.GetNodeList()
1230
    if len(nodelist) != 1 or nodelist[0] != master:
1231
      raise errors.OpPrereqError("There are still %d node(s) in"
1232
                                 " this cluster." % (len(nodelist) - 1),
1233
                                 errors.ECODE_INVAL)
1234
    instancelist = self.cfg.GetInstanceList()
1235
    if instancelist:
1236
      raise errors.OpPrereqError("There are still %d instance(s) in"
1237
                                 " this cluster." % len(instancelist),
1238
                                 errors.ECODE_INVAL)
1239

    
1240
  def Exec(self, feedback_fn):
1241
    """Destroys the cluster.
1242

1243
    """
1244
    master = self.cfg.GetMasterNode()
1245

    
1246
    # Run post hooks on master node before it's removed
1247
    _RunPostHook(self, master)
1248

    
1249
    result = self.rpc.call_node_stop_master(master, False)
1250
    result.Raise("Could not disable the master role")
1251

    
1252
    return master
1253

    
1254

    
1255
def _VerifyCertificate(filename):
1256
  """Verifies a certificate for LUClusterVerify.
1257

1258
  @type filename: string
1259
  @param filename: Path to PEM file
1260

1261
  """
1262
  try:
1263
    cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
1264
                                           utils.ReadFile(filename))
1265
  except Exception, err: # pylint: disable-msg=W0703
1266
    return (LUClusterVerify.ETYPE_ERROR,
1267
            "Failed to load X509 certificate %s: %s" % (filename, err))
1268

    
1269
  (errcode, msg) = \
1270
    utils.VerifyX509Certificate(cert, constants.SSL_CERT_EXPIRATION_WARN,
1271
                                constants.SSL_CERT_EXPIRATION_ERROR)
1272

    
1273
  if msg:
1274
    fnamemsg = "While verifying %s: %s" % (filename, msg)
1275
  else:
1276
    fnamemsg = None
1277

    
1278
  if errcode is None:
1279
    return (None, fnamemsg)
1280
  elif errcode == utils.CERT_WARNING:
1281
    return (LUClusterVerify.ETYPE_WARNING, fnamemsg)
1282
  elif errcode == utils.CERT_ERROR:
1283
    return (LUClusterVerify.ETYPE_ERROR, fnamemsg)
1284

    
1285
  raise errors.ProgrammerError("Unhandled certificate error code %r" % errcode)
1286

    
1287

    
1288
class LUClusterVerify(LogicalUnit):
1289
  """Verifies the cluster status.
1290

1291
  """
1292
  HPATH = "cluster-verify"
1293
  HTYPE = constants.HTYPE_CLUSTER
1294
  REQ_BGL = False
1295

    
1296
  TCLUSTER = "cluster"
1297
  TNODE = "node"
1298
  TINSTANCE = "instance"
1299

    
1300
  ECLUSTERCFG = (TCLUSTER, "ECLUSTERCFG")
1301
  ECLUSTERCERT = (TCLUSTER, "ECLUSTERCERT")
1302
  ECLUSTERFILECHECK = (TCLUSTER, "ECLUSTERFILECHECK")
1303
  EINSTANCEBADNODE = (TINSTANCE, "EINSTANCEBADNODE")
1304
  EINSTANCEDOWN = (TINSTANCE, "EINSTANCEDOWN")
1305
  EINSTANCELAYOUT = (TINSTANCE, "EINSTANCELAYOUT")
1306
  EINSTANCEMISSINGDISK = (TINSTANCE, "EINSTANCEMISSINGDISK")
1307
  EINSTANCEFAULTYDISK = (TINSTANCE, "EINSTANCEFAULTYDISK")
1308
  EINSTANCEWRONGNODE = (TINSTANCE, "EINSTANCEWRONGNODE")
1309
  EINSTANCESPLITGROUPS = (TINSTANCE, "EINSTANCESPLITGROUPS")
1310
  ENODEDRBD = (TNODE, "ENODEDRBD")
1311
  ENODEDRBDHELPER = (TNODE, "ENODEDRBDHELPER")
1312
  ENODEFILECHECK = (TNODE, "ENODEFILECHECK")
1313
  ENODEHOOKS = (TNODE, "ENODEHOOKS")
1314
  ENODEHV = (TNODE, "ENODEHV")
1315
  ENODELVM = (TNODE, "ENODELVM")
1316
  ENODEN1 = (TNODE, "ENODEN1")
1317
  ENODENET = (TNODE, "ENODENET")
1318
  ENODEOS = (TNODE, "ENODEOS")
1319
  ENODEORPHANINSTANCE = (TNODE, "ENODEORPHANINSTANCE")
1320
  ENODEORPHANLV = (TNODE, "ENODEORPHANLV")
1321
  ENODERPC = (TNODE, "ENODERPC")
1322
  ENODESSH = (TNODE, "ENODESSH")
1323
  ENODEVERSION = (TNODE, "ENODEVERSION")
1324
  ENODESETUP = (TNODE, "ENODESETUP")
1325
  ENODETIME = (TNODE, "ENODETIME")
1326
  ENODEOOBPATH = (TNODE, "ENODEOOBPATH")
1327

    
1328
  ETYPE_FIELD = "code"
1329
  ETYPE_ERROR = "ERROR"
1330
  ETYPE_WARNING = "WARNING"
1331

    
1332
  _HOOKS_INDENT_RE = re.compile("^", re.M)
1333

    
1334
  class NodeImage(object):
1335
    """A class representing the logical and physical status of a node.
1336

1337
    @type name: string
1338
    @ivar name: the node name to which this object refers
1339
    @ivar volumes: a structure as returned from
1340
        L{ganeti.backend.GetVolumeList} (runtime)
1341
    @ivar instances: a list of running instances (runtime)
1342
    @ivar pinst: list of configured primary instances (config)
1343
    @ivar sinst: list of configured secondary instances (config)
1344
    @ivar sbp: dictionary of {primary-node: list of instances} for all
1345
        instances for which this node is secondary (config)
1346
    @ivar mfree: free memory, as reported by hypervisor (runtime)
1347
    @ivar dfree: free disk, as reported by the node (runtime)
1348
    @ivar offline: the offline status (config)
1349
    @type rpc_fail: boolean
1350
    @ivar rpc_fail: whether the RPC verify call was successfull (overall,
1351
        not whether the individual keys were correct) (runtime)
1352
    @type lvm_fail: boolean
1353
    @ivar lvm_fail: whether the RPC call didn't return valid LVM data
1354
    @type hyp_fail: boolean
1355
    @ivar hyp_fail: whether the RPC call didn't return the instance list
1356
    @type ghost: boolean
1357
    @ivar ghost: whether this is a known node or not (config)
1358
    @type os_fail: boolean
1359
    @ivar os_fail: whether the RPC call didn't return valid OS data
1360
    @type oslist: list
1361
    @ivar oslist: list of OSes as diagnosed by DiagnoseOS
1362
    @type vm_capable: boolean
1363
    @ivar vm_capable: whether the node can host instances
1364

1365
    """
1366
    def __init__(self, offline=False, name=None, vm_capable=True):
1367
      self.name = name
1368
      self.volumes = {}
1369
      self.instances = []
1370
      self.pinst = []
1371
      self.sinst = []
1372
      self.sbp = {}
1373
      self.mfree = 0
1374
      self.dfree = 0
1375
      self.offline = offline
1376
      self.vm_capable = vm_capable
1377
      self.rpc_fail = False
1378
      self.lvm_fail = False
1379
      self.hyp_fail = False
1380
      self.ghost = False
1381
      self.os_fail = False
1382
      self.oslist = {}
1383

    
1384
  def ExpandNames(self):
1385
    self.needed_locks = {
1386
      locking.LEVEL_NODE: locking.ALL_SET,
1387
      locking.LEVEL_INSTANCE: locking.ALL_SET,
1388
    }
1389
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
1390

    
1391
  def _Error(self, ecode, item, msg, *args, **kwargs):
1392
    """Format an error message.
1393

1394
    Based on the opcode's error_codes parameter, either format a
1395
    parseable error code, or a simpler error string.
1396

1397
    This must be called only from Exec and functions called from Exec.
1398

1399
    """
1400
    ltype = kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR)
1401
    itype, etxt = ecode
1402
    # first complete the msg
1403
    if args:
1404
      msg = msg % args
1405
    # then format the whole message
1406
    if self.op.error_codes:
1407
      msg = "%s:%s:%s:%s:%s" % (ltype, etxt, itype, item, msg)
1408
    else:
1409
      if item:
1410
        item = " " + item
1411
      else:
1412
        item = ""
1413
      msg = "%s: %s%s: %s" % (ltype, itype, item, msg)
1414
    # and finally report it via the feedback_fn
1415
    self._feedback_fn("  - %s" % msg)
1416

    
1417
  def _ErrorIf(self, cond, *args, **kwargs):
1418
    """Log an error message if the passed condition is True.
1419

1420
    """
1421
    cond = bool(cond) or self.op.debug_simulate_errors
1422
    if cond:
1423
      self._Error(*args, **kwargs)
1424
    # do not mark the operation as failed for WARN cases only
1425
    if kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR) == self.ETYPE_ERROR:
1426
      self.bad = self.bad or cond
1427

    
1428
  def _VerifyNode(self, ninfo, nresult):
1429
    """Perform some basic validation on data returned from a node.
1430

1431
      - check the result data structure is well formed and has all the
1432
        mandatory fields
1433
      - check ganeti version
1434

1435
    @type ninfo: L{objects.Node}
1436
    @param ninfo: the node to check
1437
    @param nresult: the results from the node
1438
    @rtype: boolean
1439
    @return: whether overall this call was successful (and we can expect
1440
         reasonable values in the respose)
1441

1442
    """
1443
    node = ninfo.name
1444
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1445

    
1446
    # main result, nresult should be a non-empty dict
1447
    test = not nresult or not isinstance(nresult, dict)
1448
    _ErrorIf(test, self.ENODERPC, node,
1449
                  "unable to verify node: no data returned")
1450
    if test:
1451
      return False
1452

    
1453
    # compares ganeti version
1454
    local_version = constants.PROTOCOL_VERSION
1455
    remote_version = nresult.get("version", None)
1456
    test = not (remote_version and
1457
                isinstance(remote_version, (list, tuple)) and
1458
                len(remote_version) == 2)
1459
    _ErrorIf(test, self.ENODERPC, node,
1460
             "connection to node returned invalid data")
1461
    if test:
1462
      return False
1463

    
1464
    test = local_version != remote_version[0]
1465
    _ErrorIf(test, self.ENODEVERSION, node,
1466
             "incompatible protocol versions: master %s,"
1467
             " node %s", local_version, remote_version[0])
1468
    if test:
1469
      return False
1470

    
1471
    # node seems compatible, we can actually try to look into its results
1472

    
1473
    # full package version
1474
    self._ErrorIf(constants.RELEASE_VERSION != remote_version[1],
1475
                  self.ENODEVERSION, node,
1476
                  "software version mismatch: master %s, node %s",
1477
                  constants.RELEASE_VERSION, remote_version[1],
1478
                  code=self.ETYPE_WARNING)
1479

    
1480
    hyp_result = nresult.get(constants.NV_HYPERVISOR, None)
1481
    if ninfo.vm_capable and isinstance(hyp_result, dict):
1482
      for hv_name, hv_result in hyp_result.iteritems():
1483
        test = hv_result is not None
1484
        _ErrorIf(test, self.ENODEHV, node,
1485
                 "hypervisor %s verify failure: '%s'", hv_name, hv_result)
1486

    
1487
    hvp_result = nresult.get(constants.NV_HVPARAMS, None)
1488
    if ninfo.vm_capable and isinstance(hvp_result, list):
1489
      for item, hv_name, hv_result in hvp_result:
1490
        _ErrorIf(True, self.ENODEHV, node,
1491
                 "hypervisor %s parameter verify failure (source %s): %s",
1492
                 hv_name, item, hv_result)
1493

    
1494
    test = nresult.get(constants.NV_NODESETUP,
1495
                       ["Missing NODESETUP results"])
1496
    _ErrorIf(test, self.ENODESETUP, node, "node setup error: %s",
1497
             "; ".join(test))
1498

    
1499
    return True
1500

    
1501
  def _VerifyNodeTime(self, ninfo, nresult,
1502
                      nvinfo_starttime, nvinfo_endtime):
1503
    """Check the node time.
1504

1505
    @type ninfo: L{objects.Node}
1506
    @param ninfo: the node to check
1507
    @param nresult: the remote results for the node
1508
    @param nvinfo_starttime: the start time of the RPC call
1509
    @param nvinfo_endtime: the end time of the RPC call
1510

1511
    """
1512
    node = ninfo.name
1513
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1514

    
1515
    ntime = nresult.get(constants.NV_TIME, None)
1516
    try:
1517
      ntime_merged = utils.MergeTime(ntime)
1518
    except (ValueError, TypeError):
1519
      _ErrorIf(True, self.ENODETIME, node, "Node returned invalid time")
1520
      return
1521

    
1522
    if ntime_merged < (nvinfo_starttime - constants.NODE_MAX_CLOCK_SKEW):
1523
      ntime_diff = "%.01fs" % abs(nvinfo_starttime - ntime_merged)
1524
    elif ntime_merged > (nvinfo_endtime + constants.NODE_MAX_CLOCK_SKEW):
1525
      ntime_diff = "%.01fs" % abs(ntime_merged - nvinfo_endtime)
1526
    else:
1527
      ntime_diff = None
1528

    
1529
    _ErrorIf(ntime_diff is not None, self.ENODETIME, node,
1530
             "Node time diverges by at least %s from master node time",
1531
             ntime_diff)
1532

    
1533
  def _VerifyNodeLVM(self, ninfo, nresult, vg_name):
1534
    """Check the node LVM results.
1535

1536
    @type ninfo: L{objects.Node}
1537
    @param ninfo: the node to check
1538
    @param nresult: the remote results for the node
1539
    @param vg_name: the configured VG name
1540

1541
    """
1542
    if vg_name is None:
1543
      return
1544

    
1545
    node = ninfo.name
1546
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1547

    
1548
    # checks vg existence and size > 20G
1549
    vglist = nresult.get(constants.NV_VGLIST, None)
1550
    test = not vglist
1551
    _ErrorIf(test, self.ENODELVM, node, "unable to check volume groups")
1552
    if not test:
1553
      vgstatus = utils.CheckVolumeGroupSize(vglist, vg_name,
1554
                                            constants.MIN_VG_SIZE)
1555
      _ErrorIf(vgstatus, self.ENODELVM, node, vgstatus)
1556

    
1557
    # check pv names
1558
    pvlist = nresult.get(constants.NV_PVLIST, None)
1559
    test = pvlist is None
1560
    _ErrorIf(test, self.ENODELVM, node, "Can't get PV list from node")
1561
    if not test:
1562
      # check that ':' is not present in PV names, since it's a
1563
      # special character for lvcreate (denotes the range of PEs to
1564
      # use on the PV)
1565
      for _, pvname, owner_vg in pvlist:
1566
        test = ":" in pvname
1567
        _ErrorIf(test, self.ENODELVM, node, "Invalid character ':' in PV"
1568
                 " '%s' of VG '%s'", pvname, owner_vg)
1569

    
1570
  def _VerifyNodeBridges(self, ninfo, nresult, bridges):
1571
    """Check the node bridges.
1572

1573
    @type ninfo: L{objects.Node}
1574
    @param ninfo: the node to check
1575
    @param nresult: the remote results for the node
1576
    @param bridges: the expected list of bridges
1577

1578
    """
1579
    if not bridges:
1580
      return
1581

    
1582
    node = ninfo.name
1583
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1584

    
1585
    missing = nresult.get(constants.NV_BRIDGES, None)
1586
    test = not isinstance(missing, list)
1587
    _ErrorIf(test, self.ENODENET, node,
1588
             "did not return valid bridge information")
1589
    if not test:
1590
      _ErrorIf(bool(missing), self.ENODENET, node, "missing bridges: %s" %
1591
               utils.CommaJoin(sorted(missing)))
1592

    
1593
  def _VerifyNodeNetwork(self, ninfo, nresult):
1594
    """Check the node network connectivity results.
1595

1596
    @type ninfo: L{objects.Node}
1597
    @param ninfo: the node to check
1598
    @param nresult: the remote results for the node
1599

1600
    """
1601
    node = ninfo.name
1602
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1603

    
1604
    test = constants.NV_NODELIST not in nresult
1605
    _ErrorIf(test, self.ENODESSH, node,
1606
             "node hasn't returned node ssh connectivity data")
1607
    if not test:
1608
      if nresult[constants.NV_NODELIST]:
1609
        for a_node, a_msg in nresult[constants.NV_NODELIST].items():
1610
          _ErrorIf(True, self.ENODESSH, node,
1611
                   "ssh communication with node '%s': %s", a_node, a_msg)
1612

    
1613
    test = constants.NV_NODENETTEST not in nresult
1614
    _ErrorIf(test, self.ENODENET, node,
1615
             "node hasn't returned node tcp connectivity data")
1616
    if not test:
1617
      if nresult[constants.NV_NODENETTEST]:
1618
        nlist = utils.NiceSort(nresult[constants.NV_NODENETTEST].keys())
1619
        for anode in nlist:
1620
          _ErrorIf(True, self.ENODENET, node,
1621
                   "tcp communication with node '%s': %s",
1622
                   anode, nresult[constants.NV_NODENETTEST][anode])
1623

    
1624
    test = constants.NV_MASTERIP not in nresult
1625
    _ErrorIf(test, self.ENODENET, node,
1626
             "node hasn't returned node master IP reachability data")
1627
    if not test:
1628
      if not nresult[constants.NV_MASTERIP]:
1629
        if node == self.master_node:
1630
          msg = "the master node cannot reach the master IP (not configured?)"
1631
        else:
1632
          msg = "cannot reach the master IP"
1633
        _ErrorIf(True, self.ENODENET, node, msg)
1634

    
1635
  def _VerifyInstance(self, instance, instanceconfig, node_image,
1636
                      diskstatus):
1637
    """Verify an instance.
1638

1639
    This function checks to see if the required block devices are
1640
    available on the instance's node.
1641

1642
    """
1643
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1644
    node_current = instanceconfig.primary_node
1645

    
1646
    node_vol_should = {}
1647
    instanceconfig.MapLVsByNode(node_vol_should)
1648

    
1649
    for node in node_vol_should:
1650
      n_img = node_image[node]
1651
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1652
        # ignore missing volumes on offline or broken nodes
1653
        continue
1654
      for volume in node_vol_should[node]:
1655
        test = volume not in n_img.volumes
1656
        _ErrorIf(test, self.EINSTANCEMISSINGDISK, instance,
1657
                 "volume %s missing on node %s", volume, node)
1658

    
1659
    if instanceconfig.admin_up:
1660
      pri_img = node_image[node_current]
1661
      test = instance not in pri_img.instances and not pri_img.offline
1662
      _ErrorIf(test, self.EINSTANCEDOWN, instance,
1663
               "instance not running on its primary node %s",
1664
               node_current)
1665

    
1666
    for node, n_img in node_image.items():
1667
      if node != node_current:
1668
        test = instance in n_img.instances
1669
        _ErrorIf(test, self.EINSTANCEWRONGNODE, instance,
1670
                 "instance should not run on node %s", node)
1671

    
1672
    diskdata = [(nname, success, status, idx)
1673
                for (nname, disks) in diskstatus.items()
1674
                for idx, (success, status) in enumerate(disks)]
1675

    
1676
    for nname, success, bdev_status, idx in diskdata:
1677
      # the 'ghost node' construction in Exec() ensures that we have a
1678
      # node here
1679
      snode = node_image[nname]
1680
      bad_snode = snode.ghost or snode.offline
1681
      _ErrorIf(instanceconfig.admin_up and not success and not bad_snode,
1682
               self.EINSTANCEFAULTYDISK, instance,
1683
               "couldn't retrieve status for disk/%s on %s: %s",
1684
               idx, nname, bdev_status)
1685
      _ErrorIf((instanceconfig.admin_up and success and
1686
                bdev_status.ldisk_status == constants.LDS_FAULTY),
1687
               self.EINSTANCEFAULTYDISK, instance,
1688
               "disk/%s on %s is faulty", idx, nname)
1689

    
1690
  def _VerifyOrphanVolumes(self, node_vol_should, node_image, reserved):
1691
    """Verify if there are any unknown volumes in the cluster.
1692

1693
    The .os, .swap and backup volumes are ignored. All other volumes are
1694
    reported as unknown.
1695

1696
    @type reserved: L{ganeti.utils.FieldSet}
1697
    @param reserved: a FieldSet of reserved volume names
1698

1699
    """
1700
    for node, n_img in node_image.items():
1701
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1702
        # skip non-healthy nodes
1703
        continue
1704
      for volume in n_img.volumes:
1705
        test = ((node not in node_vol_should or
1706
                volume not in node_vol_should[node]) and
1707
                not reserved.Matches(volume))
1708
        self._ErrorIf(test, self.ENODEORPHANLV, node,
1709
                      "volume %s is unknown", volume)
1710

    
1711
  def _VerifyOrphanInstances(self, instancelist, node_image):
1712
    """Verify the list of running instances.
1713

1714
    This checks what instances are running but unknown to the cluster.
1715

1716
    """
1717
    for node, n_img in node_image.items():
1718
      for o_inst in n_img.instances:
1719
        test = o_inst not in instancelist
1720
        self._ErrorIf(test, self.ENODEORPHANINSTANCE, node,
1721
                      "instance %s on node %s should not exist", o_inst, node)
1722

    
1723
  def _VerifyNPlusOneMemory(self, node_image, instance_cfg):
1724
    """Verify N+1 Memory Resilience.
1725

1726
    Check that if one single node dies we can still start all the
1727
    instances it was primary for.
1728

1729
    """
1730
    cluster_info = self.cfg.GetClusterInfo()
1731
    for node, n_img in node_image.items():
1732
      # This code checks that every node which is now listed as
1733
      # secondary has enough memory to host all instances it is
1734
      # supposed to should a single other node in the cluster fail.
1735
      # FIXME: not ready for failover to an arbitrary node
1736
      # FIXME: does not support file-backed instances
1737
      # WARNING: we currently take into account down instances as well
1738
      # as up ones, considering that even if they're down someone
1739
      # might want to start them even in the event of a node failure.
1740
      if n_img.offline:
1741
        # we're skipping offline nodes from the N+1 warning, since
1742
        # most likely we don't have good memory infromation from them;
1743
        # we already list instances living on such nodes, and that's
1744
        # enough warning
1745
        continue
1746
      for prinode, instances in n_img.sbp.items():
1747
        needed_mem = 0
1748
        for instance in instances:
1749
          bep = cluster_info.FillBE(instance_cfg[instance])
1750
          if bep[constants.BE_AUTO_BALANCE]:
1751
            needed_mem += bep[constants.BE_MEMORY]
1752
        test = n_img.mfree < needed_mem
1753
        self._ErrorIf(test, self.ENODEN1, node,
1754
                      "not enough memory to accomodate instance failovers"
1755
                      " should node %s fail (%dMiB needed, %dMiB available)",
1756
                      prinode, needed_mem, n_img.mfree)
1757

    
1758
  @classmethod
1759
  def _VerifyFiles(cls, errorif, nodeinfo, master_node, all_nvinfo,
1760
                   (files_all, files_all_opt, files_mc, files_vm)):
1761
    """Verifies file checksums collected from all nodes.
1762

1763
    @param errorif: Callback for reporting errors
1764
    @param nodeinfo: List of L{objects.Node} objects
1765
    @param master_node: Name of master node
1766
    @param all_nvinfo: RPC results
1767

1768
    """
1769
    node_names = frozenset(node.name for node in nodeinfo)
1770

    
1771
    assert master_node in node_names
1772
    assert (len(files_all | files_all_opt | files_mc | files_vm) ==
1773
            sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
1774
           "Found file listed in more than one file list"
1775

    
1776
    # Define functions determining which nodes to consider for a file
1777
    file2nodefn = dict([(filename, fn)
1778
      for (files, fn) in [(files_all, None),
1779
                          (files_all_opt, None),
1780
                          (files_mc, lambda node: (node.master_candidate or
1781
                                                   node.name == master_node)),
1782
                          (files_vm, lambda node: node.vm_capable)]
1783
      for filename in files])
1784

    
1785
    fileinfo = dict((filename, {}) for filename in file2nodefn.keys())
1786

    
1787
    for node in nodeinfo:
1788
      nresult = all_nvinfo[node.name]
1789

    
1790
      if nresult.fail_msg or not nresult.payload:
1791
        node_files = None
1792
      else:
1793
        node_files = nresult.payload.get(constants.NV_FILELIST, None)
1794

    
1795
      test = not (node_files and isinstance(node_files, dict))
1796
      errorif(test, cls.ENODEFILECHECK, node.name,
1797
              "Node did not return file checksum data")
1798
      if test:
1799
        continue
1800

    
1801
      for (filename, checksum) in node_files.items():
1802
        # Check if the file should be considered for a node
1803
        fn = file2nodefn[filename]
1804
        if fn is None or fn(node):
1805
          fileinfo[filename].setdefault(checksum, set()).add(node.name)
1806

    
1807
    for (filename, checksums) in fileinfo.items():
1808
      assert compat.all(len(i) > 10 for i in checksums), "Invalid checksum"
1809

    
1810
      # Nodes having the file
1811
      with_file = frozenset(node_name
1812
                            for nodes in fileinfo[filename].values()
1813
                            for node_name in nodes)
1814

    
1815
      # Nodes missing file
1816
      missing_file = node_names - with_file
1817

    
1818
      if filename in files_all_opt:
1819
        # All or no nodes
1820
        errorif(missing_file and missing_file != node_names,
1821
                cls.ECLUSTERFILECHECK, None,
1822
                "File %s is optional, but it must exist on all or no nodes (not"
1823
                " found on %s)",
1824
                filename, utils.CommaJoin(utils.NiceSort(missing_file)))
1825
      else:
1826
        errorif(missing_file, cls.ECLUSTERFILECHECK, None,
1827
                "File %s is missing from node(s) %s", filename,
1828
                utils.CommaJoin(utils.NiceSort(missing_file)))
1829

    
1830
      # See if there are multiple versions of the file
1831
      test = len(checksums) > 1
1832
      if test:
1833
        variants = ["variant %s on %s" %
1834
                    (idx + 1, utils.CommaJoin(utils.NiceSort(nodes)))
1835
                    for (idx, (checksum, nodes)) in
1836
                      enumerate(sorted(checksums.items()))]
1837
      else:
1838
        variants = []
1839

    
1840
      errorif(test, cls.ECLUSTERFILECHECK, None,
1841
              "File %s found with %s different checksums (%s)",
1842
              filename, len(checksums), "; ".join(variants))
1843

    
1844
  def _VerifyNodeDrbd(self, ninfo, nresult, instanceinfo, drbd_helper,
1845
                      drbd_map):
1846
    """Verifies and the node DRBD status.
1847

1848
    @type ninfo: L{objects.Node}
1849
    @param ninfo: the node to check
1850
    @param nresult: the remote results for the node
1851
    @param instanceinfo: the dict of instances
1852
    @param drbd_helper: the configured DRBD usermode helper
1853
    @param drbd_map: the DRBD map as returned by
1854
        L{ganeti.config.ConfigWriter.ComputeDRBDMap}
1855

1856
    """
1857
    node = ninfo.name
1858
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1859

    
1860
    if drbd_helper:
1861
      helper_result = nresult.get(constants.NV_DRBDHELPER, None)
1862
      test = (helper_result == None)
1863
      _ErrorIf(test, self.ENODEDRBDHELPER, node,
1864
               "no drbd usermode helper returned")
1865
      if helper_result:
1866
        status, payload = helper_result
1867
        test = not status
1868
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
1869
                 "drbd usermode helper check unsuccessful: %s", payload)
1870
        test = status and (payload != drbd_helper)
1871
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
1872
                 "wrong drbd usermode helper: %s", payload)
1873

    
1874
    # compute the DRBD minors
1875
    node_drbd = {}
1876
    for minor, instance in drbd_map[node].items():
1877
      test = instance not in instanceinfo
1878
      _ErrorIf(test, self.ECLUSTERCFG, None,
1879
               "ghost instance '%s' in temporary DRBD map", instance)
1880
        # ghost instance should not be running, but otherwise we
1881
        # don't give double warnings (both ghost instance and
1882
        # unallocated minor in use)
1883
      if test:
1884
        node_drbd[minor] = (instance, False)
1885
      else:
1886
        instance = instanceinfo[instance]
1887
        node_drbd[minor] = (instance.name, instance.admin_up)
1888

    
1889
    # and now check them
1890
    used_minors = nresult.get(constants.NV_DRBDLIST, [])
1891
    test = not isinstance(used_minors, (tuple, list))
1892
    _ErrorIf(test, self.ENODEDRBD, node,
1893
             "cannot parse drbd status file: %s", str(used_minors))
1894
    if test:
1895
      # we cannot check drbd status
1896
      return
1897

    
1898
    for minor, (iname, must_exist) in node_drbd.items():
1899
      test = minor not in used_minors and must_exist
1900
      _ErrorIf(test, self.ENODEDRBD, node,
1901
               "drbd minor %d of instance %s is not active", minor, iname)
1902
    for minor in used_minors:
1903
      test = minor not in node_drbd
1904
      _ErrorIf(test, self.ENODEDRBD, node,
1905
               "unallocated drbd minor %d is in use", minor)
1906

    
1907
  def _UpdateNodeOS(self, ninfo, nresult, nimg):
1908
    """Builds the node OS structures.
1909

1910
    @type ninfo: L{objects.Node}
1911
    @param ninfo: the node to check
1912
    @param nresult: the remote results for the node
1913
    @param nimg: the node image object
1914

1915
    """
1916
    node = ninfo.name
1917
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1918

    
1919
    remote_os = nresult.get(constants.NV_OSLIST, None)
1920
    test = (not isinstance(remote_os, list) or
1921
            not compat.all(isinstance(v, list) and len(v) == 7
1922
                           for v in remote_os))
1923

    
1924
    _ErrorIf(test, self.ENODEOS, node,
1925
             "node hasn't returned valid OS data")
1926

    
1927
    nimg.os_fail = test
1928

    
1929
    if test:
1930
      return
1931

    
1932
    os_dict = {}
1933

    
1934
    for (name, os_path, status, diagnose,
1935
         variants, parameters, api_ver) in nresult[constants.NV_OSLIST]:
1936

    
1937
      if name not in os_dict:
1938
        os_dict[name] = []
1939

    
1940
      # parameters is a list of lists instead of list of tuples due to
1941
      # JSON lacking a real tuple type, fix it:
1942
      parameters = [tuple(v) for v in parameters]
1943
      os_dict[name].append((os_path, status, diagnose,
1944
                            set(variants), set(parameters), set(api_ver)))
1945

    
1946
    nimg.oslist = os_dict
1947

    
1948
  def _VerifyNodeOS(self, ninfo, nimg, base):
1949
    """Verifies the node OS list.
1950

1951
    @type ninfo: L{objects.Node}
1952
    @param ninfo: the node to check
1953
    @param nimg: the node image object
1954
    @param base: the 'template' node we match against (e.g. from the master)
1955

1956
    """
1957
    node = ninfo.name
1958
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1959

    
1960
    assert not nimg.os_fail, "Entered _VerifyNodeOS with failed OS rpc?"
1961

    
1962
    beautify_params = lambda l: ["%s: %s" % (k, v) for (k, v) in l]
1963
    for os_name, os_data in nimg.oslist.items():
1964
      assert os_data, "Empty OS status for OS %s?!" % os_name
1965
      f_path, f_status, f_diag, f_var, f_param, f_api = os_data[0]
1966
      _ErrorIf(not f_status, self.ENODEOS, node,
1967
               "Invalid OS %s (located at %s): %s", os_name, f_path, f_diag)
1968
      _ErrorIf(len(os_data) > 1, self.ENODEOS, node,
1969
               "OS '%s' has multiple entries (first one shadows the rest): %s",
1970
               os_name, utils.CommaJoin([v[0] for v in os_data]))
1971
      # this will catched in backend too
1972
      _ErrorIf(compat.any(v >= constants.OS_API_V15 for v in f_api)
1973
               and not f_var, self.ENODEOS, node,
1974
               "OS %s with API at least %d does not declare any variant",
1975
               os_name, constants.OS_API_V15)
1976
      # comparisons with the 'base' image
1977
      test = os_name not in base.oslist
1978
      _ErrorIf(test, self.ENODEOS, node,
1979
               "Extra OS %s not present on reference node (%s)",
1980
               os_name, base.name)
1981
      if test:
1982
        continue
1983
      assert base.oslist[os_name], "Base node has empty OS status?"
1984
      _, b_status, _, b_var, b_param, b_api = base.oslist[os_name][0]
1985
      if not b_status:
1986
        # base OS is invalid, skipping
1987
        continue
1988
      for kind, a, b in [("API version", f_api, b_api),
1989
                         ("variants list", f_var, b_var),
1990
                         ("parameters", beautify_params(f_param),
1991
                          beautify_params(b_param))]:
1992
        _ErrorIf(a != b, self.ENODEOS, node,
1993
                 "OS %s for %s differs from reference node %s: [%s] vs. [%s]",
1994
                 kind, os_name, base.name,
1995
                 utils.CommaJoin(sorted(a)), utils.CommaJoin(sorted(b)))
1996

    
1997
    # check any missing OSes
1998
    missing = set(base.oslist.keys()).difference(nimg.oslist.keys())
1999
    _ErrorIf(missing, self.ENODEOS, node,
2000
             "OSes present on reference node %s but missing on this node: %s",
2001
             base.name, utils.CommaJoin(missing))
2002

    
2003
  def _VerifyOob(self, ninfo, nresult):
2004
    """Verifies out of band functionality of a node.
2005

2006
    @type ninfo: L{objects.Node}
2007
    @param ninfo: the node to check
2008
    @param nresult: the remote results for the node
2009

2010
    """
2011
    node = ninfo.name
2012
    # We just have to verify the paths on master and/or master candidates
2013
    # as the oob helper is invoked on the master
2014
    if ((ninfo.master_candidate or ninfo.master_capable) and
2015
        constants.NV_OOB_PATHS in nresult):
2016
      for path_result in nresult[constants.NV_OOB_PATHS]:
2017
        self._ErrorIf(path_result, self.ENODEOOBPATH, node, path_result)
2018

    
2019
  def _UpdateNodeVolumes(self, ninfo, nresult, nimg, vg_name):
2020
    """Verifies and updates the node volume data.
2021

2022
    This function will update a L{NodeImage}'s internal structures
2023
    with data from the remote call.
2024

2025
    @type ninfo: L{objects.Node}
2026
    @param ninfo: the node to check
2027
    @param nresult: the remote results for the node
2028
    @param nimg: the node image object
2029
    @param vg_name: the configured VG name
2030

2031
    """
2032
    node = ninfo.name
2033
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2034

    
2035
    nimg.lvm_fail = True
2036
    lvdata = nresult.get(constants.NV_LVLIST, "Missing LV data")
2037
    if vg_name is None:
2038
      pass
2039
    elif isinstance(lvdata, basestring):
2040
      _ErrorIf(True, self.ENODELVM, node, "LVM problem on node: %s",
2041
               utils.SafeEncode(lvdata))
2042
    elif not isinstance(lvdata, dict):
2043
      _ErrorIf(True, self.ENODELVM, node, "rpc call to node failed (lvlist)")
2044
    else:
2045
      nimg.volumes = lvdata
2046
      nimg.lvm_fail = False
2047

    
2048
  def _UpdateNodeInstances(self, ninfo, nresult, nimg):
2049
    """Verifies and updates the node instance list.
2050

2051
    If the listing was successful, then updates this node's instance
2052
    list. Otherwise, it marks the RPC call as failed for the instance
2053
    list key.
2054

2055
    @type ninfo: L{objects.Node}
2056
    @param ninfo: the node to check
2057
    @param nresult: the remote results for the node
2058
    @param nimg: the node image object
2059

2060
    """
2061
    idata = nresult.get(constants.NV_INSTANCELIST, None)
2062
    test = not isinstance(idata, list)
2063
    self._ErrorIf(test, self.ENODEHV, ninfo.name, "rpc call to node failed"
2064
                  " (instancelist): %s", utils.SafeEncode(str(idata)))
2065
    if test:
2066
      nimg.hyp_fail = True
2067
    else:
2068
      nimg.instances = idata
2069

    
2070
  def _UpdateNodeInfo(self, ninfo, nresult, nimg, vg_name):
2071
    """Verifies and computes a node information map
2072

2073
    @type ninfo: L{objects.Node}
2074
    @param ninfo: the node to check
2075
    @param nresult: the remote results for the node
2076
    @param nimg: the node image object
2077
    @param vg_name: the configured VG name
2078

2079
    """
2080
    node = ninfo.name
2081
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2082

    
2083
    # try to read free memory (from the hypervisor)
2084
    hv_info = nresult.get(constants.NV_HVINFO, None)
2085
    test = not isinstance(hv_info, dict) or "memory_free" not in hv_info
2086
    _ErrorIf(test, self.ENODEHV, node, "rpc call to node failed (hvinfo)")
2087
    if not test:
2088
      try:
2089
        nimg.mfree = int(hv_info["memory_free"])
2090
      except (ValueError, TypeError):
2091
        _ErrorIf(True, self.ENODERPC, node,
2092
                 "node returned invalid nodeinfo, check hypervisor")
2093

    
2094
    # FIXME: devise a free space model for file based instances as well
2095
    if vg_name is not None:
2096
      test = (constants.NV_VGLIST not in nresult or
2097
              vg_name not in nresult[constants.NV_VGLIST])
2098
      _ErrorIf(test, self.ENODELVM, node,
2099
               "node didn't return data for the volume group '%s'"
2100
               " - it is either missing or broken", vg_name)
2101
      if not test:
2102
        try:
2103
          nimg.dfree = int(nresult[constants.NV_VGLIST][vg_name])
2104
        except (ValueError, TypeError):
2105
          _ErrorIf(True, self.ENODERPC, node,
2106
                   "node returned invalid LVM info, check LVM status")
2107

    
2108
  def _CollectDiskInfo(self, nodelist, node_image, instanceinfo):
2109
    """Gets per-disk status information for all instances.
2110

2111
    @type nodelist: list of strings
2112
    @param nodelist: Node names
2113
    @type node_image: dict of (name, L{objects.Node})
2114
    @param node_image: Node objects
2115
    @type instanceinfo: dict of (name, L{objects.Instance})
2116
    @param instanceinfo: Instance objects
2117
    @rtype: {instance: {node: [(succes, payload)]}}
2118
    @return: a dictionary of per-instance dictionaries with nodes as
2119
        keys and disk information as values; the disk information is a
2120
        list of tuples (success, payload)
2121

2122
    """
2123
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2124

    
2125
    node_disks = {}
2126
    node_disks_devonly = {}
2127
    diskless_instances = set()
2128
    diskless = constants.DT_DISKLESS
2129

    
2130
    for nname in nodelist:
2131
      node_instances = list(itertools.chain(node_image[nname].pinst,
2132
                                            node_image[nname].sinst))
2133
      diskless_instances.update(inst for inst in node_instances
2134
                                if instanceinfo[inst].disk_template == diskless)
2135
      disks = [(inst, disk)
2136
               for inst in node_instances
2137
               for disk in instanceinfo[inst].disks]
2138

    
2139
      if not disks:
2140
        # No need to collect data
2141
        continue
2142

    
2143
      node_disks[nname] = disks
2144

    
2145
      # Creating copies as SetDiskID below will modify the objects and that can
2146
      # lead to incorrect data returned from nodes
2147
      devonly = [dev.Copy() for (_, dev) in disks]
2148

    
2149
      for dev in devonly:
2150
        self.cfg.SetDiskID(dev, nname)
2151

    
2152
      node_disks_devonly[nname] = devonly
2153

    
2154
    assert len(node_disks) == len(node_disks_devonly)
2155

    
2156
    # Collect data from all nodes with disks
2157
    result = self.rpc.call_blockdev_getmirrorstatus_multi(node_disks.keys(),
2158
                                                          node_disks_devonly)
2159

    
2160
    assert len(result) == len(node_disks)
2161

    
2162
    instdisk = {}
2163

    
2164
    for (nname, nres) in result.items():
2165
      disks = node_disks[nname]
2166

    
2167
      if nres.offline:
2168
        # No data from this node
2169
        data = len(disks) * [(False, "node offline")]
2170
      else:
2171
        msg = nres.fail_msg
2172
        _ErrorIf(msg, self.ENODERPC, nname,
2173
                 "while getting disk information: %s", msg)
2174
        if msg:
2175
          # No data from this node
2176
          data = len(disks) * [(False, msg)]
2177
        else:
2178
          data = []
2179
          for idx, i in enumerate(nres.payload):
2180
            if isinstance(i, (tuple, list)) and len(i) == 2:
2181
              data.append(i)
2182
            else:
2183
              logging.warning("Invalid result from node %s, entry %d: %s",
2184
                              nname, idx, i)
2185
              data.append((False, "Invalid result from the remote node"))
2186

    
2187
      for ((inst, _), status) in zip(disks, data):
2188
        instdisk.setdefault(inst, {}).setdefault(nname, []).append(status)
2189

    
2190
    # Add empty entries for diskless instances.
2191
    for inst in diskless_instances:
2192
      assert inst not in instdisk
2193
      instdisk[inst] = {}
2194

    
2195
    assert compat.all(len(statuses) == len(instanceinfo[inst].disks) and
2196
                      len(nnames) <= len(instanceinfo[inst].all_nodes) and
2197
                      compat.all(isinstance(s, (tuple, list)) and
2198
                                 len(s) == 2 for s in statuses)
2199
                      for inst, nnames in instdisk.items()
2200
                      for nname, statuses in nnames.items())
2201
    assert set(instdisk) == set(instanceinfo), "instdisk consistency failure"
2202

    
2203
    return instdisk
2204

    
2205
  def _VerifyHVP(self, hvp_data):
2206
    """Verifies locally the syntax of the hypervisor parameters.
2207

2208
    """
2209
    for item, hv_name, hv_params in hvp_data:
2210
      msg = ("hypervisor %s parameters syntax check (source %s): %%s" %
2211
             (item, hv_name))
2212
      try:
2213
        hv_class = hypervisor.GetHypervisor(hv_name)
2214
        utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
2215
        hv_class.CheckParameterSyntax(hv_params)
2216
      except errors.GenericError, err:
2217
        self._ErrorIf(True, self.ECLUSTERCFG, None, msg % str(err))
2218

    
2219
  def BuildHooksEnv(self):
2220
    """Build hooks env.
2221

2222
    Cluster-Verify hooks just ran in the post phase and their failure makes
2223
    the output be logged in the verify output and the verification to fail.
2224

2225
    """
2226
    cfg = self.cfg
2227

    
2228
    env = {
2229
      "CLUSTER_TAGS": " ".join(cfg.GetClusterInfo().GetTags())
2230
      }
2231

    
2232
    env.update(("NODE_TAGS_%s" % node.name, " ".join(node.GetTags()))
2233
               for node in cfg.GetAllNodesInfo().values())
2234

    
2235
    return env
2236

    
2237
  def BuildHooksNodes(self):
2238
    """Build hooks nodes.
2239

2240
    """
2241
    return ([], self.cfg.GetNodeList())
2242

    
2243
  def Exec(self, feedback_fn):
2244
    """Verify integrity of cluster, performing various test on nodes.
2245

2246
    """
2247
    # This method has too many local variables. pylint: disable-msg=R0914
2248
    self.bad = False
2249
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2250
    verbose = self.op.verbose
2251
    self._feedback_fn = feedback_fn
2252
    feedback_fn("* Verifying global settings")
2253
    for msg in self.cfg.VerifyConfig():
2254
      _ErrorIf(True, self.ECLUSTERCFG, None, msg)
2255

    
2256
    # Check the cluster certificates
2257
    for cert_filename in constants.ALL_CERT_FILES:
2258
      (errcode, msg) = _VerifyCertificate(cert_filename)
2259
      _ErrorIf(errcode, self.ECLUSTERCERT, None, msg, code=errcode)
2260

    
2261
    vg_name = self.cfg.GetVGName()
2262
    drbd_helper = self.cfg.GetDRBDHelper()
2263
    hypervisors = self.cfg.GetClusterInfo().enabled_hypervisors
2264
    cluster = self.cfg.GetClusterInfo()
2265
    nodeinfo_byname = self.cfg.GetAllNodesInfo()
2266
    nodelist = utils.NiceSort(nodeinfo_byname.keys())
2267
    nodeinfo = [nodeinfo_byname[nname] for nname in nodelist]
2268
    instanceinfo = self.cfg.GetAllInstancesInfo()
2269
    instancelist = utils.NiceSort(instanceinfo.keys())
2270
    groupinfo = self.cfg.GetAllNodeGroupsInfo()
2271
    i_non_redundant = [] # Non redundant instances
2272
    i_non_a_balanced = [] # Non auto-balanced instances
2273
    n_offline = 0 # Count of offline nodes
2274
    n_drained = 0 # Count of nodes being drained
2275
    node_vol_should = {}
2276

    
2277
    # FIXME: verify OS list
2278

    
2279
    # File verification
2280
    filemap = _ComputeAncillaryFiles(cluster, False)
2281

    
2282
    # do local checksums
2283
    master_node = self.master_node = self.cfg.GetMasterNode()
2284
    master_ip = self.cfg.GetMasterIP()
2285

    
2286
    # Compute the set of hypervisor parameters
2287
    hvp_data = []
2288
    for hv_name in hypervisors:
2289
      hvp_data.append(("cluster", hv_name, cluster.GetHVDefaults(hv_name)))
2290
    for os_name, os_hvp in cluster.os_hvp.items():
2291
      for hv_name, hv_params in os_hvp.items():
2292
        if not hv_params:
2293
          continue
2294
        full_params = cluster.GetHVDefaults(hv_name, os_name=os_name)
2295
        hvp_data.append(("os %s" % os_name, hv_name, full_params))
2296
    # TODO: collapse identical parameter values in a single one
2297
    for instance in instanceinfo.values():
2298
      if not instance.hvparams:
2299
        continue
2300
      hvp_data.append(("instance %s" % instance.name, instance.hypervisor,
2301
                       cluster.FillHV(instance)))
2302
    # and verify them locally
2303
    self._VerifyHVP(hvp_data)
2304

    
2305
    feedback_fn("* Gathering data (%d nodes)" % len(nodelist))
2306
    node_verify_param = {
2307
      constants.NV_FILELIST:
2308
        utils.UniqueSequence(filename
2309
                             for files in filemap
2310
                             for filename in files),
2311
      constants.NV_NODELIST: [node.name for node in nodeinfo
2312
                              if not node.offline],
2313
      constants.NV_HYPERVISOR: hypervisors,
2314
      constants.NV_HVPARAMS: hvp_data,
2315
      constants.NV_NODENETTEST: [(node.name, node.primary_ip,
2316
                                  node.secondary_ip) for node in nodeinfo
2317
                                 if not node.offline],
2318
      constants.NV_INSTANCELIST: hypervisors,
2319
      constants.NV_VERSION: None,
2320
      constants.NV_HVINFO: self.cfg.GetHypervisorType(),
2321
      constants.NV_NODESETUP: None,
2322
      constants.NV_TIME: None,
2323
      constants.NV_MASTERIP: (master_node, master_ip),
2324
      constants.NV_OSLIST: None,
2325
      constants.NV_VMNODES: self.cfg.GetNonVmCapableNodeList(),
2326
      }
2327

    
2328
    if vg_name is not None:
2329
      node_verify_param[constants.NV_VGLIST] = None
2330
      node_verify_param[constants.NV_LVLIST] = vg_name
2331
      node_verify_param[constants.NV_PVLIST] = [vg_name]
2332
      node_verify_param[constants.NV_DRBDLIST] = None
2333

    
2334
    if drbd_helper:
2335
      node_verify_param[constants.NV_DRBDHELPER] = drbd_helper
2336

    
2337
    # bridge checks
2338
    # FIXME: this needs to be changed per node-group, not cluster-wide
2339
    bridges = set()
2340
    default_nicpp = cluster.nicparams[constants.PP_DEFAULT]
2341
    if default_nicpp[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2342
      bridges.add(default_nicpp[constants.NIC_LINK])
2343
    for instance in instanceinfo.values():
2344
      for nic in instance.nics:
2345
        full_nic = cluster.SimpleFillNIC(nic.nicparams)
2346
        if full_nic[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2347
          bridges.add(full_nic[constants.NIC_LINK])
2348

    
2349
    if bridges:
2350
      node_verify_param[constants.NV_BRIDGES] = list(bridges)
2351

    
2352
    # Build our expected cluster state
2353
    node_image = dict((node.name, self.NodeImage(offline=node.offline,
2354
                                                 name=node.name,
2355
                                                 vm_capable=node.vm_capable))
2356
                      for node in nodeinfo)
2357

    
2358
    # Gather OOB paths
2359
    oob_paths = []
2360
    for node in nodeinfo:
2361
      path = _SupportsOob(self.cfg, node)
2362
      if path and path not in oob_paths:
2363
        oob_paths.append(path)
2364

    
2365
    if oob_paths:
2366
      node_verify_param[constants.NV_OOB_PATHS] = oob_paths
2367

    
2368
    for instance in instancelist:
2369
      inst_config = instanceinfo[instance]
2370

    
2371
      for nname in inst_config.all_nodes:
2372
        if nname not in node_image:
2373
          # ghost node
2374
          gnode = self.NodeImage(name=nname)
2375
          gnode.ghost = True
2376
          node_image[nname] = gnode
2377

    
2378
      inst_config.MapLVsByNode(node_vol_should)
2379

    
2380
      pnode = inst_config.primary_node
2381
      node_image[pnode].pinst.append(instance)
2382

    
2383
      for snode in inst_config.secondary_nodes:
2384
        nimg = node_image[snode]
2385
        nimg.sinst.append(instance)
2386
        if pnode not in nimg.sbp:
2387
          nimg.sbp[pnode] = []
2388
        nimg.sbp[pnode].append(instance)
2389

    
2390
    # At this point, we have the in-memory data structures complete,
2391
    # except for the runtime information, which we'll gather next
2392

    
2393
    # Due to the way our RPC system works, exact response times cannot be
2394
    # guaranteed (e.g. a broken node could run into a timeout). By keeping the
2395
    # time before and after executing the request, we can at least have a time
2396
    # window.
2397
    nvinfo_starttime = time.time()
2398
    all_nvinfo = self.rpc.call_node_verify(nodelist, node_verify_param,
2399
                                           self.cfg.GetClusterName())
2400
    nvinfo_endtime = time.time()
2401

    
2402
    all_drbd_map = self.cfg.ComputeDRBDMap()
2403

    
2404
    feedback_fn("* Gathering disk information (%s nodes)" % len(nodelist))
2405
    instdisk = self._CollectDiskInfo(nodelist, node_image, instanceinfo)
2406

    
2407
    feedback_fn("* Verifying configuration file consistency")
2408
    self._VerifyFiles(_ErrorIf, nodeinfo, master_node, all_nvinfo, filemap)
2409

    
2410
    feedback_fn("* Verifying node status")
2411

    
2412
    refos_img = None
2413

    
2414
    for node_i in nodeinfo:
2415
      node = node_i.name
2416
      nimg = node_image[node]
2417

    
2418
      if node_i.offline:
2419
        if verbose:
2420
          feedback_fn("* Skipping offline node %s" % (node,))
2421
        n_offline += 1
2422
        continue
2423

    
2424
      if node == master_node:
2425
        ntype = "master"
2426
      elif node_i.master_candidate:
2427
        ntype = "master candidate"
2428
      elif node_i.drained:
2429
        ntype = "drained"
2430
        n_drained += 1
2431
      else:
2432
        ntype = "regular"
2433
      if verbose:
2434
        feedback_fn("* Verifying node %s (%s)" % (node, ntype))
2435

    
2436
      msg = all_nvinfo[node].fail_msg
2437
      _ErrorIf(msg, self.ENODERPC, node, "while contacting node: %s", msg)
2438
      if msg:
2439
        nimg.rpc_fail = True
2440
        continue
2441

    
2442
      nresult = all_nvinfo[node].payload
2443

    
2444
      nimg.call_ok = self._VerifyNode(node_i, nresult)
2445
      self._VerifyNodeTime(node_i, nresult, nvinfo_starttime, nvinfo_endtime)
2446
      self._VerifyNodeNetwork(node_i, nresult)
2447
      self._VerifyOob(node_i, nresult)
2448

    
2449
      if nimg.vm_capable:
2450
        self._VerifyNodeLVM(node_i, nresult, vg_name)
2451
        self._VerifyNodeDrbd(node_i, nresult, instanceinfo, drbd_helper,
2452
                             all_drbd_map)
2453

    
2454
        self._UpdateNodeVolumes(node_i, nresult, nimg, vg_name)
2455
        self._UpdateNodeInstances(node_i, nresult, nimg)
2456
        self._UpdateNodeInfo(node_i, nresult, nimg, vg_name)
2457
        self._UpdateNodeOS(node_i, nresult, nimg)
2458
        if not nimg.os_fail:
2459
          if refos_img is None:
2460
            refos_img = nimg
2461
          self._VerifyNodeOS(node_i, nimg, refos_img)
2462
        self._VerifyNodeBridges(node_i, nresult, bridges)
2463

    
2464
    feedback_fn("* Verifying instance status")
2465
    for instance in instancelist:
2466
      if verbose:
2467
        feedback_fn("* Verifying instance %s" % instance)
2468
      inst_config = instanceinfo[instance]
2469
      self._VerifyInstance(instance, inst_config, node_image,
2470
                           instdisk[instance])
2471
      inst_nodes_offline = []
2472

    
2473
      pnode = inst_config.primary_node
2474
      pnode_img = node_image[pnode]
2475
      _ErrorIf(pnode_img.rpc_fail and not pnode_img.offline,
2476
               self.ENODERPC, pnode, "instance %s, connection to"
2477
               " primary node failed", instance)
2478

    
2479
      _ErrorIf(inst_config.admin_up and pnode_img.offline,
2480
               self.EINSTANCEBADNODE, instance,
2481
               "instance is marked as running and lives on offline node %s",
2482
               inst_config.primary_node)
2483

    
2484
      # If the instance is non-redundant we cannot survive losing its primary
2485
      # node, so we are not N+1 compliant. On the other hand we have no disk
2486
      # templates with more than one secondary so that situation is not well
2487
      # supported either.
2488
      # FIXME: does not support file-backed instances
2489
      if not inst_config.secondary_nodes:
2490
        i_non_redundant.append(instance)
2491

    
2492
      _ErrorIf(len(inst_config.secondary_nodes) > 1, self.EINSTANCELAYOUT,
2493
               instance, "instance has multiple secondary nodes: %s",
2494
               utils.CommaJoin(inst_config.secondary_nodes),
2495
               code=self.ETYPE_WARNING)
2496

    
2497
      if inst_config.disk_template in constants.DTS_INT_MIRROR:
2498
        pnode = inst_config.primary_node
2499
        instance_nodes = utils.NiceSort(inst_config.all_nodes)
2500
        instance_groups = {}
2501

    
2502
        for node in instance_nodes:
2503
          instance_groups.setdefault(nodeinfo_byname[node].group,
2504
                                     []).append(node)
2505

    
2506
        pretty_list = [
2507
          "%s (group %s)" % (utils.CommaJoin(nodes), groupinfo[group].name)
2508
          # Sort so that we always list the primary node first.
2509
          for group, nodes in sorted(instance_groups.items(),
2510
                                     key=lambda (_, nodes): pnode in nodes,
2511
                                     reverse=True)]
2512

    
2513
        self._ErrorIf(len(instance_groups) > 1, self.EINSTANCESPLITGROUPS,
2514
                      instance, "instance has primary and secondary nodes in"
2515
                      " different groups: %s", utils.CommaJoin(pretty_list),
2516
                      code=self.ETYPE_WARNING)
2517

    
2518
      if not cluster.FillBE(inst_config)[constants.BE_AUTO_BALANCE]:
2519
        i_non_a_balanced.append(instance)
2520

    
2521
      for snode in inst_config.secondary_nodes:
2522
        s_img = node_image[snode]
2523
        _ErrorIf(s_img.rpc_fail and not s_img.offline, self.ENODERPC, snode,
2524
                 "instance %s, connection to secondary node failed", instance)
2525

    
2526
        if s_img.offline:
2527
          inst_nodes_offline.append(snode)
2528

    
2529
      # warn that the instance lives on offline nodes
2530
      _ErrorIf(inst_nodes_offline, self.EINSTANCEBADNODE, instance,
2531
               "instance has offline secondary node(s) %s",
2532
               utils.CommaJoin(inst_nodes_offline))
2533
      # ... or ghost/non-vm_capable nodes
2534
      for node in inst_config.all_nodes:
2535
        _ErrorIf(node_image[node].ghost, self.EINSTANCEBADNODE, instance,
2536
                 "instance lives on ghost node %s", node)
2537
        _ErrorIf(not node_image[node].vm_capable, self.EINSTANCEBADNODE,
2538
                 instance, "instance lives on non-vm_capable node %s", node)
2539

    
2540
    feedback_fn("* Verifying orphan volumes")
2541
    reserved = utils.FieldSet(*cluster.reserved_lvs)
2542
    self._VerifyOrphanVolumes(node_vol_should, node_image, reserved)
2543

    
2544
    feedback_fn("* Verifying orphan instances")
2545
    self._VerifyOrphanInstances(instancelist, node_image)
2546

    
2547
    if constants.VERIFY_NPLUSONE_MEM not in self.op.skip_checks:
2548
      feedback_fn("* Verifying N+1 Memory redundancy")
2549
      self._VerifyNPlusOneMemory(node_image, instanceinfo)
2550

    
2551
    feedback_fn("* Other Notes")
2552
    if i_non_redundant:
2553
      feedback_fn("  - NOTICE: %d non-redundant instance(s) found."
2554
                  % len(i_non_redundant))
2555

    
2556
    if i_non_a_balanced:
2557
      feedback_fn("  - NOTICE: %d non-auto-balanced instance(s) found."
2558
                  % len(i_non_a_balanced))
2559

    
2560
    if n_offline:
2561
      feedback_fn("  - NOTICE: %d offline node(s) found." % n_offline)
2562

    
2563
    if n_drained:
2564
      feedback_fn("  - NOTICE: %d drained node(s) found." % n_drained)
2565

    
2566
    return not self.bad
2567

    
2568
  def HooksCallBack(self, phase, hooks_results, feedback_fn, lu_result):
2569
    """Analyze the post-hooks' result
2570

2571
    This method analyses the hook result, handles it, and sends some
2572
    nicely-formatted feedback back to the user.
2573

2574
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
2575
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
2576
    @param hooks_results: the results of the multi-node hooks rpc call
2577
    @param feedback_fn: function used send feedback back to the caller
2578
    @param lu_result: previous Exec result
2579
    @return: the new Exec result, based on the previous result
2580
        and hook results
2581

2582
    """
2583
    # We only really run POST phase hooks, and are only interested in
2584
    # their results
2585
    if phase == constants.HOOKS_PHASE_POST:
2586
      # Used to change hooks' output to proper indentation
2587
      feedback_fn("* Hooks Results")
2588
      assert hooks_results, "invalid result from hooks"
2589

    
2590
      for node_name in hooks_results:
2591
        res = hooks_results[node_name]
2592
        msg = res.fail_msg
2593
        test = msg and not res.offline
2594
        self._ErrorIf(test, self.ENODEHOOKS, node_name,
2595
                      "Communication failure in hooks execution: %s", msg)
2596
        if res.offline or msg:
2597
          # No need to investigate payload if node is offline or gave an error.
2598
          # override manually lu_result here as _ErrorIf only
2599
          # overrides self.bad
2600
          lu_result = 1
2601
          continue
2602
        for script, hkr, output in res.payload:
2603
          test = hkr == constants.HKR_FAIL
2604
          self._ErrorIf(test, self.ENODEHOOKS, node_name,
2605
                        "Script %s failed, output:", script)
2606
          if test:
2607
            output = self._HOOKS_INDENT_RE.sub('      ', output)
2608
            feedback_fn("%s" % output)
2609
            lu_result = 0
2610

    
2611
      return lu_result
2612

    
2613

    
2614
class LUClusterVerifyDisks(NoHooksLU):
2615
  """Verifies the cluster disks status.
2616

2617
  """
2618
  REQ_BGL = False
2619

    
2620
  def ExpandNames(self):
2621
    self.needed_locks = {
2622
      locking.LEVEL_NODE: locking.ALL_SET,
2623
      locking.LEVEL_INSTANCE: locking.ALL_SET,
2624
    }
2625
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2626

    
2627
  def Exec(self, feedback_fn):
2628
    """Verify integrity of cluster disks.
2629

2630
    @rtype: tuple of three items
2631
    @return: a tuple of (dict of node-to-node_error, list of instances
2632
        which need activate-disks, dict of instance: (node, volume) for
2633
        missing volumes
2634

2635
    """
2636
    result = res_nodes, res_instances, res_missing = {}, [], {}
2637

    
2638
    nodes = utils.NiceSort(self.cfg.GetVmCapableNodeList())
2639
    instances = self.cfg.GetAllInstancesInfo().values()
2640

    
2641
    nv_dict = {}
2642
    for inst in instances:
2643
      inst_lvs = {}
2644
      if not inst.admin_up:
2645
        continue
2646
      inst.MapLVsByNode(inst_lvs)
2647
      # transform { iname: {node: [vol,],},} to {(node, vol): iname}
2648
      for node, vol_list in inst_lvs.iteritems():
2649
        for vol in vol_list:
2650
          nv_dict[(node, vol)] = inst
2651

    
2652
    if not nv_dict:
2653
      return result
2654

    
2655
    node_lvs = self.rpc.call_lv_list(nodes, [])
2656
    for node, node_res in node_lvs.items():
2657
      if node_res.offline:
2658
        continue
2659
      msg = node_res.fail_msg
2660
      if msg:
2661
        logging.warning("Error enumerating LVs on node %s: %s", node, msg)
2662
        res_nodes[node] = msg
2663
        continue
2664

    
2665
      lvs = node_res.payload
2666
      for lv_name, (_, _, lv_online) in lvs.items():
2667
        inst = nv_dict.pop((node, lv_name), None)
2668
        if (not lv_online and inst is not None
2669
            and inst.name not in res_instances):
2670
          res_instances.append(inst.name)
2671

    
2672
    # any leftover items in nv_dict are missing LVs, let's arrange the
2673
    # data better
2674
    for key, inst in nv_dict.iteritems():
2675
      if inst.name not in res_missing:
2676
        res_missing[inst.name] = []
2677
      res_missing[inst.name].append(key)
2678

    
2679
    return result
2680

    
2681

    
2682
class LUClusterRepairDiskSizes(NoHooksLU):
2683
  """Verifies the cluster disks sizes.
2684

2685
  """
2686
  REQ_BGL = False
2687

    
2688
  def ExpandNames(self):
2689
    if self.op.instances:
2690
      self.wanted_names = _GetWantedInstances(self, self.op.instances)
2691
      self.needed_locks = {
2692
        locking.LEVEL_NODE: [],
2693
        locking.LEVEL_INSTANCE: self.wanted_names,
2694
        }
2695
      self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE
2696
    else:
2697
      self.wanted_names = None
2698
      self.needed_locks = {
2699
        locking.LEVEL_NODE: locking.ALL_SET,
2700
        locking.LEVEL_INSTANCE: locking.ALL_SET,
2701
        }
2702
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2703

    
2704
  def DeclareLocks(self, level):
2705
    if level == locking.LEVEL_NODE and self.wanted_names is not None:
2706
      self._LockInstancesNodes(primary_only=True)
2707

    
2708
  def CheckPrereq(self):
2709
    """Check prerequisites.
2710

2711
    This only checks the optional instance list against the existing names.
2712

2713
    """
2714
    if self.wanted_names is None:
2715
      self.wanted_names = self.glm.list_owned(locking.LEVEL_INSTANCE)
2716

    
2717
    self.wanted_instances = [self.cfg.GetInstanceInfo(name) for name
2718
                             in self.wanted_names]
2719

    
2720
  def _EnsureChildSizes(self, disk):
2721
    """Ensure children of the disk have the needed disk size.
2722

2723
    This is valid mainly for DRBD8 and fixes an issue where the
2724
    children have smaller disk size.
2725

2726
    @param disk: an L{ganeti.objects.Disk} object
2727

2728
    """
2729
    if disk.dev_type == constants.LD_DRBD8:
2730
      assert disk.children, "Empty children for DRBD8?"
2731
      fchild = disk.children[0]
2732
      mismatch = fchild.size < disk.size
2733
      if mismatch:
2734
        self.LogInfo("Child disk has size %d, parent %d, fixing",
2735
                     fchild.size, disk.size)
2736
        fchild.size = disk.size
2737

    
2738
      # and we recurse on this child only, not on the metadev
2739
      return self._EnsureChildSizes(fchild) or mismatch
2740
    else:
2741
      return False
2742

    
2743
  def Exec(self, feedback_fn):
2744
    """Verify the size of cluster disks.
2745

2746
    """
2747
    # TODO: check child disks too
2748
    # TODO: check differences in size between primary/secondary nodes
2749
    per_node_disks = {}
2750
    for instance in self.wanted_instances:
2751
      pnode = instance.primary_node
2752
      if pnode not in per_node_disks:
2753
        per_node_disks[pnode] = []
2754
      for idx, disk in enumerate(instance.disks):
2755
        per_node_disks[pnode].append((instance, idx, disk))
2756

    
2757
    changed = []
2758
    for node, dskl in per_node_disks.items():
2759
      newl = [v[2].Copy() for v in dskl]
2760
      for dsk in newl:
2761
        self.cfg.SetDiskID(dsk, node)
2762
      result = self.rpc.call_blockdev_getsize(node, newl)
2763
      if result.fail_msg:
2764
        self.LogWarning("Failure in blockdev_getsize call to node"
2765
                        " %s, ignoring", node)
2766
        continue
2767
      if len(result.payload) != len(dskl):
2768
        logging.warning("Invalid result from node %s: len(dksl)=%d,"
2769
                        " result.payload=%s", node, len(dskl), result.payload)
2770
        self.LogWarning("Invalid result from node %s, ignoring node results",
2771
                        node)
2772
        continue
2773
      for ((instance, idx, disk), size) in zip(dskl, result.payload):
2774
        if size is None:
2775
          self.LogWarning("Disk %d of instance %s did not return size"
2776
                          " information, ignoring", idx, instance.name)
2777
          continue
2778
        if not isinstance(size, (int, long)):
2779
          self.LogWarning("Disk %d of instance %s did not return valid"
2780
                          " size information, ignoring", idx, instance.name)
2781
          continue
2782
        size = size >> 20
2783
        if size != disk.size:
2784
          self.LogInfo("Disk %d of instance %s has mismatched size,"
2785
                       " correcting: recorded %d, actual %d", idx,
2786
                       instance.name, disk.size, size)
2787
          disk.size = size
2788
          self.cfg.Update(instance, feedback_fn)
2789
          changed.append((instance.name, idx, size))
2790
        if self._EnsureChildSizes(disk):
2791
          self.cfg.Update(instance, feedback_fn)
2792
          changed.append((instance.name, idx, disk.size))
2793
    return changed
2794

    
2795

    
2796
class LUClusterRename(LogicalUnit):
2797
  """Rename the cluster.
2798

2799
  """
2800
  HPATH = "cluster-rename"
2801
  HTYPE = constants.HTYPE_CLUSTER
2802

    
2803
  def BuildHooksEnv(self):
2804
    """Build hooks env.
2805

2806
    """
2807
    return {
2808
      "OP_TARGET": self.cfg.GetClusterName(),
2809
      "NEW_NAME": self.op.name,
2810
      }
2811

    
2812
  def BuildHooksNodes(self):
2813
    """Build hooks nodes.
2814

2815
    """
2816
    return ([self.cfg.GetMasterNode()], self.cfg.GetNodeList())
2817

    
2818
  def CheckPrereq(self):
2819
    """Verify that the passed name is a valid one.
2820

2821
    """
2822
    hostname = netutils.GetHostname(name=self.op.name,
2823
                                    family=self.cfg.GetPrimaryIPFamily())
2824

    
2825
    new_name = hostname.name
2826
    self.ip = new_ip = hostname.ip
2827
    old_name = self.cfg.GetClusterName()
2828
    old_ip = self.cfg.GetMasterIP()
2829
    if new_name == old_name and new_ip == old_ip:
2830
      raise errors.OpPrereqError("Neither the name nor the IP address of the"
2831
                                 " cluster has changed",
2832
                                 errors.ECODE_INVAL)
2833
    if new_ip != old_ip:
2834
      if netutils.TcpPing(new_ip, constants.DEFAULT_NODED_PORT):
2835
        raise errors.OpPrereqError("The given cluster IP address (%s) is"
2836
                                   " reachable on the network" %
2837
                                   new_ip, errors.ECODE_NOTUNIQUE)
2838

    
2839
    self.op.name = new_name
2840

    
2841
  def Exec(self, feedback_fn):
2842
    """Rename the cluster.
2843

2844
    """
2845
    clustername = self.op.name
2846
    ip = self.ip
2847

    
2848
    # shutdown the master IP
2849
    master = self.cfg.GetMasterNode()
2850
    result = self.rpc.call_node_stop_master(master, False)
2851
    result.Raise("Could not disable the master role")
2852

    
2853
    try:
2854
      cluster = self.cfg.GetClusterInfo()
2855
      cluster.cluster_name = clustername
2856
      cluster.master_ip = ip
2857
      self.cfg.Update(cluster, feedback_fn)
2858

    
2859
      # update the known hosts file
2860
      ssh.WriteKnownHostsFile(self.cfg, constants.SSH_KNOWN_HOSTS_FILE)
2861
      node_list = self.cfg.GetOnlineNodeList()
2862
      try:
2863
        node_list.remove(master)
2864
      except ValueError:
2865
        pass
2866
      _UploadHelper(self, node_list, constants.SSH_KNOWN_HOSTS_FILE)
2867
    finally:
2868
      result = self.rpc.call_node_start_master(master, False, False)
2869
      msg = result.fail_msg
2870
      if msg:
2871
        self.LogWarning("Could not re-enable the master role on"
2872
                        " the master, please restart manually: %s", msg)
2873

    
2874
    return clustername
2875

    
2876

    
2877
class LUClusterSetParams(LogicalUnit):
2878
  """Change the parameters of the cluster.
2879

2880
  """
2881
  HPATH = "cluster-modify"
2882
  HTYPE = constants.HTYPE_CLUSTER
2883
  REQ_BGL = False
2884

    
2885
  def CheckArguments(self):
2886
    """Check parameters
2887

2888
    """
2889
    if self.op.uid_pool:
2890
      uidpool.CheckUidPool(self.op.uid_pool)
2891

    
2892
    if self.op.add_uids:
2893
      uidpool.CheckUidPool(self.op.add_uids)
2894

    
2895
    if self.op.remove_uids:
2896
      uidpool.CheckUidPool(self.op.remove_uids)
2897

    
2898
  def ExpandNames(self):
2899
    # FIXME: in the future maybe other cluster params won't require checking on
2900
    # all nodes to be modified.
2901
    self.needed_locks = {
2902
      locking.LEVEL_NODE: locking.ALL_SET,
2903
    }
2904
    self.share_locks[locking.LEVEL_NODE] = 1
2905

    
2906
  def BuildHooksEnv(self):
2907
    """Build hooks env.
2908

2909
    """
2910
    return {
2911
      "OP_TARGET": self.cfg.GetClusterName(),
2912
      "NEW_VG_NAME": self.op.vg_name,
2913
      }
2914

    
2915
  def BuildHooksNodes(self):
2916
    """Build hooks nodes.
2917

2918
    """
2919
    mn = self.cfg.GetMasterNode()
2920
    return ([mn], [mn])
2921

    
2922
  def CheckPrereq(self):
2923
    """Check prerequisites.
2924

2925
    This checks whether the given params don't conflict and
2926
    if the given volume group is valid.
2927

2928
    """
2929
    if self.op.vg_name is not None and not self.op.vg_name:
2930
      if self.cfg.HasAnyDiskOfType(constants.LD_LV):
2931
        raise errors.OpPrereqError("Cannot disable lvm storage while lvm-based"
2932
                                   " instances exist", errors.ECODE_INVAL)
2933

    
2934
    if self.op.drbd_helper is not None and not self.op.drbd_helper:
2935
      if self.cfg.HasAnyDiskOfType(constants.LD_DRBD8):
2936
        raise errors.OpPrereqError("Cannot disable drbd helper while"
2937
                                   " drbd-based instances exist",
2938
                                   errors.ECODE_INVAL)
2939

    
2940
    node_list = self.glm.list_owned(locking.LEVEL_NODE)
2941

    
2942
    # if vg_name not None, checks given volume group on all nodes
2943
    if self.op.vg_name:
2944
      vglist = self.rpc.call_vg_list(node_list)
2945
      for node in node_list:
2946
        msg = vglist[node].fail_msg
2947
        if msg:
2948
          # ignoring down node
2949
          self.LogWarning("Error while gathering data on node %s"
2950
                          " (ignoring node): %s", node, msg)
2951
          continue
2952
        vgstatus = utils.CheckVolumeGroupSize(vglist[node].payload,
2953
                                              self.op.vg_name,
2954
                                              constants.MIN_VG_SIZE)
2955
        if vgstatus:
2956
          raise errors.OpPrereqError("Error on node '%s': %s" %
2957
                                     (node, vgstatus), errors.ECODE_ENVIRON)
2958

    
2959
    if self.op.drbd_helper:
2960
      # checks given drbd helper on all nodes
2961
      helpers = self.rpc.call_drbd_helper(node_list)
2962
      for node in node_list:
2963
        ninfo = self.cfg.GetNodeInfo(node)
2964
        if ninfo.offline:
2965
          self.LogInfo("Not checking drbd helper on offline node %s", node)
2966
          continue
2967
        msg = helpers[node].fail_msg
2968
        if msg:
2969
          raise errors.OpPrereqError("Error checking drbd helper on node"
2970
                                     " '%s': %s" % (node, msg),
2971
                                     errors.ECODE_ENVIRON)
2972
        node_helper = helpers[node].payload
2973
        if node_helper != self.op.drbd_helper:
2974
          raise errors.OpPrereqError("Error on node '%s': drbd helper is %s" %
2975
                                     (node, node_helper), errors.ECODE_ENVIRON)
2976

    
2977
    self.cluster = cluster = self.cfg.GetClusterInfo()
2978
    # validate params changes
2979
    if self.op.beparams:
2980
      utils.ForceDictType(self.op.beparams, constants.BES_PARAMETER_TYPES)
2981
      self.new_beparams = cluster.SimpleFillBE(self.op.beparams)
2982

    
2983
    if self.op.ndparams:
2984
      utils.ForceDictType(self.op.ndparams, constants.NDS_PARAMETER_TYPES)
2985
      self.new_ndparams = cluster.SimpleFillND(self.op.ndparams)
2986

    
2987
      # TODO: we need a more general way to handle resetting
2988
      # cluster-level parameters to default values
2989
      if self.new_ndparams["oob_program"] == "":
2990
        self.new_ndparams["oob_program"] = \
2991
            constants.NDC_DEFAULTS[constants.ND_OOB_PROGRAM]
2992

    
2993
    if self.op.nicparams:
2994
      utils.ForceDictType(self.op.nicparams, constants.NICS_PARAMETER_TYPES)
2995
      self.new_nicparams = cluster.SimpleFillNIC(self.op.nicparams)
2996
      objects.NIC.CheckParameterSyntax(self.new_nicparams)
2997
      nic_errors = []
2998

    
2999
      # check all instances for consistency
3000
      for instance in self.cfg.GetAllInstancesInfo().values():
3001
        for nic_idx, nic in enumerate(instance.nics):
3002
          params_copy = copy.deepcopy(nic.nicparams)
3003
          params_filled = objects.FillDict(self.new_nicparams, params_copy)
3004

    
3005
          # check parameter syntax
3006
          try:
3007
            objects.NIC.CheckParameterSyntax(params_filled)
3008
          except errors.ConfigurationError, err:
3009
            nic_errors.append("Instance %s, nic/%d: %s" %
3010
                              (instance.name, nic_idx, err))
3011

    
3012
          # if we're moving instances to routed, check that they have an ip
3013
          target_mode = params_filled[constants.NIC_MODE]
3014
          if target_mode == constants.NIC_MODE_ROUTED and not nic.ip:
3015
            nic_errors.append("Instance %s, nic/%d: routed NIC with no ip"
3016
                              " address" % (instance.name, nic_idx))
3017
      if nic_errors:
3018
        raise errors.OpPrereqError("Cannot apply the change, errors:\n%s" %
3019
                                   "\n".join(nic_errors))
3020

    
3021
    # hypervisor list/parameters
3022
    self.new_hvparams = new_hvp = objects.FillDict(cluster.hvparams, {})
3023
    if self.op.hvparams:
3024
      for hv_name, hv_dict in self.op.hvparams.items():
3025
        if hv_name not in self.new_hvparams:
3026
          self.new_hvparams[hv_name] = hv_dict
3027
        else:
3028
          self.new_hvparams[hv_name].update(hv_dict)
3029

    
3030
    # os hypervisor parameters
3031
    self.new_os_hvp = objects.FillDict(cluster.os_hvp, {})
3032
    if self.op.os_hvp:
3033
      for os_name, hvs in self.op.os_hvp.items():
3034
        if os_name not in self.new_os_hvp:
3035
          self.new_os_hvp[os_name] = hvs
3036
        else:
3037
          for hv_name, hv_dict in hvs.items():
3038
            if hv_name not in self.new_os_hvp[os_name]:
3039
              self.new_os_hvp[os_name][hv_name] = hv_dict
3040
            else:
3041
              self.new_os_hvp[os_name][hv_name].update(hv_dict)
3042

    
3043
    # os parameters
3044
    self.new_osp = objects.FillDict(cluster.osparams, {})
3045
    if self.op.osparams:
3046
      for os_name, osp in self.op.osparams.items():
3047
        if os_name not in self.new_osp:
3048
          self.new_osp[os_name] = {}
3049

    
3050
        self.new_osp[os_name] = _GetUpdatedParams(self.new_osp[os_name], osp,
3051
                                                  use_none=True)
3052

    
3053
        if not self.new_osp[os_name]:
3054
          # we removed all parameters
3055
          del self.new_osp[os_name]
3056
        else:
3057
          # check the parameter validity (remote check)
3058
          _CheckOSParams(self, False, [self.cfg.GetMasterNode()],
3059
                         os_name, self.new_osp[os_name])
3060

    
3061
    # changes to the hypervisor list
3062
    if self.op.enabled_hypervisors is not None:
3063
      self.hv_list = self.op.enabled_hypervisors
3064
      for hv in self.hv_list:
3065
        # if the hypervisor doesn't already exist in the cluster
3066
        # hvparams, we initialize it to empty, and then (in both
3067
        # cases) we make sure to fill the defaults, as we might not
3068
        # have a complete defaults list if the hypervisor wasn't
3069
        # enabled before
3070
        if hv not in new_hvp:
3071
          new_hvp[hv] = {}
3072
        new_hvp[hv] = objects.FillDict(constants.HVC_DEFAULTS[hv], new_hvp[hv])
3073
        utils.ForceDictType(new_hvp[hv], constants.HVS_PARAMETER_TYPES)
3074
    else:
3075
      self.hv_list = cluster.enabled_hypervisors
3076

    
3077
    if self.op.hvparams or self.op.enabled_hypervisors is not None:
3078
      # either the enabled list has changed, or the parameters have, validate
3079
      for hv_name, hv_params in self.new_hvparams.items():
3080
        if ((self.op.hvparams and hv_name in self.op.hvparams) or
3081
            (self.op.enabled_hypervisors and
3082
             hv_name in self.op.enabled_hypervisors)):
3083
          # either this is a new hypervisor, or its parameters have changed
3084
          hv_class = hypervisor.GetHypervisor(hv_name)
3085
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3086
          hv_class.CheckParameterSyntax(hv_params)
3087
          _CheckHVParams(self, node_list, hv_name, hv_params)
3088

    
3089
    if self.op.os_hvp:
3090
      # no need to check any newly-enabled hypervisors, since the
3091
      # defaults have already been checked in the above code-block
3092
      for os_name, os_hvp in self.new_os_hvp.items():
3093
        for hv_name, hv_params in os_hvp.items():
3094
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3095
          # we need to fill in the new os_hvp on top of the actual hv_p
3096
          cluster_defaults = self.new_hvparams.get(hv_name, {})
3097
          new_osp = objects.FillDict(cluster_defaults, hv_params)
3098
          hv_class = hypervisor.GetHypervisor(hv_name)
3099
          hv_class.CheckParameterSyntax(new_osp)
3100
          _CheckHVParams(self, node_list, hv_name, new_osp)
3101

    
3102
    if self.op.default_iallocator:
3103
      alloc_script = utils.FindFile(self.op.default_iallocator,
3104
                                    constants.IALLOCATOR_SEARCH_PATH,
3105
                                    os.path.isfile)
3106
      if alloc_script is None:
3107
        raise errors.OpPrereqError("Invalid default iallocator script '%s'"
3108
                                   " specified" % self.op.default_iallocator,
3109
                                   errors.ECODE_INVAL)
3110

    
3111
  def Exec(self, feedback_fn):
3112
    """Change the parameters of the cluster.
3113

3114
    """
3115
    if self.op.vg_name is not None:
3116
      new_volume = self.op.vg_name
3117
      if not new_volume:
3118
        new_volume = None
3119
      if new_volume != self.cfg.GetVGName():
3120
        self.cfg.SetVGName(new_volume)
3121
      else:
3122
        feedback_fn("Cluster LVM configuration already in desired"
3123
                    " state, not changing")
3124
    if self.op.drbd_helper is not None:
3125
      new_helper = self.op.drbd_helper
3126
      if not new_helper:
3127
        new_helper = None
3128
      if new_helper != self.cfg.GetDRBDHelper():
3129
        self.cfg.SetDRBDHelper(new_helper)
3130
      else:
3131
        feedback_fn("Cluster DRBD helper already in desired state,"
3132
                    " not changing")
3133
    if self.op.hvparams:
3134
      self.cluster.hvparams = self.new_hvparams
3135
    if self.op.os_hvp:
3136
      self.cluster.os_hvp = self.new_os_hvp
3137
    if self.op.enabled_hypervisors is not None:
3138
      self.cluster.hvparams = self.new_hvparams
3139
      self.cluster.enabled_hypervisors = self.op.enabled_hypervisors
3140
    if self.op.beparams:
3141
      self.cluster.beparams[constants.PP_DEFAULT] = self.new_beparams
3142
    if self.op.nicparams:
3143
      self.cluster.nicparams[constants.PP_DEFAULT] = self.new_nicparams
3144
    if self.op.osparams:
3145
      self.cluster.osparams = self.new_osp
3146
    if self.op.ndparams:
3147
      self.cluster.ndparams = self.new_ndparams
3148

    
3149
    if self.op.candidate_pool_size is not None:
3150
      self.cluster.candidate_pool_size = self.op.candidate_pool_size
3151
      # we need to update the pool size here, otherwise the save will fail
3152
      _AdjustCandidatePool(self, [])
3153

    
3154
    if self.op.maintain_node_health is not None:
3155
      self.cluster.maintain_node_health = self.op.maintain_node_health
3156

    
3157
    if self.op.prealloc_wipe_disks is not None:
3158
      self.cluster.prealloc_wipe_disks = self.op.prealloc_wipe_disks
3159

    
3160
    if self.op.add_uids is not None:
3161
      uidpool.AddToUidPool(self.cluster.uid_pool, self.op.add_uids)
3162

    
3163
    if self.op.remove_uids is not None:
3164
      uidpool.RemoveFromUidPool(self.cluster.uid_pool, self.op.remove_uids)
3165

    
3166
    if self.op.uid_pool is not None:
3167
      self.cluster.uid_pool = self.op.uid_pool
3168

    
3169
    if self.op.default_iallocator is not None:
3170
      self.cluster.default_iallocator = self.op.default_iallocator
3171

    
3172
    if self.op.reserved_lvs is not None:
3173
      self.cluster.reserved_lvs = self.op.reserved_lvs
3174

    
3175
    def helper_os(aname, mods, desc):
3176
      desc += " OS list"
3177
      lst = getattr(self.cluster, aname)
3178
      for key, val in mods:
3179
        if key == constants.DDM_ADD:
3180
          if val in lst:
3181
            feedback_fn("OS %s already in %s, ignoring" % (val, desc))
3182
          else:
3183
            lst.append(val)
3184
        elif key == constants.DDM_REMOVE:
3185
          if val in lst:
3186
            lst.remove(val)
3187
          else:
3188
            feedback_fn("OS %s not found in %s, ignoring" % (val, desc))
3189
        else:
3190
          raise errors.ProgrammerError("Invalid modification '%s'" % key)
3191

    
3192
    if self.op.hidden_os:
3193
      helper_os("hidden_os", self.op.hidden_os, "hidden")
3194

    
3195
    if self.op.blacklisted_os:
3196
      helper_os("blacklisted_os", self.op.blacklisted_os, "blacklisted")
3197

    
3198
    if self.op.master_netdev:
3199
      master = self.cfg.GetMasterNode()
3200
      feedback_fn("Shutting down master ip on the current netdev (%s)" %
3201
                  self.cluster.master_netdev)
3202
      result = self.rpc.call_node_stop_master(master, False)
3203
      result.Raise("Could not disable the master ip")
3204
      feedback_fn("Changing master_netdev from %s to %s" %
3205
                  (self.cluster.master_netdev, self.op.master_netdev))
3206
      self.cluster.master_netdev = self.op.master_netdev
3207

    
3208
    self.cfg.Update(self.cluster, feedback_fn)
3209

    
3210
    if self.op.master_netdev:
3211
      feedback_fn("Starting the master ip on the new master netdev (%s)" %
3212
                  self.op.master_netdev)
3213
      result = self.rpc.call_node_start_master(master, False, False)
3214
      if result.fail_msg:
3215
        self.LogWarning("Could not re-enable the master ip on"
3216
                        " the master, please restart manually: %s",
3217
                        result.fail_msg)
3218

    
3219

    
3220
def _UploadHelper(lu, nodes, fname):
3221
  """Helper for uploading a file and showing warnings.
3222

3223
  """
3224
  if os.path.exists(fname):
3225
    result = lu.rpc.call_upload_file(nodes, fname)
3226
    for to_node, to_result in result.items():
3227
      msg = to_result.fail_msg
3228
      if msg:
3229
        msg = ("Copy of file %s to node %s failed: %s" %
3230
               (fname, to_node, msg))
3231
        lu.proc.LogWarning(msg)
3232

    
3233

    
3234
def _ComputeAncillaryFiles(cluster, redist):
3235
  """Compute files external to Ganeti which need to be consistent.
3236

3237
  @type redist: boolean
3238
  @param redist: Whether to include files which need to be redistributed
3239

3240
  """
3241
  # Compute files for all nodes
3242
  files_all = set([
3243
    constants.SSH_KNOWN_HOSTS_FILE,
3244
    constants.CONFD_HMAC_KEY,
3245
    constants.CLUSTER_DOMAIN_SECRET_FILE,
3246
    ])
3247

    
3248
  if not redist:
3249
    files_all.update(constants.ALL_CERT_FILES)
3250
    files_all.update(ssconf.SimpleStore().GetFileList())
3251

    
3252
  if cluster.modify_etc_hosts:
3253
    files_all.add(constants.ETC_HOSTS)
3254

    
3255
  # Files which must either exist on all nodes or on none
3256
  files_all_opt = set([
3257
    constants.RAPI_USERS_FILE,
3258
    ])
3259

    
3260
  # Files which should only be on master candidates
3261
  files_mc = set()
3262
  if not redist:
3263
    files_mc.add(constants.CLUSTER_CONF_FILE)
3264

    
3265
  # Files which should only be on VM-capable nodes
3266
  files_vm = set(filename
3267
    for hv_name in cluster.enabled_hypervisors
3268
    for filename in hypervisor.GetHypervisor(hv_name).GetAncillaryFiles())
3269

    
3270
  # Filenames must be unique
3271
  assert (len(files_all | files_all_opt | files_mc | files_vm) ==
3272
          sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
3273
         "Found file listed in more than one file list"
3274

    
3275
  return (files_all, files_all_opt, files_mc, files_vm)
3276

    
3277

    
3278
def _RedistributeAncillaryFiles(lu, additional_nodes=None, additional_vm=True):
3279
  """Distribute additional files which are part of the cluster configuration.
3280

3281
  ConfigWriter takes care of distributing the config and ssconf files, but
3282
  there are more files which should be distributed to all nodes. This function
3283
  makes sure those are copied.
3284

3285
  @param lu: calling logical unit
3286
  @param additional_nodes: list of nodes not in the config to distribute to
3287
  @type additional_vm: boolean
3288
  @param additional_vm: whether the additional nodes are vm-capable or not
3289

3290
  """
3291
  # Gather target nodes
3292
  cluster = lu.cfg.GetClusterInfo()
3293
  master_info = lu.cfg.GetNodeInfo(lu.cfg.GetMasterNode())
3294

    
3295
  online_nodes = lu.cfg.GetOnlineNodeList()
3296
  vm_nodes = lu.cfg.GetVmCapableNodeList()
3297

    
3298
  if additional_nodes is not None:
3299
    online_nodes.extend(additional_nodes)
3300
    if additional_vm:
3301
      vm_nodes.extend(additional_nodes)
3302

    
3303
  # Never distribute to master node
3304
  for nodelist in [online_nodes, vm_nodes]:
3305
    if master_info.name in nodelist:
3306
      nodelist.remove(master_info.name)
3307

    
3308
  # Gather file lists
3309
  (files_all, files_all_opt, files_mc, files_vm) = \
3310
    _ComputeAncillaryFiles(cluster, True)
3311

    
3312
  # Never re-distribute configuration file from here
3313
  assert not (constants.CLUSTER_CONF_FILE in files_all or
3314
              constants.CLUSTER_CONF_FILE in files_vm)
3315
  assert not files_mc, "Master candidates not handled in this function"
3316

    
3317
  filemap = [
3318
    (online_nodes, files_all),
3319
    (online_nodes, files_all_opt),
3320
    (vm_nodes, files_vm),
3321
    ]
3322

    
3323
  # Upload the files
3324
  for (node_list, files) in filemap:
3325
    for fname in files:
3326
      _UploadHelper(lu, node_list, fname)
3327

    
3328

    
3329
class LUClusterRedistConf(NoHooksLU):
3330
  """Force the redistribution of cluster configuration.
3331

3332
  This is a very simple LU.
3333

3334
  """
3335
  REQ_BGL = False
3336

    
3337
  def ExpandNames(self):
3338
    self.needed_locks = {
3339
      locking.LEVEL_NODE: locking.ALL_SET,
3340
    }
3341
    self.share_locks[locking.LEVEL_NODE] = 1
3342

    
3343
  def Exec(self, feedback_fn):
3344
    """Redistribute the configuration.
3345

3346
    """
3347
    self.cfg.Update(self.cfg.GetClusterInfo(), feedback_fn)
3348
    _RedistributeAncillaryFiles(self)
3349

    
3350

    
3351
def _WaitForSync(lu, instance, disks=None, oneshot=False):
3352
  """Sleep and poll for an instance's disk to sync.
3353

3354
  """
3355
  if not instance.disks or disks is not None and not disks:
3356
    return True
3357

    
3358
  disks = _ExpandCheckDisks(instance, disks)
3359

    
3360
  if not oneshot:
3361
    lu.proc.LogInfo("Waiting for instance %s to sync disks." % instance.name)
3362

    
3363
  node = instance.primary_node
3364

    
3365
  for dev in disks:
3366
    lu.cfg.SetDiskID(dev, node)
3367

    
3368
  # TODO: Convert to utils.Retry
3369

    
3370
  retries = 0
3371
  degr_retries = 10 # in seconds, as we sleep 1 second each time
3372
  while True:
3373
    max_time = 0
3374
    done = True
3375
    cumul_degraded = False
3376
    rstats = lu.rpc.call_blockdev_getmirrorstatus(node, disks)
3377
    msg = rstats.fail_msg
3378
    if msg:
3379
      lu.LogWarning("Can't get any data from node %s: %s", node, msg)
3380
      retries += 1
3381
      if retries >= 10:
3382
        raise errors.RemoteError("Can't contact node %s for mirror data,"
3383
                                 " aborting." % node)
3384
      time.sleep(6)
3385
      continue
3386
    rstats = rstats.payload
3387
    retries = 0
3388
    for i, mstat in enumerate(rstats):
3389
      if mstat is None:
3390
        lu.LogWarning("Can't compute data for node %s/%s",
3391
                           node, disks[i].iv_name)
3392
        continue
3393

    
3394
      cumul_degraded = (cumul_degraded or
3395
                        (mstat.is_degraded and mstat.sync_percent is None))
3396
      if mstat.sync_percent is not None:
3397
        done = False
3398
        if mstat.estimated_time is not None:
3399
          rem_time = ("%s remaining (estimated)" %
3400
                      utils.FormatSeconds(mstat.estimated_time))
3401
          max_time = mstat.estimated_time
3402
        else:
3403
          rem_time = "no time estimate"
3404
        lu.proc.LogInfo("- device %s: %5.2f%% done, %s" %
3405
                        (disks[i].iv_name, mstat.sync_percent, rem_time))
3406

    
3407
    # if we're done but degraded, let's do a few small retries, to
3408
    # make sure we see a stable and not transient situation; therefore
3409
    # we force restart of the loop
3410
    if (done or oneshot) and cumul_degraded and degr_retries > 0:
3411
      logging.info("Degraded disks found, %d retries left", degr_retries)
3412
      degr_retries -= 1
3413
      time.sleep(1)
3414
      continue
3415

    
3416
    if done or oneshot:
3417
      break
3418

    
3419
    time.sleep(min(60, max_time))
3420

    
3421
  if done:
3422
    lu.proc.LogInfo("Instance %s's disks are in sync." % instance.name)
3423
  return not cumul_degraded
3424

    
3425

    
3426
def _CheckDiskConsistency(lu, dev, node, on_primary, ldisk=False):
3427
  """Check that mirrors are not degraded.
3428

3429
  The ldisk parameter, if True, will change the test from the
3430
  is_degraded attribute (which represents overall non-ok status for
3431
  the device(s)) to the ldisk (representing the local storage status).
3432

3433
  """
3434
  lu.cfg.SetDiskID(dev, node)
3435

    
3436
  result = True
3437

    
3438
  if on_primary or dev.AssembleOnSecondary():
3439
    rstats = lu.rpc.call_blockdev_find(node, dev)
3440
    msg = rstats.fail_msg
3441
    if msg:
3442
      lu.LogWarning("Can't find disk on node %s: %s", node, msg)
3443
      result = False
3444
    elif not rstats.payload:
3445
      lu.LogWarning("Can't find disk on node %s", node)
3446
      result = False
3447
    else:
3448
      if ldisk:
3449
        result = result and rstats.payload.ldisk_status == constants.LDS_OKAY
3450
      else:
3451
        result = result and not rstats.payload.is_degraded
3452

    
3453
  if dev.children:
3454
    for child in dev.children:
3455
      result = result and _CheckDiskConsistency(lu, child, node, on_primary)
3456

    
3457
  return result
3458

    
3459

    
3460
class LUOobCommand(NoHooksLU):
3461
  """Logical unit for OOB handling.
3462

3463
  """
3464
  REG_BGL = False
3465
  _SKIP_MASTER = (constants.OOB_POWER_OFF, constants.OOB_POWER_CYCLE)
3466

    
3467
  def ExpandNames(self):
3468
    """Gather locks we need.
3469

3470
    """
3471
    if self.op.node_names:
3472
      self.op.node_names = _GetWantedNodes(self, self.op.node_names)
3473
      lock_names = self.op.node_names
3474
    else:
3475
      lock_names = locking.ALL_SET
3476

    
3477
    self.needed_locks = {
3478
      locking.LEVEL_NODE: lock_names,
3479
      }
3480

    
3481
  def CheckPrereq(self):
3482
    """Check prerequisites.
3483

3484
    This checks:
3485
     - the node exists in the configuration
3486
     - OOB is supported
3487

3488
    Any errors are signaled by raising errors.OpPrereqError.
3489

3490
    """
3491
    self.nodes = []
3492
    self.master_node = self.cfg.GetMasterNode()
3493

    
3494
    assert self.op.power_delay >= 0.0
3495

    
3496
    if self.op.node_names:
3497
      if (self.op.command in self._SKIP_MASTER and
3498
          self.master_node in self.op.node_names):
3499
        master_node_obj = self.cfg.GetNodeInfo(self.master_node)
3500
        master_oob_handler = _SupportsOob(self.cfg, master_node_obj)
3501

    
3502
        if master_oob_handler:
3503
          additional_text = ("run '%s %s %s' if you want to operate on the"
3504
                             " master regardless") % (master_oob_handler,
3505
                                                      self.op.command,
3506
                                                      self.master_node)
3507
        else:
3508
          additional_text = "it does not support out-of-band operations"
3509

    
3510
        raise errors.OpPrereqError(("Operating on the master node %s is not"
3511
                                    " allowed for %s; %s") %
3512
                                   (self.master_node, self.op.command,
3513
                                    additional_text), errors.ECODE_INVAL)
3514
    else:
3515
      self.op.node_names = self.cfg.GetNodeList()
3516
      if self.op.command in self._SKIP_MASTER:
3517
        self.op.node_names.remove(self.master_node)
3518

    
3519
    if self.op.command in self._SKIP_MASTER:
3520
      assert self.master_node not in self.op.node_names
3521

    
3522
    for node_name in self.op.node_names:
3523
      node = self.cfg.GetNodeInfo(node_name)
3524

    
3525
      if node is None:
3526
        raise errors.OpPrereqError("Node %s not found" % node_name,
3527
                                   errors.ECODE_NOENT)
3528
      else:
3529
        self.nodes.append(node)
3530

    
3531
      if (not self.op.ignore_status and
3532
          (self.op.command == constants.OOB_POWER_OFF and not node.offline)):
3533
        raise errors.OpPrereqError(("Cannot power off node %s because it is"
3534
                                    " not marked offline") % node_name,
3535
                                   errors.ECODE_STATE)
3536

    
3537
  def Exec(self, feedback_fn):
3538
    """Execute OOB and return result if we expect any.
3539

3540
    """
3541
    master_node = self.master_node
3542
    ret = []
3543

    
3544
    for idx, node in enumerate(utils.NiceSort(self.nodes,
3545
                                              key=lambda node: node.name)):
3546
      node_entry = [(constants.RS_NORMAL, node.name)]
3547
      ret.append(node_entry)
3548

    
3549
      oob_program = _SupportsOob(self.cfg, node)
3550

    
3551
      if not oob_program:
3552
        node_entry.append((constants.RS_UNAVAIL, None))
3553
        continue
3554

    
3555
      logging.info("Executing out-of-band command '%s' using '%s' on %s",
3556
                   self.op.command, oob_program, node.name)
3557
      result = self.rpc.call_run_oob(master_node, oob_program,
3558
                                     self.op.command, node.name,
3559
                                     self.op.timeout)
3560

    
3561
      if result.fail_msg:
3562
        self.LogWarning("Out-of-band RPC failed on node '%s': %s",
3563
                        node.name, result.fail_msg)
3564
        node_entry.append((constants.RS_NODATA, None))
3565
      else:
3566
        try:
3567
          self._CheckPayload(result)
3568
        except errors.OpExecError, err:
3569
          self.LogWarning("Payload returned by node '%s' is not valid: %s",
3570
                          node.name, err)
3571
          node_entry.append((constants.RS_NODATA, None))
3572
        else:
3573
          if self.op.command == constants.OOB_HEALTH:
3574
            # For health we should log important events
3575
            for item, status in result.payload:
3576
              if status in [constants.OOB_STATUS_WARNING,
3577
                            constants.OOB_STATUS_CRITICAL]:
3578
                self.LogWarning("Item '%s' on node '%s' has status '%s'",
3579
                                item, node.name, status)
3580

    
3581
          if self.op.command == constants.OOB_POWER_ON:
3582
            node.powered = True
3583
          elif self.op.command == constants.OOB_POWER_OFF:
3584
            node.powered = False
3585
          elif self.op.command == constants.OOB_POWER_STATUS:
3586
            powered = result.payload[constants.OOB_POWER_STATUS_POWERED]
3587
            if powered != node.powered:
3588
              logging.warning(("Recorded power state (%s) of node '%s' does not"
3589
                               " match actual power state (%s)"), node.powered,
3590
                              node.name, powered)
3591

    
3592
          # For configuration changing commands we should update the node
3593
          if self.op.command in (constants.OOB_POWER_ON,
3594
                                 constants.OOB_POWER_OFF):
3595
            self.cfg.Update(node, feedback_fn)
3596

    
3597
          node_entry.append((constants.RS_NORMAL, result.payload))
3598

    
3599
          if (self.op.command == constants.OOB_POWER_ON and
3600
              idx < len(self.nodes) - 1):
3601
            time.sleep(self.op.power_delay)
3602

    
3603
    return ret
3604

    
3605
  def _CheckPayload(self, result):
3606
    """Checks if the payload is valid.
3607

3608
    @param result: RPC result
3609
    @raises errors.OpExecError: If payload is not valid
3610

3611
    """
3612
    errs = []
3613
    if self.op.command == constants.OOB_HEALTH:
3614
      if not isinstance(result.payload, list):
3615
        errs.append("command 'health' is expected to return a list but got %s" %
3616
                    type(result.payload))
3617
      else:
3618
        for item, status in result.payload:
3619
          if status not in constants.OOB_STATUSES:
3620
            errs.append("health item '%s' has invalid status '%s'" %
3621
                        (item, status))
3622

    
3623
    if self.op.command == constants.OOB_POWER_STATUS:
3624
      if not isinstance(result.payload, dict):
3625
        errs.append("power-status is expected to return a dict but got %s" %
3626
                    type(result.payload))
3627

    
3628
    if self.op.command in [
3629
        constants.OOB_POWER_ON,
3630
        constants.OOB_POWER_OFF,
3631
        constants.OOB_POWER_CYCLE,
3632
        ]:
3633
      if result.payload is not None:
3634
        errs.append("%s is expected to not return payload but got '%s'" %
3635
                    (self.op.command, result.payload))
3636

    
3637
    if errs:
3638
      raise errors.OpExecError("Check of out-of-band payload failed due to %s" %
3639
                               utils.CommaJoin(errs))
3640

    
3641
class _OsQuery(_QueryBase):
3642
  FIELDS = query.OS_FIELDS
3643

    
3644
  def ExpandNames(self, lu):
3645
    # Lock all nodes in shared mode
3646
    # Temporary removal of locks, should be reverted later
3647
    # TODO: reintroduce locks when they are lighter-weight
3648
    lu.needed_locks = {}
3649
    #self.share_locks[locking.LEVEL_NODE] = 1
3650
    #self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET
3651

    
3652
    # The following variables interact with _QueryBase._GetNames
3653
    if self.names:
3654
      self.wanted = self.names
3655
    else:
3656
      self.wanted = locking.ALL_SET
3657

    
3658
    self.do_locking = self.use_locking
3659

    
3660
  def DeclareLocks(self, lu, level):
3661
    pass
3662

    
3663
  @staticmethod
3664
  def _DiagnoseByOS(rlist):
3665
    """Remaps a per-node return list into an a per-os per-node dictionary
3666

3667
    @param rlist: a map with node names as keys and OS objects as values
3668

3669
    @rtype: dict
3670
    @return: a dictionary with osnames as keys and as value another
3671
        map, with nodes as keys and tuples of (path, status, diagnose,
3672
        variants, parameters, api_versions) as values, eg::
3673

3674
          {"debian-etch": {"node1": [(/usr/lib/..., True, "", [], []),
3675
                                     (/srv/..., False, "invalid api")],
3676
                           "node2": [(/srv/..., True, "", [], [])]}
3677
          }
3678

3679
    """
3680
    all_os = {}
3681
    # we build here the list of nodes that didn't fail the RPC (at RPC
3682
    # level), so that nodes with a non-responding node daemon don't
3683
    # make all OSes invalid
3684
    good_nodes = [node_name for node_name in rlist
3685
                  if not rlist[node_name].fail_msg]
3686
    for node_name, nr in rlist.items():
3687
      if nr.fail_msg or not nr.payload:
3688
        continue
3689
      for (name, path, status, diagnose, variants,
3690
           params, api_versions) in nr.payload:
3691
        if name not in all_os:
3692
          # build a list of nodes for this os containing empty lists
3693
          # for each node in node_list
3694
          all_os[name] = {}
3695
          for nname in good_nodes:
3696
            all_os[name][nname] = []
3697
        # convert params from [name, help] to (name, help)
3698
        params = [tuple(v) for v in params]
3699
        all_os[name][node_name].append((path, status, diagnose,
3700
                                        variants, params, api_versions))
3701
    return all_os
3702

    
3703
  def _GetQueryData(self, lu):
3704
    """Computes the list of nodes and their attributes.
3705

3706
    """
3707
    # Locking is not used
3708
    assert not (compat.any(lu.glm.is_owned(level)
3709
                           for level in locking.LEVELS
3710
                           if level != locking.LEVEL_CLUSTER) or
3711
                self.do_locking or self.use_locking)
3712

    
3713
    valid_nodes = [node.name
3714
                   for node in lu.cfg.GetAllNodesInfo().values()
3715
                   if not node.offline and node.vm_capable]
3716
    pol = self._DiagnoseByOS(lu.rpc.call_os_diagnose(valid_nodes))
3717
    cluster = lu.cfg.GetClusterInfo()
3718

    
3719
    data = {}
3720

    
3721
    for (os_name, os_data) in pol.items():
3722
      info = query.OsInfo(name=os_name, valid=True, node_status=os_data,
3723
                          hidden=(os_name in cluster.hidden_os),
3724
                          blacklisted=(os_name in cluster.blacklisted_os))
3725

    
3726
      variants = set()
3727
      parameters = set()
3728
      api_versions = set()
3729

    
3730
      for idx, osl in enumerate(os_data.values()):
3731
        info.valid = bool(info.valid and osl and osl[0][1])
3732
        if not info.valid:
3733
          break
3734

    
3735
        (node_variants, node_params, node_api) = osl[0][3:6]
3736
        if idx == 0:
3737
          # First entry
3738
          variants.update(node_variants)
3739
          parameters.update(node_params)
3740
          api_versions.update(node_api)
3741
        else:
3742
          # Filter out inconsistent values
3743
          variants.intersection_update(node_variants)
3744
          parameters.intersection_update(node_params)
3745
          api_versions.intersection_update(node_api)
3746

    
3747
      info.variants = list(variants)
3748
      info.parameters = list(parameters)
3749
      info.api_versions = list(api_versions)
3750

    
3751
      data[os_name] = info
3752

    
3753
    # Prepare data in requested order
3754
    return [data[name] for name in self._GetNames(lu, pol.keys(), None)
3755
            if name in data]
3756

    
3757

    
3758
class LUOsDiagnose(NoHooksLU):
3759
  """Logical unit for OS diagnose/query.
3760

3761
  """
3762
  REQ_BGL = False
3763

    
3764
  @staticmethod
3765
  def _BuildFilter(fields, names):
3766
    """Builds a filter for querying OSes.
3767

3768
    """
3769
    name_filter = qlang.MakeSimpleFilter("name", names)
3770

    
3771
    # Legacy behaviour: Hide hidden, blacklisted or invalid OSes if the
3772
    # respective field is not requested
3773
    status_filter = [[qlang.OP_NOT, [qlang.OP_TRUE, fname]]
3774
                     for fname in ["hidden", "blacklisted"]
3775
                     if fname not in fields]
3776
    if "valid" not in fields:
3777
      status_filter.append([qlang.OP_TRUE, "valid"])
3778

    
3779
    if status_filter:
3780
      status_filter.insert(0, qlang.OP_AND)
3781
    else:
3782
      status_filter = None
3783

    
3784
    if name_filter and status_filter:
3785
      return [qlang.OP_AND, name_filter, status_filter]
3786
    elif name_filter:
3787
      return name_filter
3788
    else:
3789
      return status_filter
3790

    
3791
  def CheckArguments(self):
3792
    self.oq = _OsQuery(self._BuildFilter(self.op.output_fields, self.op.names),
3793
                       self.op.output_fields, False)
3794

    
3795
  def ExpandNames(self):
3796
    self.oq.ExpandNames(self)
3797

    
3798
  def Exec(self, feedback_fn):
3799
    return self.oq.OldStyleQuery(self)
3800

    
3801

    
3802
class LUNodeRemove(LogicalUnit):
3803
  """Logical unit for removing a node.
3804

3805
  """
3806
  HPATH = "node-remove"
3807
  HTYPE = constants.HTYPE_NODE
3808

    
3809
  def BuildHooksEnv(self):
3810
    """Build hooks env.
3811

3812
    This doesn't run on the target node in the pre phase as a failed
3813
    node would then be impossible to remove.
3814

3815
    """
3816
    return {
3817
      "OP_TARGET": self.op.node_name,
3818
      "NODE_NAME": self.op.node_name,
3819
      }
3820

    
3821
  def BuildHooksNodes(self):
3822
    """Build hooks nodes.
3823

3824
    """
3825
    all_nodes = self.cfg.GetNodeList()
3826
    try:
3827
      all_nodes.remove(self.op.node_name)
3828
    except ValueError:
3829
      logging.warning("Node '%s', which is about to be removed, was not found"
3830
                      " in the list of all nodes", self.op.node_name)
3831
    return (all_nodes, all_nodes)
3832

    
3833
  def CheckPrereq(self):
3834
    """Check prerequisites.
3835

3836
    This checks:
3837
     - the node exists in the configuration
3838
     - it does not have primary or secondary instances
3839
     - it's not the master
3840

3841
    Any errors are signaled by raising errors.OpPrereqError.
3842

3843
    """
3844
    self.op.node_name = _ExpandNodeName(self.cfg, self.op.node_name)
3845
    node = self.cfg.GetNodeInfo(self.op.node_name)
3846
    assert node is not None
3847

    
3848
    instance_list = self.cfg.GetInstanceList()
3849

    
3850
    masternode = self.cfg.GetMasterNode()
3851
    if node.name == masternode:
3852
      raise errors.OpPrereqError("Node is the master node, failover to another"
3853
                                 " node is required", errors.ECODE_INVAL)
3854

    
3855
    for instance_name in instance_list:
3856
      instance = self.cfg.GetInstanceInfo(instance_name)
3857
      if node.name in instance.all_nodes:
3858
        raise errors.OpPrereqError("Instance %s is still running on the node,"
3859
                                   " please remove first" % instance_name,
3860
                                   errors.ECODE_INVAL)
3861
    self.op.node_name = node.name
3862
    self.node = node
3863

    
3864
  def Exec(self, feedback_fn):
3865
    """Removes the node from the cluster.
3866

3867
    """
3868
    node = self.node
3869
    logging.info("Stopping the node daemon and removing configs from node %s",
3870
                 node.name)
3871

    
3872
    modify_ssh_setup = self.cfg.GetClusterInfo().modify_ssh_setup
3873

    
3874
    # Promote nodes to master candidate as needed
3875
    _AdjustCandidatePool(self, exceptions=[node.name])
3876
    self.context.RemoveNode(node.name)
3877

    
3878
    # Run post hooks on the node before it's removed
3879
    _RunPostHook(self, node.name)
3880

    
3881
    result = self.rpc.call_node_leave_cluster(node.name, modify_ssh_setup)
3882
    msg = result.fail_msg
3883
    if msg:
3884
      self.LogWarning("Errors encountered on the remote node while leaving"
3885
                      " the cluster: %s", msg)
3886

    
3887
    # Remove node from our /etc/hosts
3888
    if self.cfg.GetClusterInfo().modify_etc_hosts:
3889
      master_node = self.cfg.GetMasterNode()
3890
      result = self.rpc.call_etc_hosts_modify(master_node,
3891
                                              constants.ETC_HOSTS_REMOVE,
3892
                                              node.name, None)
3893
      result.Raise("Can't update hosts file with new host data")
3894
      _RedistributeAncillaryFiles(self)
3895

    
3896

    
3897
class _NodeQuery(_QueryBase):
3898
  FIELDS = query.NODE_FIELDS
3899

    
3900
  def ExpandNames(self, lu):
3901
    lu.needed_locks = {}
3902
    lu.share_locks[locking.LEVEL_NODE] = 1
3903

    
3904
    if self.names:
3905
      self.wanted = _GetWantedNodes(lu, self.names)
3906
    else:
3907
      self.wanted = locking.ALL_SET
3908

    
3909
    self.do_locking = (self.use_locking and
3910
                       query.NQ_LIVE in self.requested_data)
3911

    
3912
    if self.do_locking:
3913
      # if we don't request only static fields, we need to lock the nodes
3914
      lu.needed_locks[locking.LEVEL_NODE] = self.wanted
3915

    
3916
  def DeclareLocks(self, lu, level):
3917
    pass
3918

    
3919
  def _GetQueryData(self, lu):
3920
    """Computes the list of nodes and their attributes.
3921

3922
    """
3923
    all_info = lu.cfg.GetAllNodesInfo()
3924

    
3925
    nodenames = self._GetNames(lu, all_info.keys(), locking.LEVEL_NODE)
3926

    
3927
    # Gather data as requested
3928
    if query.NQ_LIVE in self.requested_data:
3929
      # filter out non-vm_capable nodes
3930
      toquery_nodes = [name for name in nodenames if all_info[name].vm_capable]
3931

    
3932
      node_data = lu.rpc.call_node_info(toquery_nodes, lu.cfg.GetVGName(),
3933
                                        lu.cfg.GetHypervisorType())
3934
      live_data = dict((name, nresult.payload)
3935
                       for (name, nresult) in node_data.items()
3936
                       if not nresult.fail_msg and nresult.payload)
3937
    else:
3938
      live_data = None
3939

    
3940
    if query.NQ_INST in self.requested_data:
3941
      node_to_primary = dict([(name, set()) for name in nodenames])
3942
      node_to_secondary = dict([(name, set()) for name in nodenames])
3943

    
3944
      inst_data = lu.cfg.GetAllInstancesInfo()
3945

    
3946
      for inst in inst_data.values():
3947
        if inst.primary_node in node_to_primary:
3948
          node_to_primary[inst.primary_node].add(inst.name)
3949
        for secnode in inst.secondary_nodes:
3950
          if secnode in node_to_secondary:
3951
            node_to_secondary[secnode].add(inst.name)
3952
    else:
3953
      node_to_primary = None
3954
      node_to_secondary = None
3955

    
3956
    if query.NQ_OOB in self.requested_data:
3957
      oob_support = dict((name, bool(_SupportsOob(lu.cfg, node)))
3958
                         for name, node in all_info.iteritems())
3959
    else:
3960
      oob_support = None
3961

    
3962
    if query.NQ_GROUP in self.requested_data:
3963
      groups = lu.cfg.GetAllNodeGroupsInfo()
3964
    else:
3965
      groups = {}
3966

    
3967
    return query.NodeQueryData([all_info[name] for name in nodenames],
3968
                               live_data, lu.cfg.GetMasterNode(),
3969
                               node_to_primary, node_to_secondary, groups,
3970
                               oob_support, lu.cfg.GetClusterInfo())
3971

    
3972

    
3973
class LUNodeQuery(NoHooksLU):
3974
  """Logical unit for querying nodes.
3975

3976
  """
3977
  # pylint: disable-msg=W0142
3978
  REQ_BGL = False
3979

    
3980
  def CheckArguments(self):
3981
    self.nq = _NodeQuery(qlang.MakeSimpleFilter("name", self.op.names),
3982
                         self.op.output_fields, self.op.use_locking)
3983

    
3984
  def ExpandNames(self):
3985
    self.nq.ExpandNames(self)
3986

    
3987
  def Exec(self, feedback_fn):
3988
    return self.nq.OldStyleQuery(self)
3989

    
3990

    
3991
class LUNodeQueryvols(NoHooksLU):
3992
  """Logical unit for getting volumes on node(s).
3993

3994
  """
3995
  REQ_BGL = False
3996
  _FIELDS_DYNAMIC = utils.FieldSet("phys", "vg", "name", "size", "instance")
3997
  _FIELDS_STATIC = utils.FieldSet("node")
3998

    
3999
  def CheckArguments(self):
4000
    _CheckOutputFields(static=self._FIELDS_STATIC,
4001
                       dynamic=self._FIELDS_DYNAMIC,
4002
                       selected=self.op.output_fields)
4003

    
4004
  def ExpandNames(self):
4005
    self.needed_locks = {}
4006
    self.share_locks[locking.LEVEL_NODE] = 1
4007
    if not self.op.nodes:
4008
      self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET
4009
    else:
4010
      self.needed_locks[locking.LEVEL_NODE] = \
4011
        _GetWantedNodes(self, self.op.nodes)
4012

    
4013
  def Exec(self, feedback_fn):
4014
    """Computes the list of nodes and their attributes.
4015

4016
    """
4017
    nodenames = self.glm.list_owned(locking.LEVEL_NODE)
4018
    volumes = self.rpc.call_node_volumes(nodenames)
4019

    
4020
    ilist = [self.cfg.GetInstanceInfo(iname) for iname
4021
             in self.cfg.GetInstanceList()]
4022

    
4023
    lv_by_node = dict([(inst, inst.MapLVsByNode()) for inst in ilist])
4024

    
4025
    output = []
4026
    for node in nodenames:
4027
      nresult = volumes[node]
4028
      if nresult.offline:
4029
        continue
4030
      msg = nresult.fail_msg
4031
      if msg:
4032
        self.LogWarning("Can't compute volume data on node %s: %s", node, msg)
4033
        continue
4034

    
4035
      node_vols = nresult.payload[:]
4036
      node_vols.sort(key=lambda vol: vol['dev'])
4037

    
4038
      for vol in node_vols:
4039
        node_output = []
4040
        for field in self.op.output_fields:
4041
          if field == "node":
4042
            val = node
4043
          elif field == "phys":
4044
            val = vol['dev']
4045
          elif field == "vg":
4046
            val = vol['vg']
4047
          elif field == "name":
4048
            val = vol['name']
4049
          elif field == "size":
4050
            val = int(float(vol['size']))
4051
          elif field == "instance":
4052
            for inst in ilist:
4053
              if node not in lv_by_node[inst]:
4054
                continue
4055
              if vol['name'] in lv_by_node[inst][node]:
4056
                val = inst.name
4057
                break
4058
            else:
4059
              val = '-'
4060
          else:
4061
            raise errors.ParameterError(field)
4062
          node_output.append(str(val))
4063

    
4064
        output.append(node_output)
4065

    
4066
    return output
4067

    
4068

    
4069
class LUNodeQueryStorage(NoHooksLU):
4070
  """Logical unit for getting information on storage units on node(s).
4071

4072
  """
4073
  _FIELDS_STATIC = utils.FieldSet(constants.SF_NODE)
4074
  REQ_BGL = False
4075