Statistics
| Branch: | Tag: | Revision:

root / lib / cmdlib.py @ 3ddd94f9

History | View | Annotate | Download (460.8 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Module implementing the master-side code."""
23

    
24
# pylint: disable-msg=W0201,C0302
25

    
26
# W0201 since most LU attributes are defined in CheckPrereq or similar
27
# functions
28

    
29
# C0302: since we have waaaay to many lines in this module
30

    
31
import os
32
import os.path
33
import time
34
import re
35
import platform
36
import logging
37
import copy
38
import OpenSSL
39
import socket
40
import tempfile
41
import shutil
42
import itertools
43
import operator
44

    
45
from ganeti import ssh
46
from ganeti import utils
47
from ganeti import errors
48
from ganeti import hypervisor
49
from ganeti import locking
50
from ganeti import constants
51
from ganeti import objects
52
from ganeti import serializer
53
from ganeti import ssconf
54
from ganeti import uidpool
55
from ganeti import compat
56
from ganeti import masterd
57
from ganeti import netutils
58
from ganeti import query
59
from ganeti import qlang
60
from ganeti import opcodes
61
from ganeti import ht
62

    
63
import ganeti.masterd.instance # pylint: disable-msg=W0611
64

    
65

    
66
def _SupportsOob(cfg, node):
67
  """Tells if node supports OOB.
68

69
  @type cfg: L{config.ConfigWriter}
70
  @param cfg: The cluster configuration
71
  @type node: L{objects.Node}
72
  @param node: The node
73
  @return: The OOB script if supported or an empty string otherwise
74

75
  """
76
  return cfg.GetNdParams(node)[constants.ND_OOB_PROGRAM]
77

    
78

    
79
class ResultWithJobs:
80
  """Data container for LU results with jobs.
81

82
  Instances of this class returned from L{LogicalUnit.Exec} will be recognized
83
  by L{mcpu.Processor._ProcessResult}. The latter will then submit the jobs
84
  contained in the C{jobs} attribute and include the job IDs in the opcode
85
  result.
86

87
  """
88
  def __init__(self, jobs, **kwargs):
89
    """Initializes this class.
90

91
    Additional return values can be specified as keyword arguments.
92

93
    @type jobs: list of lists of L{opcode.OpCode}
94
    @param jobs: A list of lists of opcode objects
95

96
    """
97
    self.jobs = jobs
98
    self.other = kwargs
99

    
100

    
101
class LogicalUnit(object):
102
  """Logical Unit base class.
103

104
  Subclasses must follow these rules:
105
    - implement ExpandNames
106
    - implement CheckPrereq (except when tasklets are used)
107
    - implement Exec (except when tasklets are used)
108
    - implement BuildHooksEnv
109
    - implement BuildHooksNodes
110
    - redefine HPATH and HTYPE
111
    - optionally redefine their run requirements:
112
        REQ_BGL: the LU needs to hold the Big Ganeti Lock exclusively
113

114
  Note that all commands require root permissions.
115

116
  @ivar dry_run_result: the value (if any) that will be returned to the caller
117
      in dry-run mode (signalled by opcode dry_run parameter)
118

119
  """
120
  HPATH = None
121
  HTYPE = None
122
  REQ_BGL = True
123

    
124
  def __init__(self, processor, op, context, rpc):
125
    """Constructor for LogicalUnit.
126

127
    This needs to be overridden in derived classes in order to check op
128
    validity.
129

130
    """
131
    self.proc = processor
132
    self.op = op
133
    self.cfg = context.cfg
134
    self.glm = context.glm
135
    self.context = context
136
    self.rpc = rpc
137
    # Dicts used to declare locking needs to mcpu
138
    self.needed_locks = None
139
    self.share_locks = dict.fromkeys(locking.LEVELS, 0)
140
    self.add_locks = {}
141
    self.remove_locks = {}
142
    # Used to force good behavior when calling helper functions
143
    self.recalculate_locks = {}
144
    # logging
145
    self.Log = processor.Log # pylint: disable-msg=C0103
146
    self.LogWarning = processor.LogWarning # pylint: disable-msg=C0103
147
    self.LogInfo = processor.LogInfo # pylint: disable-msg=C0103
148
    self.LogStep = processor.LogStep # pylint: disable-msg=C0103
149
    # support for dry-run
150
    self.dry_run_result = None
151
    # support for generic debug attribute
152
    if (not hasattr(self.op, "debug_level") or
153
        not isinstance(self.op.debug_level, int)):
154
      self.op.debug_level = 0
155

    
156
    # Tasklets
157
    self.tasklets = None
158

    
159
    # Validate opcode parameters and set defaults
160
    self.op.Validate(True)
161

    
162
    self.CheckArguments()
163

    
164
  def CheckArguments(self):
165
    """Check syntactic validity for the opcode arguments.
166

167
    This method is for doing a simple syntactic check and ensure
168
    validity of opcode parameters, without any cluster-related
169
    checks. While the same can be accomplished in ExpandNames and/or
170
    CheckPrereq, doing these separate is better because:
171

172
      - ExpandNames is left as as purely a lock-related function
173
      - CheckPrereq is run after we have acquired locks (and possible
174
        waited for them)
175

176
    The function is allowed to change the self.op attribute so that
177
    later methods can no longer worry about missing parameters.
178

179
    """
180
    pass
181

    
182
  def ExpandNames(self):
183
    """Expand names for this LU.
184

185
    This method is called before starting to execute the opcode, and it should
186
    update all the parameters of the opcode to their canonical form (e.g. a
187
    short node name must be fully expanded after this method has successfully
188
    completed). This way locking, hooks, logging, etc. can work correctly.
189

190
    LUs which implement this method must also populate the self.needed_locks
191
    member, as a dict with lock levels as keys, and a list of needed lock names
192
    as values. Rules:
193

194
      - use an empty dict if you don't need any lock
195
      - if you don't need any lock at a particular level omit that level
196
      - don't put anything for the BGL level
197
      - if you want all locks at a level use locking.ALL_SET as a value
198

199
    If you need to share locks (rather than acquire them exclusively) at one
200
    level you can modify self.share_locks, setting a true value (usually 1) for
201
    that level. By default locks are not shared.
202

203
    This function can also define a list of tasklets, which then will be
204
    executed in order instead of the usual LU-level CheckPrereq and Exec
205
    functions, if those are not defined by the LU.
206

207
    Examples::
208

209
      # Acquire all nodes and one instance
210
      self.needed_locks = {
211
        locking.LEVEL_NODE: locking.ALL_SET,
212
        locking.LEVEL_INSTANCE: ['instance1.example.com'],
213
      }
214
      # Acquire just two nodes
215
      self.needed_locks = {
216
        locking.LEVEL_NODE: ['node1.example.com', 'node2.example.com'],
217
      }
218
      # Acquire no locks
219
      self.needed_locks = {} # No, you can't leave it to the default value None
220

221
    """
222
    # The implementation of this method is mandatory only if the new LU is
223
    # concurrent, so that old LUs don't need to be changed all at the same
224
    # time.
225
    if self.REQ_BGL:
226
      self.needed_locks = {} # Exclusive LUs don't need locks.
227
    else:
228
      raise NotImplementedError
229

    
230
  def DeclareLocks(self, level):
231
    """Declare LU locking needs for a level
232

233
    While most LUs can just declare their locking needs at ExpandNames time,
234
    sometimes there's the need to calculate some locks after having acquired
235
    the ones before. This function is called just before acquiring locks at a
236
    particular level, but after acquiring the ones at lower levels, and permits
237
    such calculations. It can be used to modify self.needed_locks, and by
238
    default it does nothing.
239

240
    This function is only called if you have something already set in
241
    self.needed_locks for the level.
242

243
    @param level: Locking level which is going to be locked
244
    @type level: member of ganeti.locking.LEVELS
245

246
    """
247

    
248
  def CheckPrereq(self):
249
    """Check prerequisites for this LU.
250

251
    This method should check that the prerequisites for the execution
252
    of this LU are fulfilled. It can do internode communication, but
253
    it should be idempotent - no cluster or system changes are
254
    allowed.
255

256
    The method should raise errors.OpPrereqError in case something is
257
    not fulfilled. Its return value is ignored.
258

259
    This method should also update all the parameters of the opcode to
260
    their canonical form if it hasn't been done by ExpandNames before.
261

262
    """
263
    if self.tasklets is not None:
264
      for (idx, tl) in enumerate(self.tasklets):
265
        logging.debug("Checking prerequisites for tasklet %s/%s",
266
                      idx + 1, len(self.tasklets))
267
        tl.CheckPrereq()
268
    else:
269
      pass
270

    
271
  def Exec(self, feedback_fn):
272
    """Execute the LU.
273

274
    This method should implement the actual work. It should raise
275
    errors.OpExecError for failures that are somewhat dealt with in
276
    code, or expected.
277

278
    """
279
    if self.tasklets is not None:
280
      for (idx, tl) in enumerate(self.tasklets):
281
        logging.debug("Executing tasklet %s/%s", idx + 1, len(self.tasklets))
282
        tl.Exec(feedback_fn)
283
    else:
284
      raise NotImplementedError
285

    
286
  def BuildHooksEnv(self):
287
    """Build hooks environment for this LU.
288

289
    @rtype: dict
290
    @return: Dictionary containing the environment that will be used for
291
      running the hooks for this LU. The keys of the dict must not be prefixed
292
      with "GANETI_"--that'll be added by the hooks runner. The hooks runner
293
      will extend the environment with additional variables. If no environment
294
      should be defined, an empty dictionary should be returned (not C{None}).
295
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
296
      will not be called.
297

298
    """
299
    raise NotImplementedError
300

    
301
  def BuildHooksNodes(self):
302
    """Build list of nodes to run LU's hooks.
303

304
    @rtype: tuple; (list, list)
305
    @return: Tuple containing a list of node names on which the hook
306
      should run before the execution and a list of node names on which the
307
      hook should run after the execution. No nodes should be returned as an
308
      empty list (and not None).
309
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
310
      will not be called.
311

312
    """
313
    raise NotImplementedError
314

    
315
  def HooksCallBack(self, phase, hook_results, feedback_fn, lu_result):
316
    """Notify the LU about the results of its hooks.
317

318
    This method is called every time a hooks phase is executed, and notifies
319
    the Logical Unit about the hooks' result. The LU can then use it to alter
320
    its result based on the hooks.  By default the method does nothing and the
321
    previous result is passed back unchanged but any LU can define it if it
322
    wants to use the local cluster hook-scripts somehow.
323

324
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
325
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
326
    @param hook_results: the results of the multi-node hooks rpc call
327
    @param feedback_fn: function used send feedback back to the caller
328
    @param lu_result: the previous Exec result this LU had, or None
329
        in the PRE phase
330
    @return: the new Exec result, based on the previous result
331
        and hook results
332

333
    """
334
    # API must be kept, thus we ignore the unused argument and could
335
    # be a function warnings
336
    # pylint: disable-msg=W0613,R0201
337
    return lu_result
338

    
339
  def _ExpandAndLockInstance(self):
340
    """Helper function to expand and lock an instance.
341

342
    Many LUs that work on an instance take its name in self.op.instance_name
343
    and need to expand it and then declare the expanded name for locking. This
344
    function does it, and then updates self.op.instance_name to the expanded
345
    name. It also initializes needed_locks as a dict, if this hasn't been done
346
    before.
347

348
    """
349
    if self.needed_locks is None:
350
      self.needed_locks = {}
351
    else:
352
      assert locking.LEVEL_INSTANCE not in self.needed_locks, \
353
        "_ExpandAndLockInstance called with instance-level locks set"
354
    self.op.instance_name = _ExpandInstanceName(self.cfg,
355
                                                self.op.instance_name)
356
    self.needed_locks[locking.LEVEL_INSTANCE] = self.op.instance_name
357

    
358
  def _LockInstancesNodes(self, primary_only=False):
359
    """Helper function to declare instances' nodes for locking.
360

361
    This function should be called after locking one or more instances to lock
362
    their nodes. Its effect is populating self.needed_locks[locking.LEVEL_NODE]
363
    with all primary or secondary nodes for instances already locked and
364
    present in self.needed_locks[locking.LEVEL_INSTANCE].
365

366
    It should be called from DeclareLocks, and for safety only works if
367
    self.recalculate_locks[locking.LEVEL_NODE] is set.
368

369
    In the future it may grow parameters to just lock some instance's nodes, or
370
    to just lock primaries or secondary nodes, if needed.
371

372
    If should be called in DeclareLocks in a way similar to::
373

374
      if level == locking.LEVEL_NODE:
375
        self._LockInstancesNodes()
376

377
    @type primary_only: boolean
378
    @param primary_only: only lock primary nodes of locked instances
379

380
    """
381
    assert locking.LEVEL_NODE in self.recalculate_locks, \
382
      "_LockInstancesNodes helper function called with no nodes to recalculate"
383

    
384
    # TODO: check if we're really been called with the instance locks held
385

    
386
    # For now we'll replace self.needed_locks[locking.LEVEL_NODE], but in the
387
    # future we might want to have different behaviors depending on the value
388
    # of self.recalculate_locks[locking.LEVEL_NODE]
389
    wanted_nodes = []
390
    for instance_name in self.glm.list_owned(locking.LEVEL_INSTANCE):
391
      instance = self.context.cfg.GetInstanceInfo(instance_name)
392
      wanted_nodes.append(instance.primary_node)
393
      if not primary_only:
394
        wanted_nodes.extend(instance.secondary_nodes)
395

    
396
    if self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_REPLACE:
397
      self.needed_locks[locking.LEVEL_NODE] = wanted_nodes
398
    elif self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_APPEND:
399
      self.needed_locks[locking.LEVEL_NODE].extend(wanted_nodes)
400

    
401
    del self.recalculate_locks[locking.LEVEL_NODE]
402

    
403

    
404
class NoHooksLU(LogicalUnit): # pylint: disable-msg=W0223
405
  """Simple LU which runs no hooks.
406

407
  This LU is intended as a parent for other LogicalUnits which will
408
  run no hooks, in order to reduce duplicate code.
409

410
  """
411
  HPATH = None
412
  HTYPE = None
413

    
414
  def BuildHooksEnv(self):
415
    """Empty BuildHooksEnv for NoHooksLu.
416

417
    This just raises an error.
418

419
    """
420
    raise AssertionError("BuildHooksEnv called for NoHooksLUs")
421

    
422
  def BuildHooksNodes(self):
423
    """Empty BuildHooksNodes for NoHooksLU.
424

425
    """
426
    raise AssertionError("BuildHooksNodes called for NoHooksLU")
427

    
428

    
429
class Tasklet:
430
  """Tasklet base class.
431

432
  Tasklets are subcomponents for LUs. LUs can consist entirely of tasklets or
433
  they can mix legacy code with tasklets. Locking needs to be done in the LU,
434
  tasklets know nothing about locks.
435

436
  Subclasses must follow these rules:
437
    - Implement CheckPrereq
438
    - Implement Exec
439

440
  """
441
  def __init__(self, lu):
442
    self.lu = lu
443

    
444
    # Shortcuts
445
    self.cfg = lu.cfg
446
    self.rpc = lu.rpc
447

    
448
  def CheckPrereq(self):
449
    """Check prerequisites for this tasklets.
450

451
    This method should check whether the prerequisites for the execution of
452
    this tasklet are fulfilled. It can do internode communication, but it
453
    should be idempotent - no cluster or system changes are allowed.
454

455
    The method should raise errors.OpPrereqError in case something is not
456
    fulfilled. Its return value is ignored.
457

458
    This method should also update all parameters to their canonical form if it
459
    hasn't been done before.
460

461
    """
462
    pass
463

    
464
  def Exec(self, feedback_fn):
465
    """Execute the tasklet.
466

467
    This method should implement the actual work. It should raise
468
    errors.OpExecError for failures that are somewhat dealt with in code, or
469
    expected.
470

471
    """
472
    raise NotImplementedError
473

    
474

    
475
class _QueryBase:
476
  """Base for query utility classes.
477

478
  """
479
  #: Attribute holding field definitions
480
  FIELDS = None
481

    
482
  def __init__(self, filter_, fields, use_locking):
483
    """Initializes this class.
484

485
    """
486
    self.use_locking = use_locking
487

    
488
    self.query = query.Query(self.FIELDS, fields, filter_=filter_,
489
                             namefield="name")
490
    self.requested_data = self.query.RequestedData()
491
    self.names = self.query.RequestedNames()
492

    
493
    # Sort only if no names were requested
494
    self.sort_by_name = not self.names
495

    
496
    self.do_locking = None
497
    self.wanted = None
498

    
499
  def _GetNames(self, lu, all_names, lock_level):
500
    """Helper function to determine names asked for in the query.
501

502
    """
503
    if self.do_locking:
504
      names = lu.glm.list_owned(lock_level)
505
    else:
506
      names = all_names
507

    
508
    if self.wanted == locking.ALL_SET:
509
      assert not self.names
510
      # caller didn't specify names, so ordering is not important
511
      return utils.NiceSort(names)
512

    
513
    # caller specified names and we must keep the same order
514
    assert self.names
515
    assert not self.do_locking or lu.glm.is_owned(lock_level)
516

    
517
    missing = set(self.wanted).difference(names)
518
    if missing:
519
      raise errors.OpExecError("Some items were removed before retrieving"
520
                               " their data: %s" % missing)
521

    
522
    # Return expanded names
523
    return self.wanted
524

    
525
  def ExpandNames(self, lu):
526
    """Expand names for this query.
527

528
    See L{LogicalUnit.ExpandNames}.
529

530
    """
531
    raise NotImplementedError()
532

    
533
  def DeclareLocks(self, lu, level):
534
    """Declare locks for this query.
535

536
    See L{LogicalUnit.DeclareLocks}.
537

538
    """
539
    raise NotImplementedError()
540

    
541
  def _GetQueryData(self, lu):
542
    """Collects all data for this query.
543

544
    @return: Query data object
545

546
    """
547
    raise NotImplementedError()
548

    
549
  def NewStyleQuery(self, lu):
550
    """Collect data and execute query.
551

552
    """
553
    return query.GetQueryResponse(self.query, self._GetQueryData(lu),
554
                                  sort_by_name=self.sort_by_name)
555

    
556
  def OldStyleQuery(self, lu):
557
    """Collect data and execute query.
558

559
    """
560
    return self.query.OldStyleQuery(self._GetQueryData(lu),
561
                                    sort_by_name=self.sort_by_name)
562

    
563

    
564
def _GetWantedNodes(lu, nodes):
565
  """Returns list of checked and expanded node names.
566

567
  @type lu: L{LogicalUnit}
568
  @param lu: the logical unit on whose behalf we execute
569
  @type nodes: list
570
  @param nodes: list of node names or None for all nodes
571
  @rtype: list
572
  @return: the list of nodes, sorted
573
  @raise errors.ProgrammerError: if the nodes parameter is wrong type
574

575
  """
576
  if nodes:
577
    return [_ExpandNodeName(lu.cfg, name) for name in nodes]
578

    
579
  return utils.NiceSort(lu.cfg.GetNodeList())
580

    
581

    
582
def _GetWantedInstances(lu, instances):
583
  """Returns list of checked and expanded instance names.
584

585
  @type lu: L{LogicalUnit}
586
  @param lu: the logical unit on whose behalf we execute
587
  @type instances: list
588
  @param instances: list of instance names or None for all instances
589
  @rtype: list
590
  @return: the list of instances, sorted
591
  @raise errors.OpPrereqError: if the instances parameter is wrong type
592
  @raise errors.OpPrereqError: if any of the passed instances is not found
593

594
  """
595
  if instances:
596
    wanted = [_ExpandInstanceName(lu.cfg, name) for name in instances]
597
  else:
598
    wanted = utils.NiceSort(lu.cfg.GetInstanceList())
599
  return wanted
600

    
601

    
602
def _GetUpdatedParams(old_params, update_dict,
603
                      use_default=True, use_none=False):
604
  """Return the new version of a parameter dictionary.
605

606
  @type old_params: dict
607
  @param old_params: old parameters
608
  @type update_dict: dict
609
  @param update_dict: dict containing new parameter values, or
610
      constants.VALUE_DEFAULT to reset the parameter to its default
611
      value
612
  @param use_default: boolean
613
  @type use_default: whether to recognise L{constants.VALUE_DEFAULT}
614
      values as 'to be deleted' values
615
  @param use_none: boolean
616
  @type use_none: whether to recognise C{None} values as 'to be
617
      deleted' values
618
  @rtype: dict
619
  @return: the new parameter dictionary
620

621
  """
622
  params_copy = copy.deepcopy(old_params)
623
  for key, val in update_dict.iteritems():
624
    if ((use_default and val == constants.VALUE_DEFAULT) or
625
        (use_none and val is None)):
626
      try:
627
        del params_copy[key]
628
      except KeyError:
629
        pass
630
    else:
631
      params_copy[key] = val
632
  return params_copy
633

    
634

    
635
def _ReleaseLocks(lu, level, names=None, keep=None):
636
  """Releases locks owned by an LU.
637

638
  @type lu: L{LogicalUnit}
639
  @param level: Lock level
640
  @type names: list or None
641
  @param names: Names of locks to release
642
  @type keep: list or None
643
  @param keep: Names of locks to retain
644

645
  """
646
  assert not (keep is not None and names is not None), \
647
         "Only one of the 'names' and the 'keep' parameters can be given"
648

    
649
  if names is not None:
650
    should_release = names.__contains__
651
  elif keep:
652
    should_release = lambda name: name not in keep
653
  else:
654
    should_release = None
655

    
656
  if should_release:
657
    retain = []
658
    release = []
659

    
660
    # Determine which locks to release
661
    for name in lu.glm.list_owned(level):
662
      if should_release(name):
663
        release.append(name)
664
      else:
665
        retain.append(name)
666

    
667
    assert len(lu.glm.list_owned(level)) == (len(retain) + len(release))
668

    
669
    # Release just some locks
670
    lu.glm.release(level, names=release)
671

    
672
    assert frozenset(lu.glm.list_owned(level)) == frozenset(retain)
673
  else:
674
    # Release everything
675
    lu.glm.release(level)
676

    
677
    assert not lu.glm.is_owned(level), "No locks should be owned"
678

    
679

    
680
def _MapInstanceDisksToNodes(instances):
681
  """Creates a map from (node, volume) to instance name.
682

683
  @type instances: list of L{objects.Instance}
684
  @rtype: dict; tuple of (node name, volume name) as key, instance name as value
685

686
  """
687
  return dict(((node, vol), inst.name)
688
              for inst in instances
689
              for (node, vols) in inst.MapLVsByNode().items()
690
              for vol in vols)
691

    
692

    
693
def _RunPostHook(lu, node_name):
694
  """Runs the post-hook for an opcode on a single node.
695

696
  """
697
  hm = lu.proc.hmclass(lu.rpc.call_hooks_runner, lu)
698
  try:
699
    hm.RunPhase(constants.HOOKS_PHASE_POST, nodes=[node_name])
700
  except:
701
    # pylint: disable-msg=W0702
702
    lu.LogWarning("Errors occurred running hooks on %s" % node_name)
703

    
704

    
705
def _CheckOutputFields(static, dynamic, selected):
706
  """Checks whether all selected fields are valid.
707

708
  @type static: L{utils.FieldSet}
709
  @param static: static fields set
710
  @type dynamic: L{utils.FieldSet}
711
  @param dynamic: dynamic fields set
712

713
  """
714
  f = utils.FieldSet()
715
  f.Extend(static)
716
  f.Extend(dynamic)
717

    
718
  delta = f.NonMatching(selected)
719
  if delta:
720
    raise errors.OpPrereqError("Unknown output fields selected: %s"
721
                               % ",".join(delta), errors.ECODE_INVAL)
722

    
723

    
724
def _CheckGlobalHvParams(params):
725
  """Validates that given hypervisor params are not global ones.
726

727
  This will ensure that instances don't get customised versions of
728
  global params.
729

730
  """
731
  used_globals = constants.HVC_GLOBALS.intersection(params)
732
  if used_globals:
733
    msg = ("The following hypervisor parameters are global and cannot"
734
           " be customized at instance level, please modify them at"
735
           " cluster level: %s" % utils.CommaJoin(used_globals))
736
    raise errors.OpPrereqError(msg, errors.ECODE_INVAL)
737

    
738

    
739
def _CheckNodeOnline(lu, node, msg=None):
740
  """Ensure that a given node is online.
741

742
  @param lu: the LU on behalf of which we make the check
743
  @param node: the node to check
744
  @param msg: if passed, should be a message to replace the default one
745
  @raise errors.OpPrereqError: if the node is offline
746

747
  """
748
  if msg is None:
749
    msg = "Can't use offline node"
750
  if lu.cfg.GetNodeInfo(node).offline:
751
    raise errors.OpPrereqError("%s: %s" % (msg, node), errors.ECODE_STATE)
752

    
753

    
754
def _CheckNodeNotDrained(lu, node):
755
  """Ensure that a given node is not drained.
756

757
  @param lu: the LU on behalf of which we make the check
758
  @param node: the node to check
759
  @raise errors.OpPrereqError: if the node is drained
760

761
  """
762
  if lu.cfg.GetNodeInfo(node).drained:
763
    raise errors.OpPrereqError("Can't use drained node %s" % node,
764
                               errors.ECODE_STATE)
765

    
766

    
767
def _CheckNodeVmCapable(lu, node):
768
  """Ensure that a given node is vm capable.
769

770
  @param lu: the LU on behalf of which we make the check
771
  @param node: the node to check
772
  @raise errors.OpPrereqError: if the node is not vm capable
773

774
  """
775
  if not lu.cfg.GetNodeInfo(node).vm_capable:
776
    raise errors.OpPrereqError("Can't use non-vm_capable node %s" % node,
777
                               errors.ECODE_STATE)
778

    
779

    
780
def _CheckNodeHasOS(lu, node, os_name, force_variant):
781
  """Ensure that a node supports a given OS.
782

783
  @param lu: the LU on behalf of which we make the check
784
  @param node: the node to check
785
  @param os_name: the OS to query about
786
  @param force_variant: whether to ignore variant errors
787
  @raise errors.OpPrereqError: if the node is not supporting the OS
788

789
  """
790
  result = lu.rpc.call_os_get(node, os_name)
791
  result.Raise("OS '%s' not in supported OS list for node %s" %
792
               (os_name, node),
793
               prereq=True, ecode=errors.ECODE_INVAL)
794
  if not force_variant:
795
    _CheckOSVariant(result.payload, os_name)
796

    
797

    
798
def _CheckNodeHasSecondaryIP(lu, node, secondary_ip, prereq):
799
  """Ensure that a node has the given secondary ip.
800

801
  @type lu: L{LogicalUnit}
802
  @param lu: the LU on behalf of which we make the check
803
  @type node: string
804
  @param node: the node to check
805
  @type secondary_ip: string
806
  @param secondary_ip: the ip to check
807
  @type prereq: boolean
808
  @param prereq: whether to throw a prerequisite or an execute error
809
  @raise errors.OpPrereqError: if the node doesn't have the ip, and prereq=True
810
  @raise errors.OpExecError: if the node doesn't have the ip, and prereq=False
811

812
  """
813
  result = lu.rpc.call_node_has_ip_address(node, secondary_ip)
814
  result.Raise("Failure checking secondary ip on node %s" % node,
815
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
816
  if not result.payload:
817
    msg = ("Node claims it doesn't have the secondary ip you gave (%s),"
818
           " please fix and re-run this command" % secondary_ip)
819
    if prereq:
820
      raise errors.OpPrereqError(msg, errors.ECODE_ENVIRON)
821
    else:
822
      raise errors.OpExecError(msg)
823

    
824

    
825
def _GetClusterDomainSecret():
826
  """Reads the cluster domain secret.
827

828
  """
829
  return utils.ReadOneLineFile(constants.CLUSTER_DOMAIN_SECRET_FILE,
830
                               strict=True)
831

    
832

    
833
def _CheckInstanceDown(lu, instance, reason):
834
  """Ensure that an instance is not running."""
835
  if instance.admin_up:
836
    raise errors.OpPrereqError("Instance %s is marked to be up, %s" %
837
                               (instance.name, reason), errors.ECODE_STATE)
838

    
839
  pnode = instance.primary_node
840
  ins_l = lu.rpc.call_instance_list([pnode], [instance.hypervisor])[pnode]
841
  ins_l.Raise("Can't contact node %s for instance information" % pnode,
842
              prereq=True, ecode=errors.ECODE_ENVIRON)
843

    
844
  if instance.name in ins_l.payload:
845
    raise errors.OpPrereqError("Instance %s is running, %s" %
846
                               (instance.name, reason), errors.ECODE_STATE)
847

    
848

    
849
def _ExpandItemName(fn, name, kind):
850
  """Expand an item name.
851

852
  @param fn: the function to use for expansion
853
  @param name: requested item name
854
  @param kind: text description ('Node' or 'Instance')
855
  @return: the resolved (full) name
856
  @raise errors.OpPrereqError: if the item is not found
857

858
  """
859
  full_name = fn(name)
860
  if full_name is None:
861
    raise errors.OpPrereqError("%s '%s' not known" % (kind, name),
862
                               errors.ECODE_NOENT)
863
  return full_name
864

    
865

    
866
def _ExpandNodeName(cfg, name):
867
  """Wrapper over L{_ExpandItemName} for nodes."""
868
  return _ExpandItemName(cfg.ExpandNodeName, name, "Node")
869

    
870

    
871
def _ExpandInstanceName(cfg, name):
872
  """Wrapper over L{_ExpandItemName} for instance."""
873
  return _ExpandItemName(cfg.ExpandInstanceName, name, "Instance")
874

    
875

    
876
def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status,
877
                          memory, vcpus, nics, disk_template, disks,
878
                          bep, hvp, hypervisor_name, tags):
879
  """Builds instance related env variables for hooks
880

881
  This builds the hook environment from individual variables.
882

883
  @type name: string
884
  @param name: the name of the instance
885
  @type primary_node: string
886
  @param primary_node: the name of the instance's primary node
887
  @type secondary_nodes: list
888
  @param secondary_nodes: list of secondary nodes as strings
889
  @type os_type: string
890
  @param os_type: the name of the instance's OS
891
  @type status: boolean
892
  @param status: the should_run status of the instance
893
  @type memory: string
894
  @param memory: the memory size of the instance
895
  @type vcpus: string
896
  @param vcpus: the count of VCPUs the instance has
897
  @type nics: list
898
  @param nics: list of tuples (ip, mac, mode, link) representing
899
      the NICs the instance has
900
  @type disk_template: string
901
  @param disk_template: the disk template of the instance
902
  @type disks: list
903
  @param disks: the list of (size, mode) pairs
904
  @type bep: dict
905
  @param bep: the backend parameters for the instance
906
  @type hvp: dict
907
  @param hvp: the hypervisor parameters for the instance
908
  @type hypervisor_name: string
909
  @param hypervisor_name: the hypervisor for the instance
910
  @type tags: list
911
  @param tags: list of instance tags as strings
912
  @rtype: dict
913
  @return: the hook environment for this instance
914

915
  """
916
  if status:
917
    str_status = "up"
918
  else:
919
    str_status = "down"
920
  env = {
921
    "OP_TARGET": name,
922
    "INSTANCE_NAME": name,
923
    "INSTANCE_PRIMARY": primary_node,
924
    "INSTANCE_SECONDARIES": " ".join(secondary_nodes),
925
    "INSTANCE_OS_TYPE": os_type,
926
    "INSTANCE_STATUS": str_status,
927
    "INSTANCE_MEMORY": memory,
928
    "INSTANCE_VCPUS": vcpus,
929
    "INSTANCE_DISK_TEMPLATE": disk_template,
930
    "INSTANCE_HYPERVISOR": hypervisor_name,
931
  }
932

    
933
  if nics:
934
    nic_count = len(nics)
935
    for idx, (ip, mac, mode, link) in enumerate(nics):
936
      if ip is None:
937
        ip = ""
938
      env["INSTANCE_NIC%d_IP" % idx] = ip
939
      env["INSTANCE_NIC%d_MAC" % idx] = mac
940
      env["INSTANCE_NIC%d_MODE" % idx] = mode
941
      env["INSTANCE_NIC%d_LINK" % idx] = link
942
      if mode == constants.NIC_MODE_BRIDGED:
943
        env["INSTANCE_NIC%d_BRIDGE" % idx] = link
944
  else:
945
    nic_count = 0
946

    
947
  env["INSTANCE_NIC_COUNT"] = nic_count
948

    
949
  if disks:
950
    disk_count = len(disks)
951
    for idx, (size, mode) in enumerate(disks):
952
      env["INSTANCE_DISK%d_SIZE" % idx] = size
953
      env["INSTANCE_DISK%d_MODE" % idx] = mode
954
  else:
955
    disk_count = 0
956

    
957
  env["INSTANCE_DISK_COUNT"] = disk_count
958

    
959
  if not tags:
960
    tags = []
961

    
962
  env["INSTANCE_TAGS"] = " ".join(tags)
963

    
964
  for source, kind in [(bep, "BE"), (hvp, "HV")]:
965
    for key, value in source.items():
966
      env["INSTANCE_%s_%s" % (kind, key)] = value
967

    
968
  return env
969

    
970

    
971
def _NICListToTuple(lu, nics):
972
  """Build a list of nic information tuples.
973

974
  This list is suitable to be passed to _BuildInstanceHookEnv or as a return
975
  value in LUInstanceQueryData.
976

977
  @type lu:  L{LogicalUnit}
978
  @param lu: the logical unit on whose behalf we execute
979
  @type nics: list of L{objects.NIC}
980
  @param nics: list of nics to convert to hooks tuples
981

982
  """
983
  hooks_nics = []
984
  cluster = lu.cfg.GetClusterInfo()
985
  for nic in nics:
986
    ip = nic.ip
987
    mac = nic.mac
988
    filled_params = cluster.SimpleFillNIC(nic.nicparams)
989
    mode = filled_params[constants.NIC_MODE]
990
    link = filled_params[constants.NIC_LINK]
991
    hooks_nics.append((ip, mac, mode, link))
992
  return hooks_nics
993

    
994

    
995
def _BuildInstanceHookEnvByObject(lu, instance, override=None):
996
  """Builds instance related env variables for hooks from an object.
997

998
  @type lu: L{LogicalUnit}
999
  @param lu: the logical unit on whose behalf we execute
1000
  @type instance: L{objects.Instance}
1001
  @param instance: the instance for which we should build the
1002
      environment
1003
  @type override: dict
1004
  @param override: dictionary with key/values that will override
1005
      our values
1006
  @rtype: dict
1007
  @return: the hook environment dictionary
1008

1009
  """
1010
  cluster = lu.cfg.GetClusterInfo()
1011
  bep = cluster.FillBE(instance)
1012
  hvp = cluster.FillHV(instance)
1013
  args = {
1014
    "name": instance.name,
1015
    "primary_node": instance.primary_node,
1016
    "secondary_nodes": instance.secondary_nodes,
1017
    "os_type": instance.os,
1018
    "status": instance.admin_up,
1019
    "memory": bep[constants.BE_MEMORY],
1020
    "vcpus": bep[constants.BE_VCPUS],
1021
    "nics": _NICListToTuple(lu, instance.nics),
1022
    "disk_template": instance.disk_template,
1023
    "disks": [(disk.size, disk.mode) for disk in instance.disks],
1024
    "bep": bep,
1025
    "hvp": hvp,
1026
    "hypervisor_name": instance.hypervisor,
1027
    "tags": instance.tags,
1028
  }
1029
  if override:
1030
    args.update(override)
1031
  return _BuildInstanceHookEnv(**args) # pylint: disable-msg=W0142
1032

    
1033

    
1034
def _AdjustCandidatePool(lu, exceptions):
1035
  """Adjust the candidate pool after node operations.
1036

1037
  """
1038
  mod_list = lu.cfg.MaintainCandidatePool(exceptions)
1039
  if mod_list:
1040
    lu.LogInfo("Promoted nodes to master candidate role: %s",
1041
               utils.CommaJoin(node.name for node in mod_list))
1042
    for name in mod_list:
1043
      lu.context.ReaddNode(name)
1044
  mc_now, mc_max, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1045
  if mc_now > mc_max:
1046
    lu.LogInfo("Note: more nodes are candidates (%d) than desired (%d)" %
1047
               (mc_now, mc_max))
1048

    
1049

    
1050
def _DecideSelfPromotion(lu, exceptions=None):
1051
  """Decide whether I should promote myself as a master candidate.
1052

1053
  """
1054
  cp_size = lu.cfg.GetClusterInfo().candidate_pool_size
1055
  mc_now, mc_should, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1056
  # the new node will increase mc_max with one, so:
1057
  mc_should = min(mc_should + 1, cp_size)
1058
  return mc_now < mc_should
1059

    
1060

    
1061
def _CheckNicsBridgesExist(lu, target_nics, target_node):
1062
  """Check that the brigdes needed by a list of nics exist.
1063

1064
  """
1065
  cluster = lu.cfg.GetClusterInfo()
1066
  paramslist = [cluster.SimpleFillNIC(nic.nicparams) for nic in target_nics]
1067
  brlist = [params[constants.NIC_LINK] for params in paramslist
1068
            if params[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED]
1069
  if brlist:
1070
    result = lu.rpc.call_bridges_exist(target_node, brlist)
1071
    result.Raise("Error checking bridges on destination node '%s'" %
1072
                 target_node, prereq=True, ecode=errors.ECODE_ENVIRON)
1073

    
1074

    
1075
def _CheckInstanceBridgesExist(lu, instance, node=None):
1076
  """Check that the brigdes needed by an instance exist.
1077

1078
  """
1079
  if node is None:
1080
    node = instance.primary_node
1081
  _CheckNicsBridgesExist(lu, instance.nics, node)
1082

    
1083

    
1084
def _CheckOSVariant(os_obj, name):
1085
  """Check whether an OS name conforms to the os variants specification.
1086

1087
  @type os_obj: L{objects.OS}
1088
  @param os_obj: OS object to check
1089
  @type name: string
1090
  @param name: OS name passed by the user, to check for validity
1091

1092
  """
1093
  if not os_obj.supported_variants:
1094
    return
1095
  variant = objects.OS.GetVariant(name)
1096
  if not variant:
1097
    raise errors.OpPrereqError("OS name must include a variant",
1098
                               errors.ECODE_INVAL)
1099

    
1100
  if variant not in os_obj.supported_variants:
1101
    raise errors.OpPrereqError("Unsupported OS variant", errors.ECODE_INVAL)
1102

    
1103

    
1104
def _GetNodeInstancesInner(cfg, fn):
1105
  return [i for i in cfg.GetAllInstancesInfo().values() if fn(i)]
1106

    
1107

    
1108
def _GetNodeInstances(cfg, node_name):
1109
  """Returns a list of all primary and secondary instances on a node.
1110

1111
  """
1112

    
1113
  return _GetNodeInstancesInner(cfg, lambda inst: node_name in inst.all_nodes)
1114

    
1115

    
1116
def _GetNodePrimaryInstances(cfg, node_name):
1117
  """Returns primary instances on a node.
1118

1119
  """
1120
  return _GetNodeInstancesInner(cfg,
1121
                                lambda inst: node_name == inst.primary_node)
1122

    
1123

    
1124
def _GetNodeSecondaryInstances(cfg, node_name):
1125
  """Returns secondary instances on a node.
1126

1127
  """
1128
  return _GetNodeInstancesInner(cfg,
1129
                                lambda inst: node_name in inst.secondary_nodes)
1130

    
1131

    
1132
def _GetStorageTypeArgs(cfg, storage_type):
1133
  """Returns the arguments for a storage type.
1134

1135
  """
1136
  # Special case for file storage
1137
  if storage_type == constants.ST_FILE:
1138
    # storage.FileStorage wants a list of storage directories
1139
    return [[cfg.GetFileStorageDir(), cfg.GetSharedFileStorageDir()]]
1140

    
1141
  return []
1142

    
1143

    
1144
def _FindFaultyInstanceDisks(cfg, rpc, instance, node_name, prereq):
1145
  faulty = []
1146

    
1147
  for dev in instance.disks:
1148
    cfg.SetDiskID(dev, node_name)
1149

    
1150
  result = rpc.call_blockdev_getmirrorstatus(node_name, instance.disks)
1151
  result.Raise("Failed to get disk status from node %s" % node_name,
1152
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
1153

    
1154
  for idx, bdev_status in enumerate(result.payload):
1155
    if bdev_status and bdev_status.ldisk_status == constants.LDS_FAULTY:
1156
      faulty.append(idx)
1157

    
1158
  return faulty
1159

    
1160

    
1161
def _CheckIAllocatorOrNode(lu, iallocator_slot, node_slot):
1162
  """Check the sanity of iallocator and node arguments and use the
1163
  cluster-wide iallocator if appropriate.
1164

1165
  Check that at most one of (iallocator, node) is specified. If none is
1166
  specified, then the LU's opcode's iallocator slot is filled with the
1167
  cluster-wide default iallocator.
1168

1169
  @type iallocator_slot: string
1170
  @param iallocator_slot: the name of the opcode iallocator slot
1171
  @type node_slot: string
1172
  @param node_slot: the name of the opcode target node slot
1173

1174
  """
1175
  node = getattr(lu.op, node_slot, None)
1176
  iallocator = getattr(lu.op, iallocator_slot, None)
1177

    
1178
  if node is not None and iallocator is not None:
1179
    raise errors.OpPrereqError("Do not specify both, iallocator and node",
1180
                               errors.ECODE_INVAL)
1181
  elif node is None and iallocator is None:
1182
    default_iallocator = lu.cfg.GetDefaultIAllocator()
1183
    if default_iallocator:
1184
      setattr(lu.op, iallocator_slot, default_iallocator)
1185
    else:
1186
      raise errors.OpPrereqError("No iallocator or node given and no"
1187
                                 " cluster-wide default iallocator found;"
1188
                                 " please specify either an iallocator or a"
1189
                                 " node, or set a cluster-wide default"
1190
                                 " iallocator")
1191

    
1192

    
1193
class LUClusterPostInit(LogicalUnit):
1194
  """Logical unit for running hooks after cluster initialization.
1195

1196
  """
1197
  HPATH = "cluster-init"
1198
  HTYPE = constants.HTYPE_CLUSTER
1199

    
1200
  def BuildHooksEnv(self):
1201
    """Build hooks env.
1202

1203
    """
1204
    return {
1205
      "OP_TARGET": self.cfg.GetClusterName(),
1206
      }
1207

    
1208
  def BuildHooksNodes(self):
1209
    """Build hooks nodes.
1210

1211
    """
1212
    return ([], [self.cfg.GetMasterNode()])
1213

    
1214
  def Exec(self, feedback_fn):
1215
    """Nothing to do.
1216

1217
    """
1218
    return True
1219

    
1220

    
1221
class LUClusterDestroy(LogicalUnit):
1222
  """Logical unit for destroying the cluster.
1223

1224
  """
1225
  HPATH = "cluster-destroy"
1226
  HTYPE = constants.HTYPE_CLUSTER
1227

    
1228
  def BuildHooksEnv(self):
1229
    """Build hooks env.
1230

1231
    """
1232
    return {
1233
      "OP_TARGET": self.cfg.GetClusterName(),
1234
      }
1235

    
1236
  def BuildHooksNodes(self):
1237
    """Build hooks nodes.
1238

1239
    """
1240
    return ([], [])
1241

    
1242
  def CheckPrereq(self):
1243
    """Check prerequisites.
1244

1245
    This checks whether the cluster is empty.
1246

1247
    Any errors are signaled by raising errors.OpPrereqError.
1248

1249
    """
1250
    master = self.cfg.GetMasterNode()
1251

    
1252
    nodelist = self.cfg.GetNodeList()
1253
    if len(nodelist) != 1 or nodelist[0] != master:
1254
      raise errors.OpPrereqError("There are still %d node(s) in"
1255
                                 " this cluster." % (len(nodelist) - 1),
1256
                                 errors.ECODE_INVAL)
1257
    instancelist = self.cfg.GetInstanceList()
1258
    if instancelist:
1259
      raise errors.OpPrereqError("There are still %d instance(s) in"
1260
                                 " this cluster." % len(instancelist),
1261
                                 errors.ECODE_INVAL)
1262

    
1263
  def Exec(self, feedback_fn):
1264
    """Destroys the cluster.
1265

1266
    """
1267
    master = self.cfg.GetMasterNode()
1268

    
1269
    # Run post hooks on master node before it's removed
1270
    _RunPostHook(self, master)
1271

    
1272
    result = self.rpc.call_node_stop_master(master, False)
1273
    result.Raise("Could not disable the master role")
1274

    
1275
    return master
1276

    
1277

    
1278
def _VerifyCertificate(filename):
1279
  """Verifies a certificate for L{LUClusterVerifyConfig}.
1280

1281
  @type filename: string
1282
  @param filename: Path to PEM file
1283

1284
  """
1285
  try:
1286
    cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
1287
                                           utils.ReadFile(filename))
1288
  except Exception, err: # pylint: disable-msg=W0703
1289
    return (LUClusterVerifyConfig.ETYPE_ERROR,
1290
            "Failed to load X509 certificate %s: %s" % (filename, err))
1291

    
1292
  (errcode, msg) = \
1293
    utils.VerifyX509Certificate(cert, constants.SSL_CERT_EXPIRATION_WARN,
1294
                                constants.SSL_CERT_EXPIRATION_ERROR)
1295

    
1296
  if msg:
1297
    fnamemsg = "While verifying %s: %s" % (filename, msg)
1298
  else:
1299
    fnamemsg = None
1300

    
1301
  if errcode is None:
1302
    return (None, fnamemsg)
1303
  elif errcode == utils.CERT_WARNING:
1304
    return (LUClusterVerifyConfig.ETYPE_WARNING, fnamemsg)
1305
  elif errcode == utils.CERT_ERROR:
1306
    return (LUClusterVerifyConfig.ETYPE_ERROR, fnamemsg)
1307

    
1308
  raise errors.ProgrammerError("Unhandled certificate error code %r" % errcode)
1309

    
1310

    
1311
def _GetAllHypervisorParameters(cluster, instances):
1312
  """Compute the set of all hypervisor parameters.
1313

1314
  @type cluster: L{objects.Cluster}
1315
  @param cluster: the cluster object
1316
  @param instances: list of L{objects.Instance}
1317
  @param instances: additional instances from which to obtain parameters
1318
  @rtype: list of (origin, hypervisor, parameters)
1319
  @return: a list with all parameters found, indicating the hypervisor they
1320
       apply to, and the origin (can be "cluster", "os X", or "instance Y")
1321

1322
  """
1323
  hvp_data = []
1324

    
1325
  for hv_name in cluster.enabled_hypervisors:
1326
    hvp_data.append(("cluster", hv_name, cluster.GetHVDefaults(hv_name)))
1327

    
1328
  for os_name, os_hvp in cluster.os_hvp.items():
1329
    for hv_name, hv_params in os_hvp.items():
1330
      if hv_params:
1331
        full_params = cluster.GetHVDefaults(hv_name, os_name=os_name)
1332
        hvp_data.append(("os %s" % os_name, hv_name, full_params))
1333

    
1334
  # TODO: collapse identical parameter values in a single one
1335
  for instance in instances:
1336
    if instance.hvparams:
1337
      hvp_data.append(("instance %s" % instance.name, instance.hypervisor,
1338
                       cluster.FillHV(instance)))
1339

    
1340
  return hvp_data
1341

    
1342

    
1343
class _VerifyErrors(object):
1344
  """Mix-in for cluster/group verify LUs.
1345

1346
  It provides _Error and _ErrorIf, and updates the self.bad boolean. (Expects
1347
  self.op and self._feedback_fn to be available.)
1348

1349
  """
1350
  TCLUSTER = "cluster"
1351
  TNODE = "node"
1352
  TINSTANCE = "instance"
1353

    
1354
  ECLUSTERCFG = (TCLUSTER, "ECLUSTERCFG")
1355
  ECLUSTERCERT = (TCLUSTER, "ECLUSTERCERT")
1356
  ECLUSTERFILECHECK = (TCLUSTER, "ECLUSTERFILECHECK")
1357
  ECLUSTERDANGLINGNODES = (TNODE, "ECLUSTERDANGLINGNODES")
1358
  ECLUSTERDANGLINGINST = (TNODE, "ECLUSTERDANGLINGINST")
1359
  EINSTANCEBADNODE = (TINSTANCE, "EINSTANCEBADNODE")
1360
  EINSTANCEDOWN = (TINSTANCE, "EINSTANCEDOWN")
1361
  EINSTANCELAYOUT = (TINSTANCE, "EINSTANCELAYOUT")
1362
  EINSTANCEMISSINGDISK = (TINSTANCE, "EINSTANCEMISSINGDISK")
1363
  EINSTANCEFAULTYDISK = (TINSTANCE, "EINSTANCEFAULTYDISK")
1364
  EINSTANCEWRONGNODE = (TINSTANCE, "EINSTANCEWRONGNODE")
1365
  EINSTANCESPLITGROUPS = (TINSTANCE, "EINSTANCESPLITGROUPS")
1366
  ENODEDRBD = (TNODE, "ENODEDRBD")
1367
  ENODEDRBDHELPER = (TNODE, "ENODEDRBDHELPER")
1368
  ENODEFILECHECK = (TNODE, "ENODEFILECHECK")
1369
  ENODEHOOKS = (TNODE, "ENODEHOOKS")
1370
  ENODEHV = (TNODE, "ENODEHV")
1371
  ENODELVM = (TNODE, "ENODELVM")
1372
  ENODEN1 = (TNODE, "ENODEN1")
1373
  ENODENET = (TNODE, "ENODENET")
1374
  ENODEOS = (TNODE, "ENODEOS")
1375
  ENODEORPHANINSTANCE = (TNODE, "ENODEORPHANINSTANCE")
1376
  ENODEORPHANLV = (TNODE, "ENODEORPHANLV")
1377
  ENODERPC = (TNODE, "ENODERPC")
1378
  ENODESSH = (TNODE, "ENODESSH")
1379
  ENODEVERSION = (TNODE, "ENODEVERSION")
1380
  ENODESETUP = (TNODE, "ENODESETUP")
1381
  ENODETIME = (TNODE, "ENODETIME")
1382
  ENODEOOBPATH = (TNODE, "ENODEOOBPATH")
1383

    
1384
  ETYPE_FIELD = "code"
1385
  ETYPE_ERROR = "ERROR"
1386
  ETYPE_WARNING = "WARNING"
1387

    
1388
  def _Error(self, ecode, item, msg, *args, **kwargs):
1389
    """Format an error message.
1390

1391
    Based on the opcode's error_codes parameter, either format a
1392
    parseable error code, or a simpler error string.
1393

1394
    This must be called only from Exec and functions called from Exec.
1395

1396
    """
1397
    ltype = kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR)
1398
    itype, etxt = ecode
1399
    # first complete the msg
1400
    if args:
1401
      msg = msg % args
1402
    # then format the whole message
1403
    if self.op.error_codes: # This is a mix-in. pylint: disable-msg=E1101
1404
      msg = "%s:%s:%s:%s:%s" % (ltype, etxt, itype, item, msg)
1405
    else:
1406
      if item:
1407
        item = " " + item
1408
      else:
1409
        item = ""
1410
      msg = "%s: %s%s: %s" % (ltype, itype, item, msg)
1411
    # and finally report it via the feedback_fn
1412
    self._feedback_fn("  - %s" % msg) # Mix-in. pylint: disable-msg=E1101
1413

    
1414
  def _ErrorIf(self, cond, *args, **kwargs):
1415
    """Log an error message if the passed condition is True.
1416

1417
    """
1418
    cond = (bool(cond)
1419
            or self.op.debug_simulate_errors) # pylint: disable-msg=E1101
1420
    if cond:
1421
      self._Error(*args, **kwargs)
1422
    # do not mark the operation as failed for WARN cases only
1423
    if kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR) == self.ETYPE_ERROR:
1424
      self.bad = self.bad or cond
1425

    
1426

    
1427
class LUClusterVerifyConfig(NoHooksLU, _VerifyErrors):
1428
  """Verifies the cluster config.
1429

1430
  """
1431
  REQ_BGL = True
1432

    
1433
  def _VerifyHVP(self, hvp_data):
1434
    """Verifies locally the syntax of the hypervisor parameters.
1435

1436
    """
1437
    for item, hv_name, hv_params in hvp_data:
1438
      msg = ("hypervisor %s parameters syntax check (source %s): %%s" %
1439
             (item, hv_name))
1440
      try:
1441
        hv_class = hypervisor.GetHypervisor(hv_name)
1442
        utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
1443
        hv_class.CheckParameterSyntax(hv_params)
1444
      except errors.GenericError, err:
1445
        self._ErrorIf(True, self.ECLUSTERCFG, None, msg % str(err))
1446

    
1447
  def ExpandNames(self):
1448
    # Information can be safely retrieved as the BGL is acquired in exclusive
1449
    # mode
1450
    self.all_group_info = self.cfg.GetAllNodeGroupsInfo()
1451
    self.all_node_info = self.cfg.GetAllNodesInfo()
1452
    self.all_inst_info = self.cfg.GetAllInstancesInfo()
1453
    self.needed_locks = {}
1454

    
1455
  def Exec(self, feedback_fn):
1456
    """Verify integrity of cluster, performing various test on nodes.
1457

1458
    """
1459
    self.bad = False
1460
    self._feedback_fn = feedback_fn
1461

    
1462
    feedback_fn("* Verifying cluster config")
1463

    
1464
    for msg in self.cfg.VerifyConfig():
1465
      self._ErrorIf(True, self.ECLUSTERCFG, None, msg)
1466

    
1467
    feedback_fn("* Verifying cluster certificate files")
1468

    
1469
    for cert_filename in constants.ALL_CERT_FILES:
1470
      (errcode, msg) = _VerifyCertificate(cert_filename)
1471
      self._ErrorIf(errcode, self.ECLUSTERCERT, None, msg, code=errcode)
1472

    
1473
    feedback_fn("* Verifying hypervisor parameters")
1474

    
1475
    self._VerifyHVP(_GetAllHypervisorParameters(self.cfg.GetClusterInfo(),
1476
                                                self.all_inst_info.values()))
1477

    
1478
    feedback_fn("* Verifying all nodes belong to an existing group")
1479

    
1480
    # We do this verification here because, should this bogus circumstance
1481
    # occur, it would never be caught by VerifyGroup, which only acts on
1482
    # nodes/instances reachable from existing node groups.
1483

    
1484
    dangling_nodes = set(node.name for node in self.all_node_info.values()
1485
                         if node.group not in self.all_group_info)
1486

    
1487
    dangling_instances = {}
1488
    no_node_instances = []
1489

    
1490
    for inst in self.all_inst_info.values():
1491
      if inst.primary_node in dangling_nodes:
1492
        dangling_instances.setdefault(inst.primary_node, []).append(inst.name)
1493
      elif inst.primary_node not in self.all_node_info:
1494
        no_node_instances.append(inst.name)
1495

    
1496
    pretty_dangling = [
1497
        "%s (%s)" %
1498
        (node.name,
1499
         utils.CommaJoin(dangling_instances.get(node.name,
1500
                                                ["no instances"])))
1501
        for node in dangling_nodes]
1502

    
1503
    self._ErrorIf(bool(dangling_nodes), self.ECLUSTERDANGLINGNODES, None,
1504
                  "the following nodes (and their instances) belong to a non"
1505
                  " existing group: %s", utils.CommaJoin(pretty_dangling))
1506

    
1507
    self._ErrorIf(bool(no_node_instances), self.ECLUSTERDANGLINGINST, None,
1508
                  "the following instances have a non-existing primary-node:"
1509
                  " %s", utils.CommaJoin(no_node_instances))
1510

    
1511
    return (not self.bad, [g.name for g in self.all_group_info.values()])
1512

    
1513

    
1514
class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors):
1515
  """Verifies the status of a node group.
1516

1517
  """
1518
  HPATH = "cluster-verify"
1519
  HTYPE = constants.HTYPE_CLUSTER
1520
  REQ_BGL = False
1521

    
1522
  _HOOKS_INDENT_RE = re.compile("^", re.M)
1523

    
1524
  class NodeImage(object):
1525
    """A class representing the logical and physical status of a node.
1526

1527
    @type name: string
1528
    @ivar name: the node name to which this object refers
1529
    @ivar volumes: a structure as returned from
1530
        L{ganeti.backend.GetVolumeList} (runtime)
1531
    @ivar instances: a list of running instances (runtime)
1532
    @ivar pinst: list of configured primary instances (config)
1533
    @ivar sinst: list of configured secondary instances (config)
1534
    @ivar sbp: dictionary of {primary-node: list of instances} for all
1535
        instances for which this node is secondary (config)
1536
    @ivar mfree: free memory, as reported by hypervisor (runtime)
1537
    @ivar dfree: free disk, as reported by the node (runtime)
1538
    @ivar offline: the offline status (config)
1539
    @type rpc_fail: boolean
1540
    @ivar rpc_fail: whether the RPC verify call was successfull (overall,
1541
        not whether the individual keys were correct) (runtime)
1542
    @type lvm_fail: boolean
1543
    @ivar lvm_fail: whether the RPC call didn't return valid LVM data
1544
    @type hyp_fail: boolean
1545
    @ivar hyp_fail: whether the RPC call didn't return the instance list
1546
    @type ghost: boolean
1547
    @ivar ghost: whether this is a known node or not (config)
1548
    @type os_fail: boolean
1549
    @ivar os_fail: whether the RPC call didn't return valid OS data
1550
    @type oslist: list
1551
    @ivar oslist: list of OSes as diagnosed by DiagnoseOS
1552
    @type vm_capable: boolean
1553
    @ivar vm_capable: whether the node can host instances
1554

1555
    """
1556
    def __init__(self, offline=False, name=None, vm_capable=True):
1557
      self.name = name
1558
      self.volumes = {}
1559
      self.instances = []
1560
      self.pinst = []
1561
      self.sinst = []
1562
      self.sbp = {}
1563
      self.mfree = 0
1564
      self.dfree = 0
1565
      self.offline = offline
1566
      self.vm_capable = vm_capable
1567
      self.rpc_fail = False
1568
      self.lvm_fail = False
1569
      self.hyp_fail = False
1570
      self.ghost = False
1571
      self.os_fail = False
1572
      self.oslist = {}
1573

    
1574
  def ExpandNames(self):
1575
    # This raises errors.OpPrereqError on its own:
1576
    self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name)
1577

    
1578
    # Get instances in node group; this is unsafe and needs verification later
1579
    inst_names = self.cfg.GetNodeGroupInstances(self.group_uuid)
1580

    
1581
    self.needed_locks = {
1582
      locking.LEVEL_INSTANCE: inst_names,
1583
      locking.LEVEL_NODEGROUP: [self.group_uuid],
1584
      locking.LEVEL_NODE: [],
1585
      }
1586

    
1587
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
1588

    
1589
  def DeclareLocks(self, level):
1590
    if level == locking.LEVEL_NODE:
1591
      # Get members of node group; this is unsafe and needs verification later
1592
      nodes = set(self.cfg.GetNodeGroup(self.group_uuid).members)
1593

    
1594
      all_inst_info = self.cfg.GetAllInstancesInfo()
1595

    
1596
      # In Exec(), we warn about mirrored instances that have primary and
1597
      # secondary living in separate node groups. To fully verify that
1598
      # volumes for these instances are healthy, we will need to do an
1599
      # extra call to their secondaries. We ensure here those nodes will
1600
      # be locked.
1601
      for inst in self.glm.list_owned(locking.LEVEL_INSTANCE):
1602
        # Important: access only the instances whose lock is owned
1603
        if all_inst_info[inst].disk_template in constants.DTS_INT_MIRROR:
1604
          nodes.update(all_inst_info[inst].secondary_nodes)
1605

    
1606
      self.needed_locks[locking.LEVEL_NODE] = nodes
1607

    
1608
  def CheckPrereq(self):
1609
    group_nodes = set(self.cfg.GetNodeGroup(self.group_uuid).members)
1610
    group_instances = self.cfg.GetNodeGroupInstances(self.group_uuid)
1611

    
1612
    unlocked_nodes = \
1613
        group_nodes.difference(self.glm.list_owned(locking.LEVEL_NODE))
1614

    
1615
    unlocked_instances = \
1616
        group_instances.difference(self.glm.list_owned(locking.LEVEL_INSTANCE))
1617

    
1618
    if unlocked_nodes:
1619
      raise errors.OpPrereqError("Missing lock for nodes: %s" %
1620
                                 utils.CommaJoin(unlocked_nodes))
1621

    
1622
    if unlocked_instances:
1623
      raise errors.OpPrereqError("Missing lock for instances: %s" %
1624
                                 utils.CommaJoin(unlocked_instances))
1625

    
1626
    self.all_node_info = self.cfg.GetAllNodesInfo()
1627
    self.all_inst_info = self.cfg.GetAllInstancesInfo()
1628

    
1629
    self.my_node_names = utils.NiceSort(group_nodes)
1630
    self.my_inst_names = utils.NiceSort(group_instances)
1631

    
1632
    self.my_node_info = dict((name, self.all_node_info[name])
1633
                             for name in self.my_node_names)
1634

    
1635
    self.my_inst_info = dict((name, self.all_inst_info[name])
1636
                             for name in self.my_inst_names)
1637

    
1638
    # We detect here the nodes that will need the extra RPC calls for verifying
1639
    # split LV volumes; they should be locked.
1640
    extra_lv_nodes = set()
1641

    
1642
    for inst in self.my_inst_info.values():
1643
      if inst.disk_template in constants.DTS_INT_MIRROR:
1644
        group = self.my_node_info[inst.primary_node].group
1645
        for nname in inst.secondary_nodes:
1646
          if self.all_node_info[nname].group != group:
1647
            extra_lv_nodes.add(nname)
1648

    
1649
    unlocked_lv_nodes = \
1650
        extra_lv_nodes.difference(self.glm.list_owned(locking.LEVEL_NODE))
1651

    
1652
    if unlocked_lv_nodes:
1653
      raise errors.OpPrereqError("these nodes could be locked: %s" %
1654
                                 utils.CommaJoin(unlocked_lv_nodes))
1655
    self.extra_lv_nodes = list(extra_lv_nodes)
1656

    
1657
  def _VerifyNode(self, ninfo, nresult):
1658
    """Perform some basic validation on data returned from a node.
1659

1660
      - check the result data structure is well formed and has all the
1661
        mandatory fields
1662
      - check ganeti version
1663

1664
    @type ninfo: L{objects.Node}
1665
    @param ninfo: the node to check
1666
    @param nresult: the results from the node
1667
    @rtype: boolean
1668
    @return: whether overall this call was successful (and we can expect
1669
         reasonable values in the respose)
1670

1671
    """
1672
    node = ninfo.name
1673
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1674

    
1675
    # main result, nresult should be a non-empty dict
1676
    test = not nresult or not isinstance(nresult, dict)
1677
    _ErrorIf(test, self.ENODERPC, node,
1678
                  "unable to verify node: no data returned")
1679
    if test:
1680
      return False
1681

    
1682
    # compares ganeti version
1683
    local_version = constants.PROTOCOL_VERSION
1684
    remote_version = nresult.get("version", None)
1685
    test = not (remote_version and
1686
                isinstance(remote_version, (list, tuple)) and
1687
                len(remote_version) == 2)
1688
    _ErrorIf(test, self.ENODERPC, node,
1689
             "connection to node returned invalid data")
1690
    if test:
1691
      return False
1692

    
1693
    test = local_version != remote_version[0]
1694
    _ErrorIf(test, self.ENODEVERSION, node,
1695
             "incompatible protocol versions: master %s,"
1696
             " node %s", local_version, remote_version[0])
1697
    if test:
1698
      return False
1699

    
1700
    # node seems compatible, we can actually try to look into its results
1701

    
1702
    # full package version
1703
    self._ErrorIf(constants.RELEASE_VERSION != remote_version[1],
1704
                  self.ENODEVERSION, node,
1705
                  "software version mismatch: master %s, node %s",
1706
                  constants.RELEASE_VERSION, remote_version[1],
1707
                  code=self.ETYPE_WARNING)
1708

    
1709
    hyp_result = nresult.get(constants.NV_HYPERVISOR, None)
1710
    if ninfo.vm_capable and isinstance(hyp_result, dict):
1711
      for hv_name, hv_result in hyp_result.iteritems():
1712
        test = hv_result is not None
1713
        _ErrorIf(test, self.ENODEHV, node,
1714
                 "hypervisor %s verify failure: '%s'", hv_name, hv_result)
1715

    
1716
    hvp_result = nresult.get(constants.NV_HVPARAMS, None)
1717
    if ninfo.vm_capable and isinstance(hvp_result, list):
1718
      for item, hv_name, hv_result in hvp_result:
1719
        _ErrorIf(True, self.ENODEHV, node,
1720
                 "hypervisor %s parameter verify failure (source %s): %s",
1721
                 hv_name, item, hv_result)
1722

    
1723
    test = nresult.get(constants.NV_NODESETUP,
1724
                       ["Missing NODESETUP results"])
1725
    _ErrorIf(test, self.ENODESETUP, node, "node setup error: %s",
1726
             "; ".join(test))
1727

    
1728
    return True
1729

    
1730
  def _VerifyNodeTime(self, ninfo, nresult,
1731
                      nvinfo_starttime, nvinfo_endtime):
1732
    """Check the node time.
1733

1734
    @type ninfo: L{objects.Node}
1735
    @param ninfo: the node to check
1736
    @param nresult: the remote results for the node
1737
    @param nvinfo_starttime: the start time of the RPC call
1738
    @param nvinfo_endtime: the end time of the RPC call
1739

1740
    """
1741
    node = ninfo.name
1742
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1743

    
1744
    ntime = nresult.get(constants.NV_TIME, None)
1745
    try:
1746
      ntime_merged = utils.MergeTime(ntime)
1747
    except (ValueError, TypeError):
1748
      _ErrorIf(True, self.ENODETIME, node, "Node returned invalid time")
1749
      return
1750

    
1751
    if ntime_merged < (nvinfo_starttime - constants.NODE_MAX_CLOCK_SKEW):
1752
      ntime_diff = "%.01fs" % abs(nvinfo_starttime - ntime_merged)
1753
    elif ntime_merged > (nvinfo_endtime + constants.NODE_MAX_CLOCK_SKEW):
1754
      ntime_diff = "%.01fs" % abs(ntime_merged - nvinfo_endtime)
1755
    else:
1756
      ntime_diff = None
1757

    
1758
    _ErrorIf(ntime_diff is not None, self.ENODETIME, node,
1759
             "Node time diverges by at least %s from master node time",
1760
             ntime_diff)
1761

    
1762
  def _VerifyNodeLVM(self, ninfo, nresult, vg_name):
1763
    """Check the node LVM results.
1764

1765
    @type ninfo: L{objects.Node}
1766
    @param ninfo: the node to check
1767
    @param nresult: the remote results for the node
1768
    @param vg_name: the configured VG name
1769

1770
    """
1771
    if vg_name is None:
1772
      return
1773

    
1774
    node = ninfo.name
1775
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1776

    
1777
    # checks vg existence and size > 20G
1778
    vglist = nresult.get(constants.NV_VGLIST, None)
1779
    test = not vglist
1780
    _ErrorIf(test, self.ENODELVM, node, "unable to check volume groups")
1781
    if not test:
1782
      vgstatus = utils.CheckVolumeGroupSize(vglist, vg_name,
1783
                                            constants.MIN_VG_SIZE)
1784
      _ErrorIf(vgstatus, self.ENODELVM, node, vgstatus)
1785

    
1786
    # check pv names
1787
    pvlist = nresult.get(constants.NV_PVLIST, None)
1788
    test = pvlist is None
1789
    _ErrorIf(test, self.ENODELVM, node, "Can't get PV list from node")
1790
    if not test:
1791
      # check that ':' is not present in PV names, since it's a
1792
      # special character for lvcreate (denotes the range of PEs to
1793
      # use on the PV)
1794
      for _, pvname, owner_vg in pvlist:
1795
        test = ":" in pvname
1796
        _ErrorIf(test, self.ENODELVM, node, "Invalid character ':' in PV"
1797
                 " '%s' of VG '%s'", pvname, owner_vg)
1798

    
1799
  def _VerifyNodeBridges(self, ninfo, nresult, bridges):
1800
    """Check the node bridges.
1801

1802
    @type ninfo: L{objects.Node}
1803
    @param ninfo: the node to check
1804
    @param nresult: the remote results for the node
1805
    @param bridges: the expected list of bridges
1806

1807
    """
1808
    if not bridges:
1809
      return
1810

    
1811
    node = ninfo.name
1812
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1813

    
1814
    missing = nresult.get(constants.NV_BRIDGES, None)
1815
    test = not isinstance(missing, list)
1816
    _ErrorIf(test, self.ENODENET, node,
1817
             "did not return valid bridge information")
1818
    if not test:
1819
      _ErrorIf(bool(missing), self.ENODENET, node, "missing bridges: %s" %
1820
               utils.CommaJoin(sorted(missing)))
1821

    
1822
  def _VerifyNodeNetwork(self, ninfo, nresult):
1823
    """Check the node network connectivity results.
1824

1825
    @type ninfo: L{objects.Node}
1826
    @param ninfo: the node to check
1827
    @param nresult: the remote results for the node
1828

1829
    """
1830
    node = ninfo.name
1831
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1832

    
1833
    test = constants.NV_NODELIST not in nresult
1834
    _ErrorIf(test, self.ENODESSH, node,
1835
             "node hasn't returned node ssh connectivity data")
1836
    if not test:
1837
      if nresult[constants.NV_NODELIST]:
1838
        for a_node, a_msg in nresult[constants.NV_NODELIST].items():
1839
          _ErrorIf(True, self.ENODESSH, node,
1840
                   "ssh communication with node '%s': %s", a_node, a_msg)
1841

    
1842
    test = constants.NV_NODENETTEST not in nresult
1843
    _ErrorIf(test, self.ENODENET, node,
1844
             "node hasn't returned node tcp connectivity data")
1845
    if not test:
1846
      if nresult[constants.NV_NODENETTEST]:
1847
        nlist = utils.NiceSort(nresult[constants.NV_NODENETTEST].keys())
1848
        for anode in nlist:
1849
          _ErrorIf(True, self.ENODENET, node,
1850
                   "tcp communication with node '%s': %s",
1851
                   anode, nresult[constants.NV_NODENETTEST][anode])
1852

    
1853
    test = constants.NV_MASTERIP not in nresult
1854
    _ErrorIf(test, self.ENODENET, node,
1855
             "node hasn't returned node master IP reachability data")
1856
    if not test:
1857
      if not nresult[constants.NV_MASTERIP]:
1858
        if node == self.master_node:
1859
          msg = "the master node cannot reach the master IP (not configured?)"
1860
        else:
1861
          msg = "cannot reach the master IP"
1862
        _ErrorIf(True, self.ENODENET, node, msg)
1863

    
1864
  def _VerifyInstance(self, instance, instanceconfig, node_image,
1865
                      diskstatus):
1866
    """Verify an instance.
1867

1868
    This function checks to see if the required block devices are
1869
    available on the instance's node.
1870

1871
    """
1872
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1873
    node_current = instanceconfig.primary_node
1874

    
1875
    node_vol_should = {}
1876
    instanceconfig.MapLVsByNode(node_vol_should)
1877

    
1878
    for node in node_vol_should:
1879
      n_img = node_image[node]
1880
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1881
        # ignore missing volumes on offline or broken nodes
1882
        continue
1883
      for volume in node_vol_should[node]:
1884
        test = volume not in n_img.volumes
1885
        _ErrorIf(test, self.EINSTANCEMISSINGDISK, instance,
1886
                 "volume %s missing on node %s", volume, node)
1887

    
1888
    if instanceconfig.admin_up:
1889
      pri_img = node_image[node_current]
1890
      test = instance not in pri_img.instances and not pri_img.offline
1891
      _ErrorIf(test, self.EINSTANCEDOWN, instance,
1892
               "instance not running on its primary node %s",
1893
               node_current)
1894

    
1895
    diskdata = [(nname, success, status, idx)
1896
                for (nname, disks) in diskstatus.items()
1897
                for idx, (success, status) in enumerate(disks)]
1898

    
1899
    for nname, success, bdev_status, idx in diskdata:
1900
      # the 'ghost node' construction in Exec() ensures that we have a
1901
      # node here
1902
      snode = node_image[nname]
1903
      bad_snode = snode.ghost or snode.offline
1904
      _ErrorIf(instanceconfig.admin_up and not success and not bad_snode,
1905
               self.EINSTANCEFAULTYDISK, instance,
1906
               "couldn't retrieve status for disk/%s on %s: %s",
1907
               idx, nname, bdev_status)
1908
      _ErrorIf((instanceconfig.admin_up and success and
1909
                bdev_status.ldisk_status == constants.LDS_FAULTY),
1910
               self.EINSTANCEFAULTYDISK, instance,
1911
               "disk/%s on %s is faulty", idx, nname)
1912

    
1913
  def _VerifyOrphanVolumes(self, node_vol_should, node_image, reserved):
1914
    """Verify if there are any unknown volumes in the cluster.
1915

1916
    The .os, .swap and backup volumes are ignored. All other volumes are
1917
    reported as unknown.
1918

1919
    @type reserved: L{ganeti.utils.FieldSet}
1920
    @param reserved: a FieldSet of reserved volume names
1921

1922
    """
1923
    for node, n_img in node_image.items():
1924
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1925
        # skip non-healthy nodes
1926
        continue
1927
      for volume in n_img.volumes:
1928
        test = ((node not in node_vol_should or
1929
                volume not in node_vol_should[node]) and
1930
                not reserved.Matches(volume))
1931
        self._ErrorIf(test, self.ENODEORPHANLV, node,
1932
                      "volume %s is unknown", volume)
1933

    
1934
  def _VerifyNPlusOneMemory(self, node_image, instance_cfg):
1935
    """Verify N+1 Memory Resilience.
1936

1937
    Check that if one single node dies we can still start all the
1938
    instances it was primary for.
1939

1940
    """
1941
    cluster_info = self.cfg.GetClusterInfo()
1942
    for node, n_img in node_image.items():
1943
      # This code checks that every node which is now listed as
1944
      # secondary has enough memory to host all instances it is
1945
      # supposed to should a single other node in the cluster fail.
1946
      # FIXME: not ready for failover to an arbitrary node
1947
      # FIXME: does not support file-backed instances
1948
      # WARNING: we currently take into account down instances as well
1949
      # as up ones, considering that even if they're down someone
1950
      # might want to start them even in the event of a node failure.
1951
      if n_img.offline:
1952
        # we're skipping offline nodes from the N+1 warning, since
1953
        # most likely we don't have good memory infromation from them;
1954
        # we already list instances living on such nodes, and that's
1955
        # enough warning
1956
        continue
1957
      for prinode, instances in n_img.sbp.items():
1958
        needed_mem = 0
1959
        for instance in instances:
1960
          bep = cluster_info.FillBE(instance_cfg[instance])
1961
          if bep[constants.BE_AUTO_BALANCE]:
1962
            needed_mem += bep[constants.BE_MEMORY]
1963
        test = n_img.mfree < needed_mem
1964
        self._ErrorIf(test, self.ENODEN1, node,
1965
                      "not enough memory to accomodate instance failovers"
1966
                      " should node %s fail (%dMiB needed, %dMiB available)",
1967
                      prinode, needed_mem, n_img.mfree)
1968

    
1969
  @classmethod
1970
  def _VerifyFiles(cls, errorif, nodeinfo, master_node, all_nvinfo,
1971
                   (files_all, files_all_opt, files_mc, files_vm)):
1972
    """Verifies file checksums collected from all nodes.
1973

1974
    @param errorif: Callback for reporting errors
1975
    @param nodeinfo: List of L{objects.Node} objects
1976
    @param master_node: Name of master node
1977
    @param all_nvinfo: RPC results
1978

1979
    """
1980
    node_names = frozenset(node.name for node in nodeinfo)
1981

    
1982
    assert master_node in node_names
1983
    assert (len(files_all | files_all_opt | files_mc | files_vm) ==
1984
            sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
1985
           "Found file listed in more than one file list"
1986

    
1987
    # Define functions determining which nodes to consider for a file
1988
    file2nodefn = dict([(filename, fn)
1989
      for (files, fn) in [(files_all, None),
1990
                          (files_all_opt, None),
1991
                          (files_mc, lambda node: (node.master_candidate or
1992
                                                   node.name == master_node)),
1993
                          (files_vm, lambda node: node.vm_capable)]
1994
      for filename in files])
1995

    
1996
    fileinfo = dict((filename, {}) for filename in file2nodefn.keys())
1997

    
1998
    for node in nodeinfo:
1999
      nresult = all_nvinfo[node.name]
2000

    
2001
      if nresult.fail_msg or not nresult.payload:
2002
        node_files = None
2003
      else:
2004
        node_files = nresult.payload.get(constants.NV_FILELIST, None)
2005

    
2006
      test = not (node_files and isinstance(node_files, dict))
2007
      errorif(test, cls.ENODEFILECHECK, node.name,
2008
              "Node did not return file checksum data")
2009
      if test:
2010
        continue
2011

    
2012
      for (filename, checksum) in node_files.items():
2013
        # Check if the file should be considered for a node
2014
        fn = file2nodefn[filename]
2015
        if fn is None or fn(node):
2016
          fileinfo[filename].setdefault(checksum, set()).add(node.name)
2017

    
2018
    for (filename, checksums) in fileinfo.items():
2019
      assert compat.all(len(i) > 10 for i in checksums), "Invalid checksum"
2020

    
2021
      # Nodes having the file
2022
      with_file = frozenset(node_name
2023
                            for nodes in fileinfo[filename].values()
2024
                            for node_name in nodes)
2025

    
2026
      # Nodes missing file
2027
      missing_file = node_names - with_file
2028

    
2029
      if filename in files_all_opt:
2030
        # All or no nodes
2031
        errorif(missing_file and missing_file != node_names,
2032
                cls.ECLUSTERFILECHECK, None,
2033
                "File %s is optional, but it must exist on all or no"
2034
                " nodes (not found on %s)",
2035
                filename, utils.CommaJoin(utils.NiceSort(missing_file)))
2036
      else:
2037
        errorif(missing_file, cls.ECLUSTERFILECHECK, None,
2038
                "File %s is missing from node(s) %s", filename,
2039
                utils.CommaJoin(utils.NiceSort(missing_file)))
2040

    
2041
      # See if there are multiple versions of the file
2042
      test = len(checksums) > 1
2043
      if test:
2044
        variants = ["variant %s on %s" %
2045
                    (idx + 1, utils.CommaJoin(utils.NiceSort(nodes)))
2046
                    for (idx, (checksum, nodes)) in
2047
                      enumerate(sorted(checksums.items()))]
2048
      else:
2049
        variants = []
2050

    
2051
      errorif(test, cls.ECLUSTERFILECHECK, None,
2052
              "File %s found with %s different checksums (%s)",
2053
              filename, len(checksums), "; ".join(variants))
2054

    
2055
  def _VerifyNodeDrbd(self, ninfo, nresult, instanceinfo, drbd_helper,
2056
                      drbd_map):
2057
    """Verifies and the node DRBD status.
2058

2059
    @type ninfo: L{objects.Node}
2060
    @param ninfo: the node to check
2061
    @param nresult: the remote results for the node
2062
    @param instanceinfo: the dict of instances
2063
    @param drbd_helper: the configured DRBD usermode helper
2064
    @param drbd_map: the DRBD map as returned by
2065
        L{ganeti.config.ConfigWriter.ComputeDRBDMap}
2066

2067
    """
2068
    node = ninfo.name
2069
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2070

    
2071
    if drbd_helper:
2072
      helper_result = nresult.get(constants.NV_DRBDHELPER, None)
2073
      test = (helper_result == None)
2074
      _ErrorIf(test, self.ENODEDRBDHELPER, node,
2075
               "no drbd usermode helper returned")
2076
      if helper_result:
2077
        status, payload = helper_result
2078
        test = not status
2079
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
2080
                 "drbd usermode helper check unsuccessful: %s", payload)
2081
        test = status and (payload != drbd_helper)
2082
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
2083
                 "wrong drbd usermode helper: %s", payload)
2084

    
2085
    # compute the DRBD minors
2086
    node_drbd = {}
2087
    for minor, instance in drbd_map[node].items():
2088
      test = instance not in instanceinfo
2089
      _ErrorIf(test, self.ECLUSTERCFG, None,
2090
               "ghost instance '%s' in temporary DRBD map", instance)
2091
        # ghost instance should not be running, but otherwise we
2092
        # don't give double warnings (both ghost instance and
2093
        # unallocated minor in use)
2094
      if test:
2095
        node_drbd[minor] = (instance, False)
2096
      else:
2097
        instance = instanceinfo[instance]
2098
        node_drbd[minor] = (instance.name, instance.admin_up)
2099

    
2100
    # and now check them
2101
    used_minors = nresult.get(constants.NV_DRBDLIST, [])
2102
    test = not isinstance(used_minors, (tuple, list))
2103
    _ErrorIf(test, self.ENODEDRBD, node,
2104
             "cannot parse drbd status file: %s", str(used_minors))
2105
    if test:
2106
      # we cannot check drbd status
2107
      return
2108

    
2109
    for minor, (iname, must_exist) in node_drbd.items():
2110
      test = minor not in used_minors and must_exist
2111
      _ErrorIf(test, self.ENODEDRBD, node,
2112
               "drbd minor %d of instance %s is not active", minor, iname)
2113
    for minor in used_minors:
2114
      test = minor not in node_drbd
2115
      _ErrorIf(test, self.ENODEDRBD, node,
2116
               "unallocated drbd minor %d is in use", minor)
2117

    
2118
  def _UpdateNodeOS(self, ninfo, nresult, nimg):
2119
    """Builds the node OS structures.
2120

2121
    @type ninfo: L{objects.Node}
2122
    @param ninfo: the node to check
2123
    @param nresult: the remote results for the node
2124
    @param nimg: the node image object
2125

2126
    """
2127
    node = ninfo.name
2128
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2129

    
2130
    remote_os = nresult.get(constants.NV_OSLIST, None)
2131
    test = (not isinstance(remote_os, list) or
2132
            not compat.all(isinstance(v, list) and len(v) == 7
2133
                           for v in remote_os))
2134

    
2135
    _ErrorIf(test, self.ENODEOS, node,
2136
             "node hasn't returned valid OS data")
2137

    
2138
    nimg.os_fail = test
2139

    
2140
    if test:
2141
      return
2142

    
2143
    os_dict = {}
2144

    
2145
    for (name, os_path, status, diagnose,
2146
         variants, parameters, api_ver) in nresult[constants.NV_OSLIST]:
2147

    
2148
      if name not in os_dict:
2149
        os_dict[name] = []
2150

    
2151
      # parameters is a list of lists instead of list of tuples due to
2152
      # JSON lacking a real tuple type, fix it:
2153
      parameters = [tuple(v) for v in parameters]
2154
      os_dict[name].append((os_path, status, diagnose,
2155
                            set(variants), set(parameters), set(api_ver)))
2156

    
2157
    nimg.oslist = os_dict
2158

    
2159
  def _VerifyNodeOS(self, ninfo, nimg, base):
2160
    """Verifies the node OS list.
2161

2162
    @type ninfo: L{objects.Node}
2163
    @param ninfo: the node to check
2164
    @param nimg: the node image object
2165
    @param base: the 'template' node we match against (e.g. from the master)
2166

2167
    """
2168
    node = ninfo.name
2169
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2170

    
2171
    assert not nimg.os_fail, "Entered _VerifyNodeOS with failed OS rpc?"
2172

    
2173
    beautify_params = lambda l: ["%s: %s" % (k, v) for (k, v) in l]
2174
    for os_name, os_data in nimg.oslist.items():
2175
      assert os_data, "Empty OS status for OS %s?!" % os_name
2176
      f_path, f_status, f_diag, f_var, f_param, f_api = os_data[0]
2177
      _ErrorIf(not f_status, self.ENODEOS, node,
2178
               "Invalid OS %s (located at %s): %s", os_name, f_path, f_diag)
2179
      _ErrorIf(len(os_data) > 1, self.ENODEOS, node,
2180
               "OS '%s' has multiple entries (first one shadows the rest): %s",
2181
               os_name, utils.CommaJoin([v[0] for v in os_data]))
2182
      # this will catched in backend too
2183
      _ErrorIf(compat.any(v >= constants.OS_API_V15 for v in f_api)
2184
               and not f_var, self.ENODEOS, node,
2185
               "OS %s with API at least %d does not declare any variant",
2186
               os_name, constants.OS_API_V15)
2187
      # comparisons with the 'base' image
2188
      test = os_name not in base.oslist
2189
      _ErrorIf(test, self.ENODEOS, node,
2190
               "Extra OS %s not present on reference node (%s)",
2191
               os_name, base.name)
2192
      if test:
2193
        continue
2194
      assert base.oslist[os_name], "Base node has empty OS status?"
2195
      _, b_status, _, b_var, b_param, b_api = base.oslist[os_name][0]
2196
      if not b_status:
2197
        # base OS is invalid, skipping
2198
        continue
2199
      for kind, a, b in [("API version", f_api, b_api),
2200
                         ("variants list", f_var, b_var),
2201
                         ("parameters", beautify_params(f_param),
2202
                          beautify_params(b_param))]:
2203
        _ErrorIf(a != b, self.ENODEOS, node,
2204
                 "OS %s for %s differs from reference node %s: [%s] vs. [%s]",
2205
                 kind, os_name, base.name,
2206
                 utils.CommaJoin(sorted(a)), utils.CommaJoin(sorted(b)))
2207

    
2208
    # check any missing OSes
2209
    missing = set(base.oslist.keys()).difference(nimg.oslist.keys())
2210
    _ErrorIf(missing, self.ENODEOS, node,
2211
             "OSes present on reference node %s but missing on this node: %s",
2212
             base.name, utils.CommaJoin(missing))
2213

    
2214
  def _VerifyOob(self, ninfo, nresult):
2215
    """Verifies out of band functionality of a node.
2216

2217
    @type ninfo: L{objects.Node}
2218
    @param ninfo: the node to check
2219
    @param nresult: the remote results for the node
2220

2221
    """
2222
    node = ninfo.name
2223
    # We just have to verify the paths on master and/or master candidates
2224
    # as the oob helper is invoked on the master
2225
    if ((ninfo.master_candidate or ninfo.master_capable) and
2226
        constants.NV_OOB_PATHS in nresult):
2227
      for path_result in nresult[constants.NV_OOB_PATHS]:
2228
        self._ErrorIf(path_result, self.ENODEOOBPATH, node, path_result)
2229

    
2230
  def _UpdateNodeVolumes(self, ninfo, nresult, nimg, vg_name):
2231
    """Verifies and updates the node volume data.
2232

2233
    This function will update a L{NodeImage}'s internal structures
2234
    with data from the remote call.
2235

2236
    @type ninfo: L{objects.Node}
2237
    @param ninfo: the node to check
2238
    @param nresult: the remote results for the node
2239
    @param nimg: the node image object
2240
    @param vg_name: the configured VG name
2241

2242
    """
2243
    node = ninfo.name
2244
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2245

    
2246
    nimg.lvm_fail = True
2247
    lvdata = nresult.get(constants.NV_LVLIST, "Missing LV data")
2248
    if vg_name is None:
2249
      pass
2250
    elif isinstance(lvdata, basestring):
2251
      _ErrorIf(True, self.ENODELVM, node, "LVM problem on node: %s",
2252
               utils.SafeEncode(lvdata))
2253
    elif not isinstance(lvdata, dict):
2254
      _ErrorIf(True, self.ENODELVM, node, "rpc call to node failed (lvlist)")
2255
    else:
2256
      nimg.volumes = lvdata
2257
      nimg.lvm_fail = False
2258

    
2259
  def _UpdateNodeInstances(self, ninfo, nresult, nimg):
2260
    """Verifies and updates the node instance list.
2261

2262
    If the listing was successful, then updates this node's instance
2263
    list. Otherwise, it marks the RPC call as failed for the instance
2264
    list key.
2265

2266
    @type ninfo: L{objects.Node}
2267
    @param ninfo: the node to check
2268
    @param nresult: the remote results for the node
2269
    @param nimg: the node image object
2270

2271
    """
2272
    idata = nresult.get(constants.NV_INSTANCELIST, None)
2273
    test = not isinstance(idata, list)
2274
    self._ErrorIf(test, self.ENODEHV, ninfo.name, "rpc call to node failed"
2275
                  " (instancelist): %s", utils.SafeEncode(str(idata)))
2276
    if test:
2277
      nimg.hyp_fail = True
2278
    else:
2279
      nimg.instances = idata
2280

    
2281
  def _UpdateNodeInfo(self, ninfo, nresult, nimg, vg_name):
2282
    """Verifies and computes a node information map
2283

2284
    @type ninfo: L{objects.Node}
2285
    @param ninfo: the node to check
2286
    @param nresult: the remote results for the node
2287
    @param nimg: the node image object
2288
    @param vg_name: the configured VG name
2289

2290
    """
2291
    node = ninfo.name
2292
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2293

    
2294
    # try to read free memory (from the hypervisor)
2295
    hv_info = nresult.get(constants.NV_HVINFO, None)
2296
    test = not isinstance(hv_info, dict) or "memory_free" not in hv_info
2297
    _ErrorIf(test, self.ENODEHV, node, "rpc call to node failed (hvinfo)")
2298
    if not test:
2299
      try:
2300
        nimg.mfree = int(hv_info["memory_free"])
2301
      except (ValueError, TypeError):
2302
        _ErrorIf(True, self.ENODERPC, node,
2303
                 "node returned invalid nodeinfo, check hypervisor")
2304

    
2305
    # FIXME: devise a free space model for file based instances as well
2306
    if vg_name is not None:
2307
      test = (constants.NV_VGLIST not in nresult or
2308
              vg_name not in nresult[constants.NV_VGLIST])
2309
      _ErrorIf(test, self.ENODELVM, node,
2310
               "node didn't return data for the volume group '%s'"
2311
               " - it is either missing or broken", vg_name)
2312
      if not test:
2313
        try:
2314
          nimg.dfree = int(nresult[constants.NV_VGLIST][vg_name])
2315
        except (ValueError, TypeError):
2316
          _ErrorIf(True, self.ENODERPC, node,
2317
                   "node returned invalid LVM info, check LVM status")
2318

    
2319
  def _CollectDiskInfo(self, nodelist, node_image, instanceinfo):
2320
    """Gets per-disk status information for all instances.
2321

2322
    @type nodelist: list of strings
2323
    @param nodelist: Node names
2324
    @type node_image: dict of (name, L{objects.Node})
2325
    @param node_image: Node objects
2326
    @type instanceinfo: dict of (name, L{objects.Instance})
2327
    @param instanceinfo: Instance objects
2328
    @rtype: {instance: {node: [(succes, payload)]}}
2329
    @return: a dictionary of per-instance dictionaries with nodes as
2330
        keys and disk information as values; the disk information is a
2331
        list of tuples (success, payload)
2332

2333
    """
2334
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2335

    
2336
    node_disks = {}
2337
    node_disks_devonly = {}
2338
    diskless_instances = set()
2339
    diskless = constants.DT_DISKLESS
2340

    
2341
    for nname in nodelist:
2342
      node_instances = list(itertools.chain(node_image[nname].pinst,
2343
                                            node_image[nname].sinst))
2344
      diskless_instances.update(inst for inst in node_instances
2345
                                if instanceinfo[inst].disk_template == diskless)
2346
      disks = [(inst, disk)
2347
               for inst in node_instances
2348
               for disk in instanceinfo[inst].disks]
2349

    
2350
      if not disks:
2351
        # No need to collect data
2352
        continue
2353

    
2354
      node_disks[nname] = disks
2355

    
2356
      # Creating copies as SetDiskID below will modify the objects and that can
2357
      # lead to incorrect data returned from nodes
2358
      devonly = [dev.Copy() for (_, dev) in disks]
2359

    
2360
      for dev in devonly:
2361
        self.cfg.SetDiskID(dev, nname)
2362

    
2363
      node_disks_devonly[nname] = devonly
2364

    
2365
    assert len(node_disks) == len(node_disks_devonly)
2366

    
2367
    # Collect data from all nodes with disks
2368
    result = self.rpc.call_blockdev_getmirrorstatus_multi(node_disks.keys(),
2369
                                                          node_disks_devonly)
2370

    
2371
    assert len(result) == len(node_disks)
2372

    
2373
    instdisk = {}
2374

    
2375
    for (nname, nres) in result.items():
2376
      disks = node_disks[nname]
2377

    
2378
      if nres.offline:
2379
        # No data from this node
2380
        data = len(disks) * [(False, "node offline")]
2381
      else:
2382
        msg = nres.fail_msg
2383
        _ErrorIf(msg, self.ENODERPC, nname,
2384
                 "while getting disk information: %s", msg)
2385
        if msg:
2386
          # No data from this node
2387
          data = len(disks) * [(False, msg)]
2388
        else:
2389
          data = []
2390
          for idx, i in enumerate(nres.payload):
2391
            if isinstance(i, (tuple, list)) and len(i) == 2:
2392
              data.append(i)
2393
            else:
2394
              logging.warning("Invalid result from node %s, entry %d: %s",
2395
                              nname, idx, i)
2396
              data.append((False, "Invalid result from the remote node"))
2397

    
2398
      for ((inst, _), status) in zip(disks, data):
2399
        instdisk.setdefault(inst, {}).setdefault(nname, []).append(status)
2400

    
2401
    # Add empty entries for diskless instances.
2402
    for inst in diskless_instances:
2403
      assert inst not in instdisk
2404
      instdisk[inst] = {}
2405

    
2406
    assert compat.all(len(statuses) == len(instanceinfo[inst].disks) and
2407
                      len(nnames) <= len(instanceinfo[inst].all_nodes) and
2408
                      compat.all(isinstance(s, (tuple, list)) and
2409
                                 len(s) == 2 for s in statuses)
2410
                      for inst, nnames in instdisk.items()
2411
                      for nname, statuses in nnames.items())
2412
    assert set(instdisk) == set(instanceinfo), "instdisk consistency failure"
2413

    
2414
    return instdisk
2415

    
2416
  def BuildHooksEnv(self):
2417
    """Build hooks env.
2418

2419
    Cluster-Verify hooks just ran in the post phase and their failure makes
2420
    the output be logged in the verify output and the verification to fail.
2421

2422
    """
2423
    env = {
2424
      "CLUSTER_TAGS": " ".join(self.cfg.GetClusterInfo().GetTags())
2425
      }
2426

    
2427
    env.update(("NODE_TAGS_%s" % node.name, " ".join(node.GetTags()))
2428
               for node in self.my_node_info.values())
2429

    
2430
    return env
2431

    
2432
  def BuildHooksNodes(self):
2433
    """Build hooks nodes.
2434

2435
    """
2436
    return ([], self.my_node_names)
2437

    
2438
  def Exec(self, feedback_fn):
2439
    """Verify integrity of the node group, performing various test on nodes.
2440

2441
    """
2442
    # This method has too many local variables. pylint: disable-msg=R0914
2443

    
2444
    if not self.my_node_names:
2445
      # empty node group
2446
      feedback_fn("* Empty node group, skipping verification")
2447
      return True
2448

    
2449
    self.bad = False
2450
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2451
    verbose = self.op.verbose
2452
    self._feedback_fn = feedback_fn
2453

    
2454
    vg_name = self.cfg.GetVGName()
2455
    drbd_helper = self.cfg.GetDRBDHelper()
2456
    cluster = self.cfg.GetClusterInfo()
2457
    groupinfo = self.cfg.GetAllNodeGroupsInfo()
2458
    hypervisors = cluster.enabled_hypervisors
2459
    node_data_list = [self.my_node_info[name] for name in self.my_node_names]
2460

    
2461
    i_non_redundant = [] # Non redundant instances
2462
    i_non_a_balanced = [] # Non auto-balanced instances
2463
    n_offline = 0 # Count of offline nodes
2464
    n_drained = 0 # Count of nodes being drained
2465
    node_vol_should = {}
2466

    
2467
    # FIXME: verify OS list
2468

    
2469
    # File verification
2470
    filemap = _ComputeAncillaryFiles(cluster, False)
2471

    
2472
    # do local checksums
2473
    master_node = self.master_node = self.cfg.GetMasterNode()
2474
    master_ip = self.cfg.GetMasterIP()
2475

    
2476
    feedback_fn("* Gathering data (%d nodes)" % len(self.my_node_names))
2477

    
2478
    # We will make nodes contact all nodes in their group, and one node from
2479
    # every other group.
2480
    # TODO: should it be a *random* node, different every time?
2481
    online_nodes = [node.name for node in node_data_list if not node.offline]
2482
    other_group_nodes = {}
2483

    
2484
    for name in sorted(self.all_node_info):
2485
      node = self.all_node_info[name]
2486
      if (node.group not in other_group_nodes
2487
          and node.group != self.group_uuid
2488
          and not node.offline):
2489
        other_group_nodes[node.group] = node.name
2490

    
2491
    node_verify_param = {
2492
      constants.NV_FILELIST:
2493
        utils.UniqueSequence(filename
2494
                             for files in filemap
2495
                             for filename in files),
2496
      constants.NV_NODELIST: online_nodes + other_group_nodes.values(),
2497
      constants.NV_HYPERVISOR: hypervisors,
2498
      constants.NV_HVPARAMS:
2499
        _GetAllHypervisorParameters(cluster, self.all_inst_info.values()),
2500
      constants.NV_NODENETTEST: [(node.name, node.primary_ip, node.secondary_ip)
2501
                                 for node in node_data_list
2502
                                 if not node.offline],
2503
      constants.NV_INSTANCELIST: hypervisors,
2504
      constants.NV_VERSION: None,
2505
      constants.NV_HVINFO: self.cfg.GetHypervisorType(),
2506
      constants.NV_NODESETUP: None,
2507
      constants.NV_TIME: None,
2508
      constants.NV_MASTERIP: (master_node, master_ip),
2509
      constants.NV_OSLIST: None,
2510
      constants.NV_VMNODES: self.cfg.GetNonVmCapableNodeList(),
2511
      }
2512

    
2513
    if vg_name is not None:
2514
      node_verify_param[constants.NV_VGLIST] = None
2515
      node_verify_param[constants.NV_LVLIST] = vg_name
2516
      node_verify_param[constants.NV_PVLIST] = [vg_name]
2517
      node_verify_param[constants.NV_DRBDLIST] = None
2518

    
2519
    if drbd_helper:
2520
      node_verify_param[constants.NV_DRBDHELPER] = drbd_helper
2521

    
2522
    # bridge checks
2523
    # FIXME: this needs to be changed per node-group, not cluster-wide
2524
    bridges = set()
2525
    default_nicpp = cluster.nicparams[constants.PP_DEFAULT]
2526
    if default_nicpp[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2527
      bridges.add(default_nicpp[constants.NIC_LINK])
2528
    for instance in self.my_inst_info.values():
2529
      for nic in instance.nics:
2530
        full_nic = cluster.SimpleFillNIC(nic.nicparams)
2531
        if full_nic[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2532
          bridges.add(full_nic[constants.NIC_LINK])
2533

    
2534
    if bridges:
2535
      node_verify_param[constants.NV_BRIDGES] = list(bridges)
2536

    
2537
    # Build our expected cluster state
2538
    node_image = dict((node.name, self.NodeImage(offline=node.offline,
2539
                                                 name=node.name,
2540
                                                 vm_capable=node.vm_capable))
2541
                      for node in node_data_list)
2542

    
2543
    # Gather OOB paths
2544
    oob_paths = []
2545
    for node in self.all_node_info.values():
2546
      path = _SupportsOob(self.cfg, node)
2547
      if path and path not in oob_paths:
2548
        oob_paths.append(path)
2549

    
2550
    if oob_paths:
2551
      node_verify_param[constants.NV_OOB_PATHS] = oob_paths
2552

    
2553
    for instance in self.my_inst_names:
2554
      inst_config = self.my_inst_info[instance]
2555

    
2556
      for nname in inst_config.all_nodes:
2557
        if nname not in node_image:
2558
          gnode = self.NodeImage(name=nname)
2559
          gnode.ghost = (nname not in self.all_node_info)
2560
          node_image[nname] = gnode
2561

    
2562
      inst_config.MapLVsByNode(node_vol_should)
2563

    
2564
      pnode = inst_config.primary_node
2565
      node_image[pnode].pinst.append(instance)
2566

    
2567
      for snode in inst_config.secondary_nodes:
2568
        nimg = node_image[snode]
2569
        nimg.sinst.append(instance)
2570
        if pnode not in nimg.sbp:
2571
          nimg.sbp[pnode] = []
2572
        nimg.sbp[pnode].append(instance)
2573

    
2574
    # At this point, we have the in-memory data structures complete,
2575
    # except for the runtime information, which we'll gather next
2576

    
2577
    # Due to the way our RPC system works, exact response times cannot be
2578
    # guaranteed (e.g. a broken node could run into a timeout). By keeping the
2579
    # time before and after executing the request, we can at least have a time
2580
    # window.
2581
    nvinfo_starttime = time.time()
2582
    all_nvinfo = self.rpc.call_node_verify(self.my_node_names,
2583
                                           node_verify_param,
2584
                                           self.cfg.GetClusterName())
2585
    nvinfo_endtime = time.time()
2586

    
2587
    if self.extra_lv_nodes and vg_name is not None:
2588
      extra_lv_nvinfo = \
2589
          self.rpc.call_node_verify(self.extra_lv_nodes,
2590
                                    {constants.NV_LVLIST: vg_name},
2591
                                    self.cfg.GetClusterName())
2592
    else:
2593
      extra_lv_nvinfo = {}
2594

    
2595
    all_drbd_map = self.cfg.ComputeDRBDMap()
2596

    
2597
    feedback_fn("* Gathering disk information (%s nodes)" %
2598
                len(self.my_node_names))
2599
    instdisk = self._CollectDiskInfo(self.my_node_names, node_image,
2600
                                     self.my_inst_info)
2601

    
2602
    feedback_fn("* Verifying configuration file consistency")
2603

    
2604
    # If not all nodes are being checked, we need to make sure the master node
2605
    # and a non-checked vm_capable node are in the list.
2606
    absent_nodes = set(self.all_node_info).difference(self.my_node_info)
2607
    if absent_nodes:
2608
      vf_nvinfo = all_nvinfo.copy()
2609
      vf_node_info = list(self.my_node_info.values())
2610
      additional_nodes = []
2611
      if master_node not in self.my_node_info:
2612
        additional_nodes.append(master_node)
2613
        vf_node_info.append(self.all_node_info[master_node])
2614
      # Add the first vm_capable node we find which is not included
2615
      for node in absent_nodes:
2616
        nodeinfo = self.all_node_info[node]
2617
        if nodeinfo.vm_capable and not nodeinfo.offline:
2618
          additional_nodes.append(node)
2619
          vf_node_info.append(self.all_node_info[node])
2620
          break
2621
      key = constants.NV_FILELIST
2622
      vf_nvinfo.update(self.rpc.call_node_verify(additional_nodes,
2623
                                                 {key: node_verify_param[key]},
2624
                                                 self.cfg.GetClusterName()))
2625
    else:
2626
      vf_nvinfo = all_nvinfo
2627
      vf_node_info = self.my_node_info.values()
2628

    
2629
    self._VerifyFiles(_ErrorIf, vf_node_info, master_node, vf_nvinfo, filemap)
2630

    
2631
    feedback_fn("* Verifying node status")
2632

    
2633
    refos_img = None
2634

    
2635
    for node_i in node_data_list:
2636
      node = node_i.name
2637
      nimg = node_image[node]
2638

    
2639
      if node_i.offline:
2640
        if verbose:
2641
          feedback_fn("* Skipping offline node %s" % (node,))
2642
        n_offline += 1
2643
        continue
2644

    
2645
      if node == master_node:
2646
        ntype = "master"
2647
      elif node_i.master_candidate:
2648
        ntype = "master candidate"
2649
      elif node_i.drained:
2650
        ntype = "drained"
2651
        n_drained += 1
2652
      else:
2653
        ntype = "regular"
2654
      if verbose:
2655
        feedback_fn("* Verifying node %s (%s)" % (node, ntype))
2656

    
2657
      msg = all_nvinfo[node].fail_msg
2658
      _ErrorIf(msg, self.ENODERPC, node, "while contacting node: %s", msg)
2659
      if msg:
2660
        nimg.rpc_fail = True
2661
        continue
2662

    
2663
      nresult = all_nvinfo[node].payload
2664

    
2665
      nimg.call_ok = self._VerifyNode(node_i, nresult)
2666
      self._VerifyNodeTime(node_i, nresult, nvinfo_starttime, nvinfo_endtime)
2667
      self._VerifyNodeNetwork(node_i, nresult)
2668
      self._VerifyOob(node_i, nresult)
2669

    
2670
      if nimg.vm_capable:
2671
        self._VerifyNodeLVM(node_i, nresult, vg_name)
2672
        self._VerifyNodeDrbd(node_i, nresult, self.all_inst_info, drbd_helper,
2673
                             all_drbd_map)
2674

    
2675
        self._UpdateNodeVolumes(node_i, nresult, nimg, vg_name)
2676
        self._UpdateNodeInstances(node_i, nresult, nimg)
2677
        self._UpdateNodeInfo(node_i, nresult, nimg, vg_name)
2678
        self._UpdateNodeOS(node_i, nresult, nimg)
2679

    
2680
        if not nimg.os_fail:
2681
          if refos_img is None:
2682
            refos_img = nimg
2683
          self._VerifyNodeOS(node_i, nimg, refos_img)
2684
        self._VerifyNodeBridges(node_i, nresult, bridges)
2685

    
2686
        # Check whether all running instancies are primary for the node. (This
2687
        # can no longer be done from _VerifyInstance below, since some of the
2688
        # wrong instances could be from other node groups.)
2689
        non_primary_inst = set(nimg.instances).difference(nimg.pinst)
2690

    
2691
        for inst in non_primary_inst:
2692
          test = inst in self.all_inst_info
2693
          _ErrorIf(test, self.EINSTANCEWRONGNODE, inst,
2694
                   "instance should not run on node %s", node_i.name)
2695
          _ErrorIf(not test, self.ENODEORPHANINSTANCE, node_i.name,
2696
                   "node is running unknown instance %s", inst)
2697

    
2698
    for node, result in extra_lv_nvinfo.items():
2699
      self._UpdateNodeVolumes(self.all_node_info[node], result.payload,
2700
                              node_image[node], vg_name)
2701

    
2702
    feedback_fn("* Verifying instance status")
2703
    for instance in self.my_inst_names:
2704
      if verbose:
2705
        feedback_fn("* Verifying instance %s" % instance)
2706
      inst_config = self.my_inst_info[instance]
2707
      self._VerifyInstance(instance, inst_config, node_image,
2708
                           instdisk[instance])
2709
      inst_nodes_offline = []
2710

    
2711
      pnode = inst_config.primary_node
2712
      pnode_img = node_image[pnode]
2713
      _ErrorIf(pnode_img.rpc_fail and not pnode_img.offline,
2714
               self.ENODERPC, pnode, "instance %s, connection to"
2715
               " primary node failed", instance)
2716

    
2717
      _ErrorIf(inst_config.admin_up and pnode_img.offline,
2718
               self.EINSTANCEBADNODE, instance,
2719
               "instance is marked as running and lives on offline node %s",
2720
               inst_config.primary_node)
2721

    
2722
      # If the instance is non-redundant we cannot survive losing its primary
2723
      # node, so we are not N+1 compliant. On the other hand we have no disk
2724
      # templates with more than one secondary so that situation is not well
2725
      # supported either.
2726
      # FIXME: does not support file-backed instances
2727
      if not inst_config.secondary_nodes:
2728
        i_non_redundant.append(instance)
2729

    
2730
      _ErrorIf(len(inst_config.secondary_nodes) > 1, self.EINSTANCELAYOUT,
2731
               instance, "instance has multiple secondary nodes: %s",
2732
               utils.CommaJoin(inst_config.secondary_nodes),
2733
               code=self.ETYPE_WARNING)
2734

    
2735
      if inst_config.disk_template in constants.DTS_INT_MIRROR:
2736
        pnode = inst_config.primary_node
2737
        instance_nodes = utils.NiceSort(inst_config.all_nodes)
2738
        instance_groups = {}
2739

    
2740
        for node in instance_nodes:
2741
          instance_groups.setdefault(self.all_node_info[node].group,
2742
                                     []).append(node)
2743

    
2744
        pretty_list = [
2745
          "%s (group %s)" % (utils.CommaJoin(nodes), groupinfo[group].name)
2746
          # Sort so that we always list the primary node first.
2747
          for group, nodes in sorted(instance_groups.items(),
2748
                                     key=lambda (_, nodes): pnode in nodes,
2749
                                     reverse=True)]
2750

    
2751
        self._ErrorIf(len(instance_groups) > 1, self.EINSTANCESPLITGROUPS,
2752
                      instance, "instance has primary and secondary nodes in"
2753
                      " different groups: %s", utils.CommaJoin(pretty_list),
2754
                      code=self.ETYPE_WARNING)
2755

    
2756
      if not cluster.FillBE(inst_config)[constants.BE_AUTO_BALANCE]:
2757
        i_non_a_balanced.append(instance)
2758

    
2759
      for snode in inst_config.secondary_nodes:
2760
        s_img = node_image[snode]
2761
        _ErrorIf(s_img.rpc_fail and not s_img.offline, self.ENODERPC, snode,
2762
                 "instance %s, connection to secondary node failed", instance)
2763

    
2764
        if s_img.offline:
2765
          inst_nodes_offline.append(snode)
2766

    
2767
      # warn that the instance lives on offline nodes
2768
      _ErrorIf(inst_nodes_offline, self.EINSTANCEBADNODE, instance,
2769
               "instance has offline secondary node(s) %s",
2770
               utils.CommaJoin(inst_nodes_offline))
2771
      # ... or ghost/non-vm_capable nodes
2772
      for node in inst_config.all_nodes:
2773
        _ErrorIf(node_image[node].ghost, self.EINSTANCEBADNODE, instance,
2774
                 "instance lives on ghost node %s", node)
2775
        _ErrorIf(not node_image[node].vm_capable, self.EINSTANCEBADNODE,
2776
                 instance, "instance lives on non-vm_capable node %s", node)
2777

    
2778
    feedback_fn("* Verifying orphan volumes")
2779
    reserved = utils.FieldSet(*cluster.reserved_lvs)
2780

    
2781
    # We will get spurious "unknown volume" warnings if any node of this group
2782
    # is secondary for an instance whose primary is in another group. To avoid
2783
    # them, we find these instances and add their volumes to node_vol_should.
2784
    for inst in self.all_inst_info.values():
2785
      for secondary in inst.secondary_nodes:
2786
        if (secondary in self.my_node_info
2787
            and inst.name not in self.my_inst_info):
2788
          inst.MapLVsByNode(node_vol_should)
2789
          break
2790

    
2791
    self._VerifyOrphanVolumes(node_vol_should, node_image, reserved)
2792

    
2793
    if constants.VERIFY_NPLUSONE_MEM not in self.op.skip_checks:
2794
      feedback_fn("* Verifying N+1 Memory redundancy")
2795
      self._VerifyNPlusOneMemory(node_image, self.my_inst_info)
2796

    
2797
    feedback_fn("* Other Notes")
2798
    if i_non_redundant:
2799
      feedback_fn("  - NOTICE: %d non-redundant instance(s) found."
2800
                  % len(i_non_redundant))
2801

    
2802
    if i_non_a_balanced:
2803
      feedback_fn("  - NOTICE: %d non-auto-balanced instance(s) found."
2804
                  % len(i_non_a_balanced))
2805

    
2806
    if n_offline:
2807
      feedback_fn("  - NOTICE: %d offline node(s) found." % n_offline)
2808

    
2809
    if n_drained:
2810
      feedback_fn("  - NOTICE: %d drained node(s) found." % n_drained)
2811

    
2812
    return not self.bad
2813

    
2814
  def HooksCallBack(self, phase, hooks_results, feedback_fn, lu_result):
2815
    """Analyze the post-hooks' result
2816

2817
    This method analyses the hook result, handles it, and sends some
2818
    nicely-formatted feedback back to the user.
2819

2820
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
2821
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
2822
    @param hooks_results: the results of the multi-node hooks rpc call
2823
    @param feedback_fn: function used send feedback back to the caller
2824
    @param lu_result: previous Exec result
2825
    @return: the new Exec result, based on the previous result
2826
        and hook results
2827

2828
    """
2829
    # We only really run POST phase hooks, only for non-empty groups,
2830
    # and are only interested in their results
2831
    if not self.my_node_names:
2832
      # empty node group
2833
      pass
2834
    elif phase == constants.HOOKS_PHASE_POST:
2835
      # Used to change hooks' output to proper indentation
2836
      feedback_fn("* Hooks Results")
2837
      assert hooks_results, "invalid result from hooks"
2838

    
2839
      for node_name in hooks_results:
2840
        res = hooks_results[node_name]
2841
        msg = res.fail_msg
2842
        test = msg and not res.offline
2843
        self._ErrorIf(test, self.ENODEHOOKS, node_name,
2844
                      "Communication failure in hooks execution: %s", msg)
2845
        if res.offline or msg:
2846
          # No need to investigate payload if node is offline or gave an error.
2847
          # override manually lu_result here as _ErrorIf only
2848
          # overrides self.bad
2849
          lu_result = 1
2850
          continue
2851
        for script, hkr, output in res.payload:
2852
          test = hkr == constants.HKR_FAIL
2853
          self._ErrorIf(test, self.ENODEHOOKS, node_name,
2854
                        "Script %s failed, output:", script)
2855
          if test:
2856
            output = self._HOOKS_INDENT_RE.sub("      ", output)
2857
            feedback_fn("%s" % output)
2858
            lu_result = 0
2859

    
2860
    return lu_result
2861

    
2862

    
2863
class LUClusterVerifyDisks(NoHooksLU):
2864
  """Verifies the cluster disks status.
2865

2866
  """
2867
  REQ_BGL = False
2868

    
2869
  def ExpandNames(self):
2870
    self.needed_locks = {
2871
      locking.LEVEL_NODE: locking.ALL_SET,
2872
      locking.LEVEL_INSTANCE: locking.ALL_SET,
2873
    }
2874
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2875

    
2876
  def Exec(self, feedback_fn):
2877
    """Verify integrity of cluster disks.
2878

2879
    @rtype: tuple of three items
2880
    @return: a tuple of (dict of node-to-node_error, list of instances
2881
        which need activate-disks, dict of instance: (node, volume) for
2882
        missing volumes
2883

2884
    """
2885
    result = res_nodes, res_instances, res_missing = {}, [], {}
2886

    
2887
    nodes = utils.NiceSort(self.cfg.GetVmCapableNodeList())
2888
    instances = self.cfg.GetAllInstancesInfo().values()
2889

    
2890
    nv_dict = {}
2891
    for inst in instances:
2892
      inst_lvs = {}
2893
      if not inst.admin_up:
2894
        continue
2895
      inst.MapLVsByNode(inst_lvs)
2896
      # transform { iname: {node: [vol,],},} to {(node, vol): iname}
2897
      for node, vol_list in inst_lvs.iteritems():
2898
        for vol in vol_list:
2899
          nv_dict[(node, vol)] = inst
2900

    
2901
    if not nv_dict:
2902
      return result
2903

    
2904
    node_lvs = self.rpc.call_lv_list(nodes, [])
2905
    for node, node_res in node_lvs.items():
2906
      if node_res.offline:
2907
        continue
2908
      msg = node_res.fail_msg
2909
      if msg:
2910
        logging.warning("Error enumerating LVs on node %s: %s", node, msg)
2911
        res_nodes[node] = msg
2912
        continue
2913

    
2914
      lvs = node_res.payload
2915
      for lv_name, (_, _, lv_online) in lvs.items():
2916
        inst = nv_dict.pop((node, lv_name), None)
2917
        if (not lv_online and inst is not None
2918
            and inst.name not in res_instances):
2919
          res_instances.append(inst.name)
2920

    
2921
    # any leftover items in nv_dict are missing LVs, let's arrange the
2922
    # data better
2923
    for key, inst in nv_dict.iteritems():
2924
      if inst.name not in res_missing:
2925
        res_missing[inst.name] = []
2926
      res_missing[inst.name].append(key)
2927

    
2928
    return result
2929

    
2930

    
2931
class LUClusterRepairDiskSizes(NoHooksLU):
2932
  """Verifies the cluster disks sizes.
2933

2934
  """
2935
  REQ_BGL = False
2936

    
2937
  def ExpandNames(self):
2938
    if self.op.instances:
2939
      self.wanted_names = _GetWantedInstances(self, self.op.instances)
2940
      self.needed_locks = {
2941
        locking.LEVEL_NODE: [],
2942
        locking.LEVEL_INSTANCE: self.wanted_names,
2943
        }
2944
      self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE
2945
    else:
2946
      self.wanted_names = None
2947
      self.needed_locks = {
2948
        locking.LEVEL_NODE: locking.ALL_SET,
2949
        locking.LEVEL_INSTANCE: locking.ALL_SET,
2950
        }
2951
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2952

    
2953
  def DeclareLocks(self, level):
2954
    if level == locking.LEVEL_NODE and self.wanted_names is not None:
2955
      self._LockInstancesNodes(primary_only=True)
2956

    
2957
  def CheckPrereq(self):
2958
    """Check prerequisites.
2959

2960
    This only checks the optional instance list against the existing names.
2961

2962
    """
2963
    if self.wanted_names is None:
2964
      self.wanted_names = self.glm.list_owned(locking.LEVEL_INSTANCE)
2965

    
2966
    self.wanted_instances = [self.cfg.GetInstanceInfo(name) for name
2967
                             in self.wanted_names]
2968

    
2969
  def _EnsureChildSizes(self, disk):
2970
    """Ensure children of the disk have the needed disk size.
2971

2972
    This is valid mainly for DRBD8 and fixes an issue where the
2973
    children have smaller disk size.
2974

2975
    @param disk: an L{ganeti.objects.Disk} object
2976

2977
    """
2978
    if disk.dev_type == constants.LD_DRBD8:
2979
      assert disk.children, "Empty children for DRBD8?"
2980
      fchild = disk.children[0]
2981
      mismatch = fchild.size < disk.size
2982
      if mismatch:
2983
        self.LogInfo("Child disk has size %d, parent %d, fixing",
2984
                     fchild.size, disk.size)
2985
        fchild.size = disk.size
2986

    
2987
      # and we recurse on this child only, not on the metadev
2988
      return self._EnsureChildSizes(fchild) or mismatch
2989
    else:
2990
      return False
2991

    
2992
  def Exec(self, feedback_fn):
2993
    """Verify the size of cluster disks.
2994

2995
    """
2996
    # TODO: check child disks too
2997
    # TODO: check differences in size between primary/secondary nodes
2998
    per_node_disks = {}
2999
    for instance in self.wanted_instances:
3000
      pnode = instance.primary_node
3001
      if pnode not in per_node_disks:
3002
        per_node_disks[pnode] = []
3003
      for idx, disk in enumerate(instance.disks):
3004
        per_node_disks[pnode].append((instance, idx, disk))
3005

    
3006
    changed = []
3007
    for node, dskl in per_node_disks.items():
3008
      newl = [v[2].Copy() for v in dskl]
3009
      for dsk in newl:
3010
        self.cfg.SetDiskID(dsk, node)
3011
      result = self.rpc.call_blockdev_getsize(node, newl)
3012
      if result.fail_msg:
3013
        self.LogWarning("Failure in blockdev_getsize call to node"
3014
                        " %s, ignoring", node)
3015
        continue
3016
      if len(result.payload) != len(dskl):
3017
        logging.warning("Invalid result from node %s: len(dksl)=%d,"
3018
                        " result.payload=%s", node, len(dskl), result.payload)
3019
        self.LogWarning("Invalid result from node %s, ignoring node results",
3020
                        node)
3021
        continue
3022
      for ((instance, idx, disk), size) in zip(dskl, result.payload):
3023
        if size is None:
3024
          self.LogWarning("Disk %d of instance %s did not return size"
3025
                          " information, ignoring", idx, instance.name)
3026
          continue
3027
        if not isinstance(size, (int, long)):
3028
          self.LogWarning("Disk %d of instance %s did not return valid"
3029
                          " size information, ignoring", idx, instance.name)
3030
          continue
3031
        size = size >> 20
3032
        if size != disk.size:
3033
          self.LogInfo("Disk %d of instance %s has mismatched size,"
3034
                       " correcting: recorded %d, actual %d", idx,
3035
                       instance.name, disk.size, size)
3036
          disk.size = size
3037
          self.cfg.Update(instance, feedback_fn)
3038
          changed.append((instance.name, idx, size))
3039
        if self._EnsureChildSizes(disk):
3040
          self.cfg.Update(instance, feedback_fn)
3041
          changed.append((instance.name, idx, disk.size))
3042
    return changed
3043

    
3044

    
3045
class LUClusterRename(LogicalUnit):
3046
  """Rename the cluster.
3047

3048
  """
3049
  HPATH = "cluster-rename"
3050
  HTYPE = constants.HTYPE_CLUSTER
3051

    
3052
  def BuildHooksEnv(self):
3053
    """Build hooks env.
3054

3055
    """
3056
    return {
3057
      "OP_TARGET": self.cfg.GetClusterName(),
3058
      "NEW_NAME": self.op.name,
3059
      }
3060

    
3061
  def BuildHooksNodes(self):
3062
    """Build hooks nodes.
3063

3064
    """
3065
    return ([self.cfg.GetMasterNode()], self.cfg.GetNodeList())
3066

    
3067
  def CheckPrereq(self):
3068
    """Verify that the passed name is a valid one.
3069

3070
    """
3071
    hostname = netutils.GetHostname(name=self.op.name,
3072
                                    family=self.cfg.GetPrimaryIPFamily())
3073

    
3074
    new_name = hostname.name
3075
    self.ip = new_ip = hostname.ip
3076
    old_name = self.cfg.GetClusterName()
3077
    old_ip = self.cfg.GetMasterIP()
3078
    if new_name == old_name and new_ip == old_ip:
3079
      raise errors.OpPrereqError("Neither the name nor the IP address of the"
3080
                                 " cluster has changed",
3081
                                 errors.ECODE_INVAL)
3082
    if new_ip != old_ip:
3083
      if netutils.TcpPing(new_ip, constants.DEFAULT_NODED_PORT):
3084
        raise errors.OpPrereqError("The given cluster IP address (%s) is"
3085
                                   " reachable on the network" %
3086
                                   new_ip, errors.ECODE_NOTUNIQUE)
3087

    
3088
    self.op.name = new_name
3089

    
3090
  def Exec(self, feedback_fn):
3091
    """Rename the cluster.
3092

3093
    """
3094
    clustername = self.op.name
3095
    ip = self.ip
3096

    
3097
    # shutdown the master IP
3098
    master = self.cfg.GetMasterNode()
3099
    result = self.rpc.call_node_stop_master(master, False)
3100
    result.Raise("Could not disable the master role")
3101

    
3102
    try:
3103
      cluster = self.cfg.GetClusterInfo()
3104
      cluster.cluster_name = clustername
3105
      cluster.master_ip = ip
3106
      self.cfg.Update(cluster, feedback_fn)
3107

    
3108
      # update the known hosts file
3109
      ssh.WriteKnownHostsFile(self.cfg, constants.SSH_KNOWN_HOSTS_FILE)
3110
      node_list = self.cfg.GetOnlineNodeList()
3111
      try:
3112
        node_list.remove(master)
3113
      except ValueError:
3114
        pass
3115
      _UploadHelper(self, node_list, constants.SSH_KNOWN_HOSTS_FILE)
3116
    finally:
3117
      result = self.rpc.call_node_start_master(master, False, False)
3118
      msg = result.fail_msg
3119
      if msg:
3120
        self.LogWarning("Could not re-enable the master role on"
3121
                        " the master, please restart manually: %s", msg)
3122

    
3123
    return clustername
3124

    
3125

    
3126
class LUClusterSetParams(LogicalUnit):
3127
  """Change the parameters of the cluster.
3128

3129
  """
3130
  HPATH = "cluster-modify"
3131
  HTYPE = constants.HTYPE_CLUSTER
3132
  REQ_BGL = False
3133

    
3134
  def CheckArguments(self):
3135
    """Check parameters
3136

3137
    """
3138
    if self.op.uid_pool:
3139
      uidpool.CheckUidPool(self.op.uid_pool)
3140

    
3141
    if self.op.add_uids:
3142
      uidpool.CheckUidPool(self.op.add_uids)
3143

    
3144
    if self.op.remove_uids:
3145
      uidpool.CheckUidPool(self.op.remove_uids)
3146

    
3147
  def ExpandNames(self):
3148
    # FIXME: in the future maybe other cluster params won't require checking on
3149
    # all nodes to be modified.
3150
    self.needed_locks = {
3151
      locking.LEVEL_NODE: locking.ALL_SET,
3152
    }
3153
    self.share_locks[locking.LEVEL_NODE] = 1
3154

    
3155
  def BuildHooksEnv(self):
3156
    """Build hooks env.
3157

3158
    """
3159
    return {
3160
      "OP_TARGET": self.cfg.GetClusterName(),
3161
      "NEW_VG_NAME": self.op.vg_name,
3162
      }
3163

    
3164
  def BuildHooksNodes(self):
3165
    """Build hooks nodes.
3166

3167
    """
3168
    mn = self.cfg.GetMasterNode()
3169
    return ([mn], [mn])
3170

    
3171
  def CheckPrereq(self):
3172
    """Check prerequisites.
3173

3174
    This checks whether the given params don't conflict and
3175
    if the given volume group is valid.
3176

3177
    """
3178
    if self.op.vg_name is not None and not self.op.vg_name:
3179
      if self.cfg.HasAnyDiskOfType(constants.LD_LV):
3180
        raise errors.OpPrereqError("Cannot disable lvm storage while lvm-based"
3181
                                   " instances exist", errors.ECODE_INVAL)
3182

    
3183
    if self.op.drbd_helper is not None and not self.op.drbd_helper:
3184
      if self.cfg.HasAnyDiskOfType(constants.LD_DRBD8):
3185
        raise errors.OpPrereqError("Cannot disable drbd helper while"
3186
                                   " drbd-based instances exist",
3187
                                   errors.ECODE_INVAL)
3188

    
3189
    node_list = self.glm.list_owned(locking.LEVEL_NODE)
3190

    
3191
    # if vg_name not None, checks given volume group on all nodes
3192
    if self.op.vg_name:
3193
      vglist = self.rpc.call_vg_list(node_list)
3194
      for node in node_list:
3195
        msg = vglist[node].fail_msg
3196
        if msg:
3197
          # ignoring down node
3198
          self.LogWarning("Error while gathering data on node %s"
3199
                          " (ignoring node): %s", node, msg)
3200
          continue
3201
        vgstatus = utils.CheckVolumeGroupSize(vglist[node].payload,
3202
                                              self.op.vg_name,
3203
                                              constants.MIN_VG_SIZE)
3204
        if vgstatus:
3205
          raise errors.OpPrereqError("Error on node '%s': %s" %
3206
                                     (node, vgstatus), errors.ECODE_ENVIRON)
3207

    
3208
    if self.op.drbd_helper:
3209
      # checks given drbd helper on all nodes
3210
      helpers = self.rpc.call_drbd_helper(node_list)
3211
      for node in node_list:
3212
        ninfo = self.cfg.GetNodeInfo(node)
3213
        if ninfo.offline:
3214
          self.LogInfo("Not checking drbd helper on offline node %s", node)
3215
          continue
3216
        msg = helpers[node].fail_msg
3217
        if msg:
3218
          raise errors.OpPrereqError("Error checking drbd helper on node"
3219
                                     " '%s': %s" % (node, msg),
3220
                                     errors.ECODE_ENVIRON)
3221
        node_helper = helpers[node].payload
3222
        if node_helper != self.op.drbd_helper:
3223
          raise errors.OpPrereqError("Error on node '%s': drbd helper is %s" %
3224
                                     (node, node_helper), errors.ECODE_ENVIRON)
3225

    
3226
    self.cluster = cluster = self.cfg.GetClusterInfo()
3227
    # validate params changes
3228
    if self.op.beparams:
3229
      utils.ForceDictType(self.op.beparams, constants.BES_PARAMETER_TYPES)
3230
      self.new_beparams = cluster.SimpleFillBE(self.op.beparams)
3231

    
3232
    if self.op.ndparams:
3233
      utils.ForceDictType(self.op.ndparams, constants.NDS_PARAMETER_TYPES)
3234
      self.new_ndparams = cluster.SimpleFillND(self.op.ndparams)
3235

    
3236
      # TODO: we need a more general way to handle resetting
3237
      # cluster-level parameters to default values
3238
      if self.new_ndparams["oob_program"] == "":
3239
        self.new_ndparams["oob_program"] = \
3240
            constants.NDC_DEFAULTS[constants.ND_OOB_PROGRAM]
3241

    
3242
    if self.op.nicparams:
3243
      utils.ForceDictType(self.op.nicparams, constants.NICS_PARAMETER_TYPES)
3244
      self.new_nicparams = cluster.SimpleFillNIC(self.op.nicparams)
3245
      objects.NIC.CheckParameterSyntax(self.new_nicparams)
3246
      nic_errors = []
3247

    
3248
      # check all instances for consistency
3249
      for instance in self.cfg.GetAllInstancesInfo().values():
3250
        for nic_idx, nic in enumerate(instance.nics):
3251
          params_copy = copy.deepcopy(nic.nicparams)
3252
          params_filled = objects.FillDict(self.new_nicparams, params_copy)
3253

    
3254
          # check parameter syntax
3255
          try:
3256
            objects.NIC.CheckParameterSyntax(params_filled)
3257
          except errors.ConfigurationError, err:
3258
            nic_errors.append("Instance %s, nic/%d: %s" %
3259
                              (instance.name, nic_idx, err))
3260

    
3261
          # if we're moving instances to routed, check that they have an ip
3262
          target_mode = params_filled[constants.NIC_MODE]
3263
          if target_mode == constants.NIC_MODE_ROUTED and not nic.ip:
3264
            nic_errors.append("Instance %s, nic/%d: routed NIC with no ip"
3265
                              " address" % (instance.name, nic_idx))
3266
      if nic_errors:
3267
        raise errors.OpPrereqError("Cannot apply the change, errors:\n%s" %
3268
                                   "\n".join(nic_errors))
3269

    
3270
    # hypervisor list/parameters
3271
    self.new_hvparams = new_hvp = objects.FillDict(cluster.hvparams, {})
3272
    if self.op.hvparams:
3273
      for hv_name, hv_dict in self.op.hvparams.items():
3274
        if hv_name not in self.new_hvparams:
3275
          self.new_hvparams[hv_name] = hv_dict
3276
        else:
3277
          self.new_hvparams[hv_name].update(hv_dict)
3278

    
3279
    # os hypervisor parameters
3280
    self.new_os_hvp = objects.FillDict(cluster.os_hvp, {})
3281
    if self.op.os_hvp:
3282
      for os_name, hvs in self.op.os_hvp.items():
3283
        if os_name not in self.new_os_hvp:
3284
          self.new_os_hvp[os_name] = hvs
3285
        else:
3286
          for hv_name, hv_dict in hvs.items():
3287
            if hv_name not in self.new_os_hvp[os_name]:
3288
              self.new_os_hvp[os_name][hv_name] = hv_dict
3289
            else:
3290
              self.new_os_hvp[os_name][hv_name].update(hv_dict)
3291

    
3292
    # os parameters
3293
    self.new_osp = objects.FillDict(cluster.osparams, {})
3294
    if self.op.osparams:
3295
      for os_name, osp in self.op.osparams.items():
3296
        if os_name not in self.new_osp:
3297
          self.new_osp[os_name] = {}
3298

    
3299
        self.new_osp[os_name] = _GetUpdatedParams(self.new_osp[os_name], osp,
3300
                                                  use_none=True)
3301

    
3302
        if not self.new_osp[os_name]:
3303
          # we removed all parameters
3304
          del self.new_osp[os_name]
3305
        else:
3306
          # check the parameter validity (remote check)
3307
          _CheckOSParams(self, False, [self.cfg.GetMasterNode()],
3308
                         os_name, self.new_osp[os_name])
3309

    
3310
    # changes to the hypervisor list
3311
    if self.op.enabled_hypervisors is not None:
3312
      self.hv_list = self.op.enabled_hypervisors
3313
      for hv in self.hv_list:
3314
        # if the hypervisor doesn't already exist in the cluster
3315
        # hvparams, we initialize it to empty, and then (in both
3316
        # cases) we make sure to fill the defaults, as we might not
3317
        # have a complete defaults list if the hypervisor wasn't
3318
        # enabled before
3319
        if hv not in new_hvp:
3320
          new_hvp[hv] = {}
3321
        new_hvp[hv] = objects.FillDict(constants.HVC_DEFAULTS[hv], new_hvp[hv])
3322
        utils.ForceDictType(new_hvp[hv], constants.HVS_PARAMETER_TYPES)
3323
    else:
3324
      self.hv_list = cluster.enabled_hypervisors
3325

    
3326
    if self.op.hvparams or self.op.enabled_hypervisors is not None:
3327
      # either the enabled list has changed, or the parameters have, validate
3328
      for hv_name, hv_params in self.new_hvparams.items():
3329
        if ((self.op.hvparams and hv_name in self.op.hvparams) or
3330
            (self.op.enabled_hypervisors and
3331
             hv_name in self.op.enabled_hypervisors)):
3332
          # either this is a new hypervisor, or its parameters have changed
3333
          hv_class = hypervisor.GetHypervisor(hv_name)
3334
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3335
          hv_class.CheckParameterSyntax(hv_params)
3336
          _CheckHVParams(self, node_list, hv_name, hv_params)
3337

    
3338
    if self.op.os_hvp:
3339
      # no need to check any newly-enabled hypervisors, since the
3340
      # defaults have already been checked in the above code-block
3341
      for os_name, os_hvp in self.new_os_hvp.items():
3342
        for hv_name, hv_params in os_hvp.items():
3343
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3344
          # we need to fill in the new os_hvp on top of the actual hv_p
3345
          cluster_defaults = self.new_hvparams.get(hv_name, {})
3346
          new_osp = objects.FillDict(cluster_defaults, hv_params)
3347
          hv_class = hypervisor.GetHypervisor(hv_name)
3348
          hv_class.CheckParameterSyntax(new_osp)
3349
          _CheckHVParams(self, node_list, hv_name, new_osp)
3350

    
3351
    if self.op.default_iallocator:
3352
      alloc_script = utils.FindFile(self.op.default_iallocator,
3353
                                    constants.IALLOCATOR_SEARCH_PATH,
3354
                                    os.path.isfile)
3355
      if alloc_script is None:
3356
        raise errors.OpPrereqError("Invalid default iallocator script '%s'"
3357
                                   " specified" % self.op.default_iallocator,
3358
                                   errors.ECODE_INVAL)
3359

    
3360
  def Exec(self, feedback_fn):
3361
    """Change the parameters of the cluster.
3362

3363
    """
3364
    if self.op.vg_name is not None:
3365
      new_volume = self.op.vg_name
3366
      if not new_volume:
3367
        new_volume = None
3368
      if new_volume != self.cfg.GetVGName():
3369
        self.cfg.SetVGName(new_volume)
3370
      else:
3371
        feedback_fn("Cluster LVM configuration already in desired"
3372
                    " state, not changing")
3373
    if self.op.drbd_helper is not None:
3374
      new_helper = self.op.drbd_helper
3375
      if not new_helper:
3376
        new_helper = None
3377
      if new_helper != self.cfg.GetDRBDHelper():
3378
        self.cfg.SetDRBDHelper(new_helper)
3379
      else:
3380
        feedback_fn("Cluster DRBD helper already in desired state,"
3381
                    " not changing")
3382
    if self.op.hvparams:
3383
      self.cluster.hvparams = self.new_hvparams
3384
    if self.op.os_hvp:
3385
      self.cluster.os_hvp = self.new_os_hvp
3386
    if self.op.enabled_hypervisors is not None:
3387
      self.cluster.hvparams = self.new_hvparams
3388
      self.cluster.enabled_hypervisors = self.op.enabled_hypervisors
3389
    if self.op.beparams:
3390
      self.cluster.beparams[constants.PP_DEFAULT] = self.new_beparams
3391
    if self.op.nicparams:
3392
      self.cluster.nicparams[constants.PP_DEFAULT] = self.new_nicparams
3393
    if self.op.osparams:
3394
      self.cluster.osparams = self.new_osp
3395
    if self.op.ndparams:
3396
      self.cluster.ndparams = self.new_ndparams
3397

    
3398
    if self.op.candidate_pool_size is not None:
3399
      self.cluster.candidate_pool_size = self.op.candidate_pool_size
3400
      # we need to update the pool size here, otherwise the save will fail
3401
      _AdjustCandidatePool(self, [])
3402

    
3403
    if self.op.maintain_node_health is not None:
3404
      self.cluster.maintain_node_health = self.op.maintain_node_health
3405

    
3406
    if self.op.prealloc_wipe_disks is not None:
3407
      self.cluster.prealloc_wipe_disks = self.op.prealloc_wipe_disks
3408

    
3409
    if self.op.add_uids is not None:
3410
      uidpool.AddToUidPool(self.cluster.uid_pool, self.op.add_uids)
3411

    
3412
    if self.op.remove_uids is not None:
3413
      uidpool.RemoveFromUidPool(self.cluster.uid_pool, self.op.remove_uids)
3414

    
3415
    if self.op.uid_pool is not None:
3416
      self.cluster.uid_pool = self.op.uid_pool
3417

    
3418
    if self.op.default_iallocator is not None:
3419
      self.cluster.default_iallocator = self.op.default_iallocator
3420

    
3421
    if self.op.reserved_lvs is not None:
3422
      self.cluster.reserved_lvs = self.op.reserved_lvs
3423

    
3424
    def helper_os(aname, mods, desc):
3425
      desc += " OS list"
3426
      lst = getattr(self.cluster, aname)
3427
      for key, val in mods:
3428
        if key == constants.DDM_ADD:
3429
          if val in lst:
3430
            feedback_fn("OS %s already in %s, ignoring" % (val, desc))
3431
          else:
3432
            lst.append(val)
3433
        elif key == constants.DDM_REMOVE:
3434
          if val in lst:
3435
            lst.remove(val)
3436
          else:
3437
            feedback_fn("OS %s not found in %s, ignoring" % (val, desc))
3438
        else:
3439
          raise errors.ProgrammerError("Invalid modification '%s'" % key)
3440

    
3441
    if self.op.hidden_os:
3442
      helper_os("hidden_os", self.op.hidden_os, "hidden")
3443

    
3444
    if self.op.blacklisted_os:
3445
      helper_os("blacklisted_os", self.op.blacklisted_os, "blacklisted")
3446

    
3447
    if self.op.master_netdev:
3448
      master = self.cfg.GetMasterNode()
3449
      feedback_fn("Shutting down master ip on the current netdev (%s)" %
3450
                  self.cluster.master_netdev)
3451
      result = self.rpc.call_node_stop_master(master, False)
3452
      result.Raise("Could not disable the master ip")
3453
      feedback_fn("Changing master_netdev from %s to %s" %
3454
                  (self.cluster.master_netdev, self.op.master_netdev))
3455
      self.cluster.master_netdev = self.op.master_netdev
3456

    
3457
    self.cfg.Update(self.cluster, feedback_fn)
3458

    
3459
    if self.op.master_netdev:
3460
      feedback_fn("Starting the master ip on the new master netdev (%s)" %
3461
                  self.op.master_netdev)
3462
      result = self.rpc.call_node_start_master(master, False, False)
3463
      if result.fail_msg:
3464
        self.LogWarning("Could not re-enable the master ip on"
3465
                        " the master, please restart manually: %s",
3466
                        result.fail_msg)
3467

    
3468

    
3469
def _UploadHelper(lu, nodes, fname):
3470
  """Helper for uploading a file and showing warnings.
3471

3472
  """
3473
  if os.path.exists(fname):
3474
    result = lu.rpc.call_upload_file(nodes, fname)
3475
    for to_node, to_result in result.items():
3476
      msg = to_result.fail_msg
3477
      if msg:
3478
        msg = ("Copy of file %s to node %s failed: %s" %
3479
               (fname, to_node, msg))
3480
        lu.proc.LogWarning(msg)
3481

    
3482

    
3483
def _ComputeAncillaryFiles(cluster, redist):
3484
  """Compute files external to Ganeti which need to be consistent.
3485

3486
  @type redist: boolean
3487
  @param redist: Whether to include files which need to be redistributed
3488

3489
  """
3490
  # Compute files for all nodes
3491
  files_all = set([
3492
    constants.SSH_KNOWN_HOSTS_FILE,
3493
    constants.CONFD_HMAC_KEY,
3494
    constants.CLUSTER_DOMAIN_SECRET_FILE,
3495
    ])
3496

    
3497
  if not redist:
3498
    files_all.update(constants.ALL_CERT_FILES)
3499
    files_all.update(ssconf.SimpleStore().GetFileList())
3500

    
3501
  if cluster.modify_etc_hosts:
3502
    files_all.add(constants.ETC_HOSTS)
3503

    
3504
  # Files which must either exist on all nodes or on none
3505
  files_all_opt = set([
3506
    constants.RAPI_USERS_FILE,
3507
    ])
3508

    
3509
  # Files which should only be on master candidates
3510
  files_mc = set()
3511
  if not redist:
3512
    files_mc.add(constants.CLUSTER_CONF_FILE)
3513

    
3514
  # Files which should only be on VM-capable nodes
3515
  files_vm = set(filename
3516
    for hv_name in cluster.enabled_hypervisors
3517
    for filename in hypervisor.GetHypervisor(hv_name).GetAncillaryFiles())
3518

    
3519
  # Filenames must be unique
3520
  assert (len(files_all | files_all_opt | files_mc | files_vm) ==
3521
          sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
3522
         "Found file listed in more than one file list"
3523

    
3524
  return (files_all, files_all_opt, files_mc, files_vm)
3525

    
3526

    
3527
def _RedistributeAncillaryFiles(lu, additional_nodes=None, additional_vm=True):
3528
  """Distribute additional files which are part of the cluster configuration.
3529

3530
  ConfigWriter takes care of distributing the config and ssconf files, but
3531
  there are more files which should be distributed to all nodes. This function
3532
  makes sure those are copied.
3533

3534
  @param lu: calling logical unit
3535
  @param additional_nodes: list of nodes not in the config to distribute to
3536
  @type additional_vm: boolean
3537
  @param additional_vm: whether the additional nodes are vm-capable or not
3538

3539
  """
3540
  # Gather target nodes
3541
  cluster = lu.cfg.GetClusterInfo()
3542
  master_info = lu.cfg.GetNodeInfo(lu.cfg.GetMasterNode())
3543

    
3544
  online_nodes = lu.cfg.GetOnlineNodeList()
3545
  vm_nodes = lu.cfg.GetVmCapableNodeList()
3546

    
3547
  if additional_nodes is not None:
3548
    online_nodes.extend(additional_nodes)
3549
    if additional_vm:
3550
      vm_nodes.extend(additional_nodes)
3551

    
3552
  # Never distribute to master node
3553
  for nodelist in [online_nodes, vm_nodes]:
3554
    if master_info.name in nodelist:
3555
      nodelist.remove(master_info.name)
3556

    
3557
  # Gather file lists
3558
  (files_all, files_all_opt, files_mc, files_vm) = \
3559
    _ComputeAncillaryFiles(cluster, True)
3560

    
3561
  # Never re-distribute configuration file from here
3562
  assert not (constants.CLUSTER_CONF_FILE in files_all or
3563
              constants.CLUSTER_CONF_FILE in files_vm)
3564
  assert not files_mc, "Master candidates not handled in this function"
3565

    
3566
  filemap = [
3567
    (online_nodes, files_all),
3568
    (online_nodes, files_all_opt),
3569
    (vm_nodes, files_vm),
3570
    ]
3571

    
3572
  # Upload the files
3573
  for (node_list, files) in filemap:
3574
    for fname in files:
3575
      _UploadHelper(lu, node_list, fname)
3576

    
3577

    
3578
class LUClusterRedistConf(NoHooksLU):
3579
  """Force the redistribution of cluster configuration.
3580

3581
  This is a very simple LU.
3582

3583
  """
3584
  REQ_BGL = False
3585

    
3586
  def ExpandNames(self):
3587
    self.needed_locks = {
3588
      locking.LEVEL_NODE: locking.ALL_SET,
3589
    }
3590
    self.share_locks[locking.LEVEL_NODE] = 1
3591

    
3592
  def Exec(self, feedback_fn):
3593
    """Redistribute the configuration.
3594

3595
    """
3596
    self.cfg.Update(self.cfg.GetClusterInfo(), feedback_fn)
3597
    _RedistributeAncillaryFiles(self)
3598

    
3599

    
3600
def _WaitForSync(lu, instance, disks=None, oneshot=False):
3601
  """Sleep and poll for an instance's disk to sync.
3602

3603
  """
3604
  if not instance.disks or disks is not None and not disks:
3605
    return True
3606

    
3607
  disks = _ExpandCheckDisks(instance, disks)
3608

    
3609
  if not oneshot:
3610
    lu.proc.LogInfo("Waiting for instance %s to sync disks." % instance.name)
3611

    
3612
  node = instance.primary_node
3613

    
3614
  for dev in disks:
3615
    lu.cfg.SetDiskID(dev, node)
3616

    
3617
  # TODO: Convert to utils.Retry
3618

    
3619
  retries = 0
3620
  degr_retries = 10 # in seconds, as we sleep 1 second each time
3621
  while True:
3622
    max_time = 0
3623
    done = True
3624
    cumul_degraded = False
3625
    rstats = lu.rpc.call_blockdev_getmirrorstatus(node, disks)
3626
    msg = rstats.fail_msg
3627
    if msg:
3628
      lu.LogWarning("Can't get any data from node %s: %s", node, msg)
3629
      retries += 1
3630
      if retries >= 10:
3631
        raise errors.RemoteError("Can't contact node %s for mirror data,"
3632
                                 " aborting." % node)
3633
      time.sleep(6)
3634
      continue
3635
    rstats = rstats.payload
3636
    retries = 0
3637
    for i, mstat in enumerate(rstats):
3638
      if mstat is None:
3639
        lu.LogWarning("Can't compute data for node %s/%s",
3640
                           node, disks[i].iv_name)
3641
        continue
3642

    
3643
      cumul_degraded = (cumul_degraded or
3644
                        (mstat.is_degraded and mstat.sync_percent is None))
3645
      if mstat.sync_percent is not None:
3646
        done = False
3647
        if mstat.estimated_time is not None:
3648
          rem_time = ("%s remaining (estimated)" %
3649
                      utils.FormatSeconds(mstat.estimated_time))
3650
          max_time = mstat.estimated_time
3651
        else:
3652
          rem_time = "no time estimate"
3653
        lu.proc.LogInfo("- device %s: %5.2f%% done, %s" %
3654
                        (disks[i].iv_name, mstat.sync_percent, rem_time))
3655

    
3656
    # if we're done but degraded, let's do a few small retries, to
3657
    # make sure we see a stable and not transient situation; therefore
3658
    # we force restart of the loop
3659
    if (done or oneshot) and cumul_degraded and degr_retries > 0:
3660
      logging.info("Degraded disks found, %d retries left", degr_retries)
3661
      degr_retries -= 1
3662
      time.sleep(1)
3663
      continue
3664

    
3665
    if done or oneshot:
3666
      break
3667

    
3668
    time.sleep(min(60, max_time))
3669

    
3670
  if done:
3671
    lu.proc.LogInfo("Instance %s's disks are in sync." % instance.name)
3672
  return not cumul_degraded
3673

    
3674

    
3675
def _CheckDiskConsistency(lu, dev, node, on_primary, ldisk=False):
3676
  """Check that mirrors are not degraded.
3677

3678
  The ldisk parameter, if True, will change the test from the
3679
  is_degraded attribute (which represents overall non-ok status for
3680
  the device(s)) to the ldisk (representing the local storage status).
3681

3682
  """
3683
  lu.cfg.SetDiskID(dev, node)
3684

    
3685
  result = True
3686

    
3687
  if on_primary or dev.AssembleOnSecondary():
3688
    rstats = lu.rpc.call_blockdev_find(node, dev)
3689
    msg = rstats.fail_msg
3690
    if msg:
3691
      lu.LogWarning("Can't find disk on node %s: %s", node, msg)
3692
      result = False
3693
    elif not rstats.payload:
3694
      lu.LogWarning("Can't find disk on node %s", node)
3695
      result = False
3696
    else:
3697
      if ldisk:
3698
        result = result and rstats.payload.ldisk_status == constants.LDS_OKAY
3699
      else:
3700
        result = result and not rstats.payload.is_degraded
3701

    
3702
  if dev.children:
3703
    for child in dev.children:
3704
      result = result and _CheckDiskConsistency(lu, child, node, on_primary)
3705

    
3706
  return result
3707

    
3708

    
3709
class LUOobCommand(NoHooksLU):
3710
  """Logical unit for OOB handling.
3711

3712
  """
3713
  REG_BGL = False
3714
  _SKIP_MASTER = (constants.OOB_POWER_OFF, constants.OOB_POWER_CYCLE)
3715

    
3716
  def ExpandNames(self):
3717
    """Gather locks we need.
3718

3719
    """
3720
    if self.op.node_names:
3721
      self.op.node_names = _GetWantedNodes(self, self.op.node_names)
3722
      lock_names = self.op.node_names
3723
    else:
3724
      lock_names = locking.ALL_SET
3725

    
3726
    self.needed_locks = {
3727
      locking.LEVEL_NODE: lock_names,
3728
      }
3729

    
3730
  def CheckPrereq(self):
3731
    """Check prerequisites.
3732

3733
    This checks:
3734
     - the node exists in the configuration
3735
     - OOB is supported
3736

3737
    Any errors are signaled by raising errors.OpPrereqError.
3738

3739
    """
3740
    self.nodes = []
3741
    self.master_node = self.cfg.GetMasterNode()
3742

    
3743
    assert self.op.power_delay >= 0.0
3744

    
3745
    if self.op.node_names:
3746
      if (self.op.command in self._SKIP_MASTER and
3747
          self.master_node in self.op.node_names):
3748
        master_node_obj = self.cfg.GetNodeInfo(self.master_node)
3749
        master_oob_handler = _SupportsOob(self.cfg, master_node_obj)
3750

    
3751
        if master_oob_handler:
3752
          additional_text = ("run '%s %s %s' if you want to operate on the"
3753
                             " master regardless") % (master_oob_handler,
3754
                                                      self.op.command,
3755
                                                      self.master_node)
3756
        else:
3757
          additional_text = "it does not support out-of-band operations"
3758

    
3759
        raise errors.OpPrereqError(("Operating on the master node %s is not"
3760
                                    " allowed for %s; %s") %
3761
                                   (self.master_node, self.op.command,
3762
                                    additional_text), errors.ECODE_INVAL)
3763
    else:
3764
      self.op.node_names = self.cfg.GetNodeList()
3765
      if self.op.command in self._SKIP_MASTER:
3766
        self.op.node_names.remove(self.master_node)
3767

    
3768
    if self.op.command in self._SKIP_MASTER:
3769
      assert self.master_node not in self.op.node_names
3770

    
3771
    for node_name in self.op.node_names:
3772
      node = self.cfg.GetNodeInfo(node_name)
3773

    
3774
      if node is None:
3775
        raise errors.OpPrereqError("Node %s not found" % node_name,
3776
                                   errors.ECODE_NOENT)
3777
      else:
3778
        self.nodes.append(node)
3779

    
3780
      if (not self.op.ignore_status and
3781
          (self.op.command == constants.OOB_POWER_OFF and not node.offline)):
3782
        raise errors.OpPrereqError(("Cannot power off node %s because it is"
3783
                                    " not marked offline") % node_name,
3784
                                   errors.ECODE_STATE)
3785

    
3786
  def Exec(self, feedback_fn):
3787
    """Execute OOB and return result if we expect any.
3788

3789
    """
3790
    master_node = self.master_node
3791
    ret = []
3792

    
3793
    for idx, node in enumerate(utils.NiceSort(self.nodes,
3794
                                              key=lambda node: node.name)):
3795
      node_entry = [(constants.RS_NORMAL, node.name)]
3796
      ret.append(node_entry)
3797

    
3798
      oob_program = _SupportsOob(self.cfg, node)
3799

    
3800
      if not oob_program:
3801
        node_entry.append((constants.RS_UNAVAIL, None))
3802
        continue
3803

    
3804
      logging.info("Executing out-of-band command '%s' using '%s' on %s",
3805
                   self.op.command, oob_program, node.name)
3806
      result = self.rpc.call_run_oob(master_node, oob_program,
3807
                                     self.op.command, node.name,
3808
                                     self.op.timeout)
3809

    
3810
      if result.fail_msg:
3811
        self.LogWarning("Out-of-band RPC failed on node '%s': %s",
3812
                        node.name, result.fail_msg)
3813
        node_entry.append((constants.RS_NODATA, None))
3814
      else:
3815
        try:
3816
          self._CheckPayload(result)
3817
        except errors.OpExecError, err:
3818
          self.LogWarning("Payload returned by node '%s' is not valid: %s",
3819
                          node.name, err)
3820
          node_entry.append((constants.RS_NODATA, None))
3821
        else:
3822
          if self.op.command == constants.OOB_HEALTH:
3823
            # For health we should log important events
3824
            for item, status in result.payload:
3825
              if status in [constants.OOB_STATUS_WARNING,
3826
                            constants.OOB_STATUS_CRITICAL]:
3827
                self.LogWarning("Item '%s' on node '%s' has status '%s'",
3828
                                item, node.name, status)
3829

    
3830
          if self.op.command == constants.OOB_POWER_ON:
3831
            node.powered = True
3832
          elif self.op.command == constants.OOB_POWER_OFF:
3833
            node.powered = False
3834
          elif self.op.command == constants.OOB_POWER_STATUS:
3835
            powered = result.payload[constants.OOB_POWER_STATUS_POWERED]
3836
            if powered != node.powered:
3837
              logging.warning(("Recorded power state (%s) of node '%s' does not"
3838
                               " match actual power state (%s)"), node.powered,
3839
                              node.name, powered)
3840

    
3841
          # For configuration changing commands we should update the node
3842
          if self.op.command in (constants.OOB_POWER_ON,
3843
                                 constants.OOB_POWER_OFF):
3844
            self.cfg.Update(node, feedback_fn)
3845

    
3846
          node_entry.append((constants.RS_NORMAL, result.payload))
3847

    
3848
          if (self.op.command == constants.OOB_POWER_ON and
3849
              idx < len(self.nodes) - 1):
3850
            time.sleep(self.op.power_delay)
3851

    
3852
    return ret
3853

    
3854
  def _CheckPayload(self, result):
3855
    """Checks if the payload is valid.
3856

3857
    @param result: RPC result
3858
    @raises errors.OpExecError: If payload is not valid
3859

3860
    """
3861
    errs = []
3862
    if self.op.command == constants.OOB_HEALTH:
3863
      if not isinstance(result.payload, list):
3864
        errs.append("command 'health' is expected to return a list but got %s" %
3865
                    type(result.payload))
3866
      else:
3867
        for item, status in result.payload:
3868
          if status not in constants.OOB_STATUSES:
3869
            errs.append("health item '%s' has invalid status '%s'" %
3870
                        (item, status))
3871

    
3872
    if self.op.command == constants.OOB_POWER_STATUS:
3873
      if not isinstance(result.payload, dict):
3874
        errs.append("power-status is expected to return a dict but got %s" %
3875
                    type(result.payload))
3876

    
3877
    if self.op.command in [
3878
        constants.OOB_POWER_ON,
3879
        constants.OOB_POWER_OFF,
3880
        constants.OOB_POWER_CYCLE,
3881
        ]:
3882
      if result.payload is not None:
3883
        errs.append("%s is expected to not return payload but got '%s'" %
3884
                    (self.op.command, result.payload))
3885

    
3886
    if errs:
3887
      raise errors.OpExecError("Check of out-of-band payload failed due to %s" %
3888
                               utils.CommaJoin(errs))
3889

    
3890
class _OsQuery(_QueryBase):
3891
  FIELDS = query.OS_FIELDS
3892

    
3893
  def ExpandNames(self, lu):
3894
    # Lock all nodes in shared mode
3895
    # Temporary removal of locks, should be reverted later
3896
    # TODO: reintroduce locks when they are lighter-weight
3897
    lu.needed_locks = {}
3898
    #self.share_locks[locking.LEVEL_NODE] = 1
3899
    #self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET
3900

    
3901
    # The following variables interact with _QueryBase._GetNames
3902
    if self.names:
3903
      self.wanted = self.names
3904
    else:
3905
      self.wanted = locking.ALL_SET
3906

    
3907
    self.do_locking = self.use_locking
3908

    
3909
  def DeclareLocks(self, lu, level):
3910
    pass
3911

    
3912
  @staticmethod
3913
  def _DiagnoseByOS(rlist):
3914
    """Remaps a per-node return list into an a per-os per-node dictionary
3915

3916
    @param rlist: a map with node names as keys and OS objects as values
3917

3918
    @rtype: dict
3919
    @return: a dictionary with osnames as keys and as value another
3920
        map, with nodes as keys and tuples of (path, status, diagnose,
3921
        variants, parameters, api_versions) as values, eg::
3922

3923
          {"debian-etch": {"node1": [(/usr/lib/..., True, "", [], []),
3924
                                     (/srv/..., False, "invalid api")],
3925
                           "node2": [(/srv/..., True, "", [], [])]}
3926
          }
3927

3928
    """
3929
    all_os = {}
3930
    # we build here the list of nodes that didn't fail the RPC (at RPC
3931
    # level), so that nodes with a non-responding node daemon don't
3932
    # make all OSes invalid
3933
    good_nodes = [node_name for node_name in rlist
3934
                  if not rlist[node_name].fail_msg]
3935
    for node_name, nr in rlist.items():
3936
      if nr.fail_msg or not nr.payload:
3937
        continue
3938
      for (name, path, status, diagnose, variants,
3939
           params, api_versions) in nr.payload:
3940
        if name not in all_os:
3941
          # build a list of nodes for this os containing empty lists
3942
          # for each node in node_list
3943
          all_os[name] = {}
3944
          for nname in good_nodes:
3945
            all_os[name][nname] = []
3946
        # convert params from [name, help] to (name, help)
3947
        params = [tuple(v) for v in params]
3948
        all_os[name][node_name].append((path, status, diagnose,
3949
                                        variants, params, api_versions))
3950
    return all_os
3951

    
3952
  def _GetQueryData(self, lu):
3953
    """Computes the list of nodes and their attributes.
3954

3955
    """
3956
    # Locking is not used
3957
    assert not (compat.any(lu.glm.is_owned(level)
3958
                           for level in locking.LEVELS
3959
                           if level != locking.LEVEL_CLUSTER) or
3960
                self.do_locking or self.use_locking)
3961

    
3962
    valid_nodes = [node.name
3963
                   for node in lu.cfg.GetAllNodesInfo().values()
3964
                   if not node.offline and node.vm_capable]
3965
    pol = self._DiagnoseByOS(lu.rpc.call_os_diagnose(valid_nodes))
3966
    cluster = lu.cfg.GetClusterInfo()
3967

    
3968
    data = {}
3969

    
3970
    for (os_name, os_data) in pol.items():
3971
      info = query.OsInfo(name=os_name, valid=True, node_status=os_data,
3972
                          hidden=(os_name in cluster.hidden_os),
3973
                          blacklisted=(os_name in cluster.blacklisted_os))
3974

    
3975
      variants = set()
3976
      parameters = set()
3977
      api_versions = set()
3978

    
3979
      for idx, osl in enumerate(os_data.values()):
3980
        info.valid = bool(info.valid and osl and osl[0][1])
3981
        if not info.valid:
3982
          break
3983

    
3984
        (node_variants, node_params, node_api) = osl[0][3:6]
3985
        if idx == 0:
3986
          # First entry
3987
          variants.update(node_variants)
3988
          parameters.update(node_params)
3989
          api_versions.update(node_api)
3990
        else:
3991
          # Filter out inconsistent values
3992
          variants.intersection_update(node_variants)
3993
          parameters.intersection_update(node_params)
3994
          api_versions.intersection_update(node_api)
3995

    
3996
      info.variants = list(variants)
3997
      info.parameters = list(parameters)
3998
      info.api_versions = list(api_versions)
3999

    
4000
      data[os_name] = info
4001

    
4002
    # Prepare data in requested order
4003
    return [data[name] for name in self._GetNames(lu, pol.keys(), None)
4004
            if name in data]
4005

    
4006

    
4007
class LUOsDiagnose(NoHooksLU):
4008
  """Logical unit for OS diagnose/query.
4009

4010
  """
4011
  REQ_BGL = False
4012

    
4013
  @staticmethod
4014
  def _BuildFilter(fields, names):
4015
    """Builds a filter for querying OSes.
4016

4017
    """
4018
    name_filter = qlang.MakeSimpleFilter("name", names)
4019

    
4020
    # Legacy behaviour: Hide hidden, blacklisted or invalid OSes if the
4021
    # respective field is not requested
4022
    status_filter = [[qlang.OP_NOT, [qlang.OP_TRUE, fname]]
4023
                     for fname in ["hidden", "blacklisted"]
4024
                     if fname not in fields]
4025
    if "valid" not in fields:
4026
      status_filter.append([qlang.OP_TRUE, "valid"])
4027

    
4028
    if status_filter:
4029
      status_filter.insert(0, qlang.OP_AND)
4030
    else:
4031
      status_filter = None
4032

    
4033
    if name_filter and status_filter:
4034
      return [qlang.OP_AND, name_filter, status_filter]
4035
    elif name_filter:
4036
      return name_filter
4037
    else:
4038
      return status_filter
4039

    
4040
  def CheckArguments(self):
4041
    self.oq = _OsQuery(self._BuildFilter(self.op.output_fields, self.op.names),
4042
                       self.op.output_fields, False)
4043

    
4044
  def ExpandNames(self):
4045
    self.oq.ExpandNames(self)
4046

    
4047
  def Exec(self, feedback_fn):
4048
    return self.oq.OldStyleQuery(self)
4049

    
4050

    
4051
class LUNodeRemove(LogicalUnit):
4052
  """Logical unit for removing a node.
4053

4054
  """
4055
  HPATH = "node-remove"
4056
  HTYPE = constants.HTYPE_NODE
4057

    
4058
  def BuildHooksEnv(self):
4059
    """Build hooks env.
4060

4061
    This doesn't run on the target node in the pre phase as a failed
4062
    node would then be impossible to remove.
4063

4064
    """
4065
    return {
4066
      "OP_TARGET": self.op.node_name,
4067
      "NODE_NAME": self.op.node_name,
4068
      }
4069

    
4070
  def BuildHooksNodes(self):
4071
    """Build hooks nodes.
4072

4073
    """
4074
    all_nodes = self.cfg.GetNodeList()
4075
    try:
4076
      all_nodes.remove(self.op.node_name)
4077
    except ValueError:
4078
      logging.warning("Node '%s', which is about to be removed, was not found"
4079
                      " in the list of all nodes", self.op.node_name)
4080
    return (all_nodes, all_nodes)
4081

    
4082
  def CheckPrereq(self):
4083
    """Check prerequisites.
4084

4085
    This checks:
4086
     - the node exists in the configuration
4087
     - it does not have primary or secondary instances
4088
     - it's not the master
4089

4090
    Any errors are signaled by raising errors.OpPrereqError.
4091