Statistics
| Branch: | Tag: | Revision:

root / lib / cmdlib.py @ adfa3b26

History | View | Annotate | Download (442.9 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Module implementing the master-side code."""
23

    
24
# pylint: disable-msg=W0201,C0302
25

    
26
# W0201 since most LU attributes are defined in CheckPrereq or similar
27
# functions
28

    
29
# C0302: since we have waaaay to many lines in this module
30

    
31
import os
32
import os.path
33
import time
34
import re
35
import platform
36
import logging
37
import copy
38
import OpenSSL
39
import socket
40
import tempfile
41
import shutil
42
import itertools
43

    
44
from ganeti import ssh
45
from ganeti import utils
46
from ganeti import errors
47
from ganeti import hypervisor
48
from ganeti import locking
49
from ganeti import constants
50
from ganeti import objects
51
from ganeti import serializer
52
from ganeti import ssconf
53
from ganeti import uidpool
54
from ganeti import compat
55
from ganeti import masterd
56
from ganeti import netutils
57
from ganeti import query
58
from ganeti import qlang
59
from ganeti import opcodes
60
from ganeti import ht
61

    
62
import ganeti.masterd.instance # pylint: disable-msg=W0611
63

    
64

    
65
def _SupportsOob(cfg, node):
66
  """Tells if node supports OOB.
67

68
  @type cfg: L{config.ConfigWriter}
69
  @param cfg: The cluster configuration
70
  @type node: L{objects.Node}
71
  @param node: The node
72
  @return: The OOB script if supported or an empty string otherwise
73

74
  """
75
  return cfg.GetNdParams(node)[constants.ND_OOB_PROGRAM]
76

    
77

    
78
class ResultWithJobs:
79
  """Data container for LU results with jobs.
80

81
  Instances of this class returned from L{LogicalUnit.Exec} will be recognized
82
  by L{mcpu.Processor._ProcessResult}. The latter will then submit the jobs
83
  contained in the C{jobs} attribute and include the job IDs in the opcode
84
  result.
85

86
  """
87
  def __init__(self, jobs, **kwargs):
88
    """Initializes this class.
89

90
    Additional return values can be specified as keyword arguments.
91

92
    @type jobs: list of lists of L{opcode.OpCode}
93
    @param jobs: A list of lists of opcode objects
94

95
    """
96
    self.jobs = jobs
97
    self.other = kwargs
98

    
99

    
100
class LogicalUnit(object):
101
  """Logical Unit base class.
102

103
  Subclasses must follow these rules:
104
    - implement ExpandNames
105
    - implement CheckPrereq (except when tasklets are used)
106
    - implement Exec (except when tasklets are used)
107
    - implement BuildHooksEnv
108
    - implement BuildHooksNodes
109
    - redefine HPATH and HTYPE
110
    - optionally redefine their run requirements:
111
        REQ_BGL: the LU needs to hold the Big Ganeti Lock exclusively
112

113
  Note that all commands require root permissions.
114

115
  @ivar dry_run_result: the value (if any) that will be returned to the caller
116
      in dry-run mode (signalled by opcode dry_run parameter)
117

118
  """
119
  HPATH = None
120
  HTYPE = None
121
  REQ_BGL = True
122

    
123
  def __init__(self, processor, op, context, rpc):
124
    """Constructor for LogicalUnit.
125

126
    This needs to be overridden in derived classes in order to check op
127
    validity.
128

129
    """
130
    self.proc = processor
131
    self.op = op
132
    self.cfg = context.cfg
133
    self.glm = context.glm
134
    self.context = context
135
    self.rpc = rpc
136
    # Dicts used to declare locking needs to mcpu
137
    self.needed_locks = None
138
    self.share_locks = dict.fromkeys(locking.LEVELS, 0)
139
    self.add_locks = {}
140
    self.remove_locks = {}
141
    # Used to force good behavior when calling helper functions
142
    self.recalculate_locks = {}
143
    # logging
144
    self.Log = processor.Log # pylint: disable-msg=C0103
145
    self.LogWarning = processor.LogWarning # pylint: disable-msg=C0103
146
    self.LogInfo = processor.LogInfo # pylint: disable-msg=C0103
147
    self.LogStep = processor.LogStep # pylint: disable-msg=C0103
148
    # support for dry-run
149
    self.dry_run_result = None
150
    # support for generic debug attribute
151
    if (not hasattr(self.op, "debug_level") or
152
        not isinstance(self.op.debug_level, int)):
153
      self.op.debug_level = 0
154

    
155
    # Tasklets
156
    self.tasklets = None
157

    
158
    # Validate opcode parameters and set defaults
159
    self.op.Validate(True)
160

    
161
    self.CheckArguments()
162

    
163
  def CheckArguments(self):
164
    """Check syntactic validity for the opcode arguments.
165

166
    This method is for doing a simple syntactic check and ensure
167
    validity of opcode parameters, without any cluster-related
168
    checks. While the same can be accomplished in ExpandNames and/or
169
    CheckPrereq, doing these separate is better because:
170

171
      - ExpandNames is left as as purely a lock-related function
172
      - CheckPrereq is run after we have acquired locks (and possible
173
        waited for them)
174

175
    The function is allowed to change the self.op attribute so that
176
    later methods can no longer worry about missing parameters.
177

178
    """
179
    pass
180

    
181
  def ExpandNames(self):
182
    """Expand names for this LU.
183

184
    This method is called before starting to execute the opcode, and it should
185
    update all the parameters of the opcode to their canonical form (e.g. a
186
    short node name must be fully expanded after this method has successfully
187
    completed). This way locking, hooks, logging, etc. can work correctly.
188

189
    LUs which implement this method must also populate the self.needed_locks
190
    member, as a dict with lock levels as keys, and a list of needed lock names
191
    as values. Rules:
192

193
      - use an empty dict if you don't need any lock
194
      - if you don't need any lock at a particular level omit that level
195
      - don't put anything for the BGL level
196
      - if you want all locks at a level use locking.ALL_SET as a value
197

198
    If you need to share locks (rather than acquire them exclusively) at one
199
    level you can modify self.share_locks, setting a true value (usually 1) for
200
    that level. By default locks are not shared.
201

202
    This function can also define a list of tasklets, which then will be
203
    executed in order instead of the usual LU-level CheckPrereq and Exec
204
    functions, if those are not defined by the LU.
205

206
    Examples::
207

208
      # Acquire all nodes and one instance
209
      self.needed_locks = {
210
        locking.LEVEL_NODE: locking.ALL_SET,
211
        locking.LEVEL_INSTANCE: ['instance1.example.com'],
212
      }
213
      # Acquire just two nodes
214
      self.needed_locks = {
215
        locking.LEVEL_NODE: ['node1.example.com', 'node2.example.com'],
216
      }
217
      # Acquire no locks
218
      self.needed_locks = {} # No, you can't leave it to the default value None
219

220
    """
221
    # The implementation of this method is mandatory only if the new LU is
222
    # concurrent, so that old LUs don't need to be changed all at the same
223
    # time.
224
    if self.REQ_BGL:
225
      self.needed_locks = {} # Exclusive LUs don't need locks.
226
    else:
227
      raise NotImplementedError
228

    
229
  def DeclareLocks(self, level):
230
    """Declare LU locking needs for a level
231

232
    While most LUs can just declare their locking needs at ExpandNames time,
233
    sometimes there's the need to calculate some locks after having acquired
234
    the ones before. This function is called just before acquiring locks at a
235
    particular level, but after acquiring the ones at lower levels, and permits
236
    such calculations. It can be used to modify self.needed_locks, and by
237
    default it does nothing.
238

239
    This function is only called if you have something already set in
240
    self.needed_locks for the level.
241

242
    @param level: Locking level which is going to be locked
243
    @type level: member of ganeti.locking.LEVELS
244

245
    """
246

    
247
  def CheckPrereq(self):
248
    """Check prerequisites for this LU.
249

250
    This method should check that the prerequisites for the execution
251
    of this LU are fulfilled. It can do internode communication, but
252
    it should be idempotent - no cluster or system changes are
253
    allowed.
254

255
    The method should raise errors.OpPrereqError in case something is
256
    not fulfilled. Its return value is ignored.
257

258
    This method should also update all the parameters of the opcode to
259
    their canonical form if it hasn't been done by ExpandNames before.
260

261
    """
262
    if self.tasklets is not None:
263
      for (idx, tl) in enumerate(self.tasklets):
264
        logging.debug("Checking prerequisites for tasklet %s/%s",
265
                      idx + 1, len(self.tasklets))
266
        tl.CheckPrereq()
267
    else:
268
      pass
269

    
270
  def Exec(self, feedback_fn):
271
    """Execute the LU.
272

273
    This method should implement the actual work. It should raise
274
    errors.OpExecError for failures that are somewhat dealt with in
275
    code, or expected.
276

277
    """
278
    if self.tasklets is not None:
279
      for (idx, tl) in enumerate(self.tasklets):
280
        logging.debug("Executing tasklet %s/%s", idx + 1, len(self.tasklets))
281
        tl.Exec(feedback_fn)
282
    else:
283
      raise NotImplementedError
284

    
285
  def BuildHooksEnv(self):
286
    """Build hooks environment for this LU.
287

288
    @rtype: dict
289
    @return: Dictionary containing the environment that will be used for
290
      running the hooks for this LU. The keys of the dict must not be prefixed
291
      with "GANETI_"--that'll be added by the hooks runner. The hooks runner
292
      will extend the environment with additional variables. If no environment
293
      should be defined, an empty dictionary should be returned (not C{None}).
294
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
295
      will not be called.
296

297
    """
298
    raise NotImplementedError
299

    
300
  def BuildHooksNodes(self):
301
    """Build list of nodes to run LU's hooks.
302

303
    @rtype: tuple; (list, list)
304
    @return: Tuple containing a list of node names on which the hook
305
      should run before the execution and a list of node names on which the
306
      hook should run after the execution. No nodes should be returned as an
307
      empty list (and not None).
308
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
309
      will not be called.
310

311
    """
312
    raise NotImplementedError
313

    
314
  def HooksCallBack(self, phase, hook_results, feedback_fn, lu_result):
315
    """Notify the LU about the results of its hooks.
316

317
    This method is called every time a hooks phase is executed, and notifies
318
    the Logical Unit about the hooks' result. The LU can then use it to alter
319
    its result based on the hooks.  By default the method does nothing and the
320
    previous result is passed back unchanged but any LU can define it if it
321
    wants to use the local cluster hook-scripts somehow.
322

323
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
324
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
325
    @param hook_results: the results of the multi-node hooks rpc call
326
    @param feedback_fn: function used send feedback back to the caller
327
    @param lu_result: the previous Exec result this LU had, or None
328
        in the PRE phase
329
    @return: the new Exec result, based on the previous result
330
        and hook results
331

332
    """
333
    # API must be kept, thus we ignore the unused argument and could
334
    # be a function warnings
335
    # pylint: disable-msg=W0613,R0201
336
    return lu_result
337

    
338
  def _ExpandAndLockInstance(self):
339
    """Helper function to expand and lock an instance.
340

341
    Many LUs that work on an instance take its name in self.op.instance_name
342
    and need to expand it and then declare the expanded name for locking. This
343
    function does it, and then updates self.op.instance_name to the expanded
344
    name. It also initializes needed_locks as a dict, if this hasn't been done
345
    before.
346

347
    """
348
    if self.needed_locks is None:
349
      self.needed_locks = {}
350
    else:
351
      assert locking.LEVEL_INSTANCE not in self.needed_locks, \
352
        "_ExpandAndLockInstance called with instance-level locks set"
353
    self.op.instance_name = _ExpandInstanceName(self.cfg,
354
                                                self.op.instance_name)
355
    self.needed_locks[locking.LEVEL_INSTANCE] = self.op.instance_name
356

    
357
  def _LockInstancesNodes(self, primary_only=False):
358
    """Helper function to declare instances' nodes for locking.
359

360
    This function should be called after locking one or more instances to lock
361
    their nodes. Its effect is populating self.needed_locks[locking.LEVEL_NODE]
362
    with all primary or secondary nodes for instances already locked and
363
    present in self.needed_locks[locking.LEVEL_INSTANCE].
364

365
    It should be called from DeclareLocks, and for safety only works if
366
    self.recalculate_locks[locking.LEVEL_NODE] is set.
367

368
    In the future it may grow parameters to just lock some instance's nodes, or
369
    to just lock primaries or secondary nodes, if needed.
370

371
    If should be called in DeclareLocks in a way similar to::
372

373
      if level == locking.LEVEL_NODE:
374
        self._LockInstancesNodes()
375

376
    @type primary_only: boolean
377
    @param primary_only: only lock primary nodes of locked instances
378

379
    """
380
    assert locking.LEVEL_NODE in self.recalculate_locks, \
381
      "_LockInstancesNodes helper function called with no nodes to recalculate"
382

    
383
    # TODO: check if we're really been called with the instance locks held
384

    
385
    # For now we'll replace self.needed_locks[locking.LEVEL_NODE], but in the
386
    # future we might want to have different behaviors depending on the value
387
    # of self.recalculate_locks[locking.LEVEL_NODE]
388
    wanted_nodes = []
389
    for instance_name in self.glm.list_owned(locking.LEVEL_INSTANCE):
390
      instance = self.context.cfg.GetInstanceInfo(instance_name)
391
      wanted_nodes.append(instance.primary_node)
392
      if not primary_only:
393
        wanted_nodes.extend(instance.secondary_nodes)
394

    
395
    if self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_REPLACE:
396
      self.needed_locks[locking.LEVEL_NODE] = wanted_nodes
397
    elif self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_APPEND:
398
      self.needed_locks[locking.LEVEL_NODE].extend(wanted_nodes)
399

    
400
    del self.recalculate_locks[locking.LEVEL_NODE]
401

    
402

    
403
class NoHooksLU(LogicalUnit): # pylint: disable-msg=W0223
404
  """Simple LU which runs no hooks.
405

406
  This LU is intended as a parent for other LogicalUnits which will
407
  run no hooks, in order to reduce duplicate code.
408

409
  """
410
  HPATH = None
411
  HTYPE = None
412

    
413
  def BuildHooksEnv(self):
414
    """Empty BuildHooksEnv for NoHooksLu.
415

416
    This just raises an error.
417

418
    """
419
    raise AssertionError("BuildHooksEnv called for NoHooksLUs")
420

    
421
  def BuildHooksNodes(self):
422
    """Empty BuildHooksNodes for NoHooksLU.
423

424
    """
425
    raise AssertionError("BuildHooksNodes called for NoHooksLU")
426

    
427

    
428
class Tasklet:
429
  """Tasklet base class.
430

431
  Tasklets are subcomponents for LUs. LUs can consist entirely of tasklets or
432
  they can mix legacy code with tasklets. Locking needs to be done in the LU,
433
  tasklets know nothing about locks.
434

435
  Subclasses must follow these rules:
436
    - Implement CheckPrereq
437
    - Implement Exec
438

439
  """
440
  def __init__(self, lu):
441
    self.lu = lu
442

    
443
    # Shortcuts
444
    self.cfg = lu.cfg
445
    self.rpc = lu.rpc
446

    
447
  def CheckPrereq(self):
448
    """Check prerequisites for this tasklets.
449

450
    This method should check whether the prerequisites for the execution of
451
    this tasklet are fulfilled. It can do internode communication, but it
452
    should be idempotent - no cluster or system changes are allowed.
453

454
    The method should raise errors.OpPrereqError in case something is not
455
    fulfilled. Its return value is ignored.
456

457
    This method should also update all parameters to their canonical form if it
458
    hasn't been done before.
459

460
    """
461
    pass
462

    
463
  def Exec(self, feedback_fn):
464
    """Execute the tasklet.
465

466
    This method should implement the actual work. It should raise
467
    errors.OpExecError for failures that are somewhat dealt with in code, or
468
    expected.
469

470
    """
471
    raise NotImplementedError
472

    
473

    
474
class _QueryBase:
475
  """Base for query utility classes.
476

477
  """
478
  #: Attribute holding field definitions
479
  FIELDS = None
480

    
481
  def __init__(self, filter_, fields, use_locking):
482
    """Initializes this class.
483

484
    """
485
    self.use_locking = use_locking
486

    
487
    self.query = query.Query(self.FIELDS, fields, filter_=filter_,
488
                             namefield="name")
489
    self.requested_data = self.query.RequestedData()
490
    self.names = self.query.RequestedNames()
491

    
492
    # Sort only if no names were requested
493
    self.sort_by_name = not self.names
494

    
495
    self.do_locking = None
496
    self.wanted = None
497

    
498
  def _GetNames(self, lu, all_names, lock_level):
499
    """Helper function to determine names asked for in the query.
500

501
    """
502
    if self.do_locking:
503
      names = lu.glm.list_owned(lock_level)
504
    else:
505
      names = all_names
506

    
507
    if self.wanted == locking.ALL_SET:
508
      assert not self.names
509
      # caller didn't specify names, so ordering is not important
510
      return utils.NiceSort(names)
511

    
512
    # caller specified names and we must keep the same order
513
    assert self.names
514
    assert not self.do_locking or lu.glm.is_owned(lock_level)
515

    
516
    missing = set(self.wanted).difference(names)
517
    if missing:
518
      raise errors.OpExecError("Some items were removed before retrieving"
519
                               " their data: %s" % missing)
520

    
521
    # Return expanded names
522
    return self.wanted
523

    
524
  def ExpandNames(self, lu):
525
    """Expand names for this query.
526

527
    See L{LogicalUnit.ExpandNames}.
528

529
    """
530
    raise NotImplementedError()
531

    
532
  def DeclareLocks(self, lu, level):
533
    """Declare locks for this query.
534

535
    See L{LogicalUnit.DeclareLocks}.
536

537
    """
538
    raise NotImplementedError()
539

    
540
  def _GetQueryData(self, lu):
541
    """Collects all data for this query.
542

543
    @return: Query data object
544

545
    """
546
    raise NotImplementedError()
547

    
548
  def NewStyleQuery(self, lu):
549
    """Collect data and execute query.
550

551
    """
552
    return query.GetQueryResponse(self.query, self._GetQueryData(lu),
553
                                  sort_by_name=self.sort_by_name)
554

    
555
  def OldStyleQuery(self, lu):
556
    """Collect data and execute query.
557

558
    """
559
    return self.query.OldStyleQuery(self._GetQueryData(lu),
560
                                    sort_by_name=self.sort_by_name)
561

    
562

    
563
def _GetWantedNodes(lu, nodes):
564
  """Returns list of checked and expanded node names.
565

566
  @type lu: L{LogicalUnit}
567
  @param lu: the logical unit on whose behalf we execute
568
  @type nodes: list
569
  @param nodes: list of node names or None for all nodes
570
  @rtype: list
571
  @return: the list of nodes, sorted
572
  @raise errors.ProgrammerError: if the nodes parameter is wrong type
573

574
  """
575
  if nodes:
576
    return [_ExpandNodeName(lu.cfg, name) for name in nodes]
577

    
578
  return utils.NiceSort(lu.cfg.GetNodeList())
579

    
580

    
581
def _GetWantedInstances(lu, instances):
582
  """Returns list of checked and expanded instance names.
583

584
  @type lu: L{LogicalUnit}
585
  @param lu: the logical unit on whose behalf we execute
586
  @type instances: list
587
  @param instances: list of instance names or None for all instances
588
  @rtype: list
589
  @return: the list of instances, sorted
590
  @raise errors.OpPrereqError: if the instances parameter is wrong type
591
  @raise errors.OpPrereqError: if any of the passed instances is not found
592

593
  """
594
  if instances:
595
    wanted = [_ExpandInstanceName(lu.cfg, name) for name in instances]
596
  else:
597
    wanted = utils.NiceSort(lu.cfg.GetInstanceList())
598
  return wanted
599

    
600

    
601
def _GetUpdatedParams(old_params, update_dict,
602
                      use_default=True, use_none=False):
603
  """Return the new version of a parameter dictionary.
604

605
  @type old_params: dict
606
  @param old_params: old parameters
607
  @type update_dict: dict
608
  @param update_dict: dict containing new parameter values, or
609
      constants.VALUE_DEFAULT to reset the parameter to its default
610
      value
611
  @param use_default: boolean
612
  @type use_default: whether to recognise L{constants.VALUE_DEFAULT}
613
      values as 'to be deleted' values
614
  @param use_none: boolean
615
  @type use_none: whether to recognise C{None} values as 'to be
616
      deleted' values
617
  @rtype: dict
618
  @return: the new parameter dictionary
619

620
  """
621
  params_copy = copy.deepcopy(old_params)
622
  for key, val in update_dict.iteritems():
623
    if ((use_default and val == constants.VALUE_DEFAULT) or
624
        (use_none and val is None)):
625
      try:
626
        del params_copy[key]
627
      except KeyError:
628
        pass
629
    else:
630
      params_copy[key] = val
631
  return params_copy
632

    
633

    
634
def _ReleaseLocks(lu, level, names=None, keep=None):
635
  """Releases locks owned by an LU.
636

637
  @type lu: L{LogicalUnit}
638
  @param level: Lock level
639
  @type names: list or None
640
  @param names: Names of locks to release
641
  @type keep: list or None
642
  @param keep: Names of locks to retain
643

644
  """
645
  assert not (keep is not None and names is not None), \
646
         "Only one of the 'names' and the 'keep' parameters can be given"
647

    
648
  if names is not None:
649
    should_release = names.__contains__
650
  elif keep:
651
    should_release = lambda name: name not in keep
652
  else:
653
    should_release = None
654

    
655
  if should_release:
656
    retain = []
657
    release = []
658

    
659
    # Determine which locks to release
660
    for name in lu.glm.list_owned(level):
661
      if should_release(name):
662
        release.append(name)
663
      else:
664
        retain.append(name)
665

    
666
    assert len(lu.glm.list_owned(level)) == (len(retain) + len(release))
667

    
668
    # Release just some locks
669
    lu.glm.release(level, names=release)
670

    
671
    assert frozenset(lu.glm.list_owned(level)) == frozenset(retain)
672
  else:
673
    # Release everything
674
    lu.glm.release(level)
675

    
676
    assert not lu.glm.is_owned(level), "No locks should be owned"
677

    
678

    
679
def _RunPostHook(lu, node_name):
680
  """Runs the post-hook for an opcode on a single node.
681

682
  """
683
  hm = lu.proc.hmclass(lu.rpc.call_hooks_runner, lu)
684
  try:
685
    hm.RunPhase(constants.HOOKS_PHASE_POST, nodes=[node_name])
686
  except:
687
    # pylint: disable-msg=W0702
688
    lu.LogWarning("Errors occurred running hooks on %s" % node_name)
689

    
690

    
691
def _CheckOutputFields(static, dynamic, selected):
692
  """Checks whether all selected fields are valid.
693

694
  @type static: L{utils.FieldSet}
695
  @param static: static fields set
696
  @type dynamic: L{utils.FieldSet}
697
  @param dynamic: dynamic fields set
698

699
  """
700
  f = utils.FieldSet()
701
  f.Extend(static)
702
  f.Extend(dynamic)
703

    
704
  delta = f.NonMatching(selected)
705
  if delta:
706
    raise errors.OpPrereqError("Unknown output fields selected: %s"
707
                               % ",".join(delta), errors.ECODE_INVAL)
708

    
709

    
710
def _CheckGlobalHvParams(params):
711
  """Validates that given hypervisor params are not global ones.
712

713
  This will ensure that instances don't get customised versions of
714
  global params.
715

716
  """
717
  used_globals = constants.HVC_GLOBALS.intersection(params)
718
  if used_globals:
719
    msg = ("The following hypervisor parameters are global and cannot"
720
           " be customized at instance level, please modify them at"
721
           " cluster level: %s" % utils.CommaJoin(used_globals))
722
    raise errors.OpPrereqError(msg, errors.ECODE_INVAL)
723

    
724

    
725
def _CheckNodeOnline(lu, node, msg=None):
726
  """Ensure that a given node is online.
727

728
  @param lu: the LU on behalf of which we make the check
729
  @param node: the node to check
730
  @param msg: if passed, should be a message to replace the default one
731
  @raise errors.OpPrereqError: if the node is offline
732

733
  """
734
  if msg is None:
735
    msg = "Can't use offline node"
736
  if lu.cfg.GetNodeInfo(node).offline:
737
    raise errors.OpPrereqError("%s: %s" % (msg, node), errors.ECODE_STATE)
738

    
739

    
740
def _CheckNodeNotDrained(lu, node):
741
  """Ensure that a given node is not drained.
742

743
  @param lu: the LU on behalf of which we make the check
744
  @param node: the node to check
745
  @raise errors.OpPrereqError: if the node is drained
746

747
  """
748
  if lu.cfg.GetNodeInfo(node).drained:
749
    raise errors.OpPrereqError("Can't use drained node %s" % node,
750
                               errors.ECODE_STATE)
751

    
752

    
753
def _CheckNodeVmCapable(lu, node):
754
  """Ensure that a given node is vm capable.
755

756
  @param lu: the LU on behalf of which we make the check
757
  @param node: the node to check
758
  @raise errors.OpPrereqError: if the node is not vm capable
759

760
  """
761
  if not lu.cfg.GetNodeInfo(node).vm_capable:
762
    raise errors.OpPrereqError("Can't use non-vm_capable node %s" % node,
763
                               errors.ECODE_STATE)
764

    
765

    
766
def _CheckNodeHasOS(lu, node, os_name, force_variant):
767
  """Ensure that a node supports a given OS.
768

769
  @param lu: the LU on behalf of which we make the check
770
  @param node: the node to check
771
  @param os_name: the OS to query about
772
  @param force_variant: whether to ignore variant errors
773
  @raise errors.OpPrereqError: if the node is not supporting the OS
774

775
  """
776
  result = lu.rpc.call_os_get(node, os_name)
777
  result.Raise("OS '%s' not in supported OS list for node %s" %
778
               (os_name, node),
779
               prereq=True, ecode=errors.ECODE_INVAL)
780
  if not force_variant:
781
    _CheckOSVariant(result.payload, os_name)
782

    
783

    
784
def _CheckNodeHasSecondaryIP(lu, node, secondary_ip, prereq):
785
  """Ensure that a node has the given secondary ip.
786

787
  @type lu: L{LogicalUnit}
788
  @param lu: the LU on behalf of which we make the check
789
  @type node: string
790
  @param node: the node to check
791
  @type secondary_ip: string
792
  @param secondary_ip: the ip to check
793
  @type prereq: boolean
794
  @param prereq: whether to throw a prerequisite or an execute error
795
  @raise errors.OpPrereqError: if the node doesn't have the ip, and prereq=True
796
  @raise errors.OpExecError: if the node doesn't have the ip, and prereq=False
797

798
  """
799
  result = lu.rpc.call_node_has_ip_address(node, secondary_ip)
800
  result.Raise("Failure checking secondary ip on node %s" % node,
801
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
802
  if not result.payload:
803
    msg = ("Node claims it doesn't have the secondary ip you gave (%s),"
804
           " please fix and re-run this command" % secondary_ip)
805
    if prereq:
806
      raise errors.OpPrereqError(msg, errors.ECODE_ENVIRON)
807
    else:
808
      raise errors.OpExecError(msg)
809

    
810

    
811
def _GetClusterDomainSecret():
812
  """Reads the cluster domain secret.
813

814
  """
815
  return utils.ReadOneLineFile(constants.CLUSTER_DOMAIN_SECRET_FILE,
816
                               strict=True)
817

    
818

    
819
def _CheckInstanceDown(lu, instance, reason):
820
  """Ensure that an instance is not running."""
821
  if instance.admin_up:
822
    raise errors.OpPrereqError("Instance %s is marked to be up, %s" %
823
                               (instance.name, reason), errors.ECODE_STATE)
824

    
825
  pnode = instance.primary_node
826
  ins_l = lu.rpc.call_instance_list([pnode], [instance.hypervisor])[pnode]
827
  ins_l.Raise("Can't contact node %s for instance information" % pnode,
828
              prereq=True, ecode=errors.ECODE_ENVIRON)
829

    
830
  if instance.name in ins_l.payload:
831
    raise errors.OpPrereqError("Instance %s is running, %s" %
832
                               (instance.name, reason), errors.ECODE_STATE)
833

    
834

    
835
def _ExpandItemName(fn, name, kind):
836
  """Expand an item name.
837

838
  @param fn: the function to use for expansion
839
  @param name: requested item name
840
  @param kind: text description ('Node' or 'Instance')
841
  @return: the resolved (full) name
842
  @raise errors.OpPrereqError: if the item is not found
843

844
  """
845
  full_name = fn(name)
846
  if full_name is None:
847
    raise errors.OpPrereqError("%s '%s' not known" % (kind, name),
848
                               errors.ECODE_NOENT)
849
  return full_name
850

    
851

    
852
def _ExpandNodeName(cfg, name):
853
  """Wrapper over L{_ExpandItemName} for nodes."""
854
  return _ExpandItemName(cfg.ExpandNodeName, name, "Node")
855

    
856

    
857
def _ExpandInstanceName(cfg, name):
858
  """Wrapper over L{_ExpandItemName} for instance."""
859
  return _ExpandItemName(cfg.ExpandInstanceName, name, "Instance")
860

    
861

    
862
def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status,
863
                          memory, vcpus, nics, disk_template, disks,
864
                          bep, hvp, hypervisor_name):
865
  """Builds instance related env variables for hooks
866

867
  This builds the hook environment from individual variables.
868

869
  @type name: string
870
  @param name: the name of the instance
871
  @type primary_node: string
872
  @param primary_node: the name of the instance's primary node
873
  @type secondary_nodes: list
874
  @param secondary_nodes: list of secondary nodes as strings
875
  @type os_type: string
876
  @param os_type: the name of the instance's OS
877
  @type status: boolean
878
  @param status: the should_run status of the instance
879
  @type memory: string
880
  @param memory: the memory size of the instance
881
  @type vcpus: string
882
  @param vcpus: the count of VCPUs the instance has
883
  @type nics: list
884
  @param nics: list of tuples (ip, mac, mode, link) representing
885
      the NICs the instance has
886
  @type disk_template: string
887
  @param disk_template: the disk template of the instance
888
  @type disks: list
889
  @param disks: the list of (size, mode) pairs
890
  @type bep: dict
891
  @param bep: the backend parameters for the instance
892
  @type hvp: dict
893
  @param hvp: the hypervisor parameters for the instance
894
  @type hypervisor_name: string
895
  @param hypervisor_name: the hypervisor for the instance
896
  @rtype: dict
897
  @return: the hook environment for this instance
898

899
  """
900
  if status:
901
    str_status = "up"
902
  else:
903
    str_status = "down"
904
  env = {
905
    "OP_TARGET": name,
906
    "INSTANCE_NAME": name,
907
    "INSTANCE_PRIMARY": primary_node,
908
    "INSTANCE_SECONDARIES": " ".join(secondary_nodes),
909
    "INSTANCE_OS_TYPE": os_type,
910
    "INSTANCE_STATUS": str_status,
911
    "INSTANCE_MEMORY": memory,
912
    "INSTANCE_VCPUS": vcpus,
913
    "INSTANCE_DISK_TEMPLATE": disk_template,
914
    "INSTANCE_HYPERVISOR": hypervisor_name,
915
  }
916

    
917
  if nics:
918
    nic_count = len(nics)
919
    for idx, (ip, mac, mode, link) in enumerate(nics):
920
      if ip is None:
921
        ip = ""
922
      env["INSTANCE_NIC%d_IP" % idx] = ip
923
      env["INSTANCE_NIC%d_MAC" % idx] = mac
924
      env["INSTANCE_NIC%d_MODE" % idx] = mode
925
      env["INSTANCE_NIC%d_LINK" % idx] = link
926
      if mode == constants.NIC_MODE_BRIDGED:
927
        env["INSTANCE_NIC%d_BRIDGE" % idx] = link
928
  else:
929
    nic_count = 0
930

    
931
  env["INSTANCE_NIC_COUNT"] = nic_count
932

    
933
  if disks:
934
    disk_count = len(disks)
935
    for idx, (size, mode) in enumerate(disks):
936
      env["INSTANCE_DISK%d_SIZE" % idx] = size
937
      env["INSTANCE_DISK%d_MODE" % idx] = mode
938
  else:
939
    disk_count = 0
940

    
941
  env["INSTANCE_DISK_COUNT"] = disk_count
942

    
943
  for source, kind in [(bep, "BE"), (hvp, "HV")]:
944
    for key, value in source.items():
945
      env["INSTANCE_%s_%s" % (kind, key)] = value
946

    
947
  return env
948

    
949

    
950
def _NICListToTuple(lu, nics):
951
  """Build a list of nic information tuples.
952

953
  This list is suitable to be passed to _BuildInstanceHookEnv or as a return
954
  value in LUInstanceQueryData.
955

956
  @type lu:  L{LogicalUnit}
957
  @param lu: the logical unit on whose behalf we execute
958
  @type nics: list of L{objects.NIC}
959
  @param nics: list of nics to convert to hooks tuples
960

961
  """
962
  hooks_nics = []
963
  cluster = lu.cfg.GetClusterInfo()
964
  for nic in nics:
965
    ip = nic.ip
966
    mac = nic.mac
967
    filled_params = cluster.SimpleFillNIC(nic.nicparams)
968
    mode = filled_params[constants.NIC_MODE]
969
    link = filled_params[constants.NIC_LINK]
970
    hooks_nics.append((ip, mac, mode, link))
971
  return hooks_nics
972

    
973

    
974
def _BuildInstanceHookEnvByObject(lu, instance, override=None):
975
  """Builds instance related env variables for hooks from an object.
976

977
  @type lu: L{LogicalUnit}
978
  @param lu: the logical unit on whose behalf we execute
979
  @type instance: L{objects.Instance}
980
  @param instance: the instance for which we should build the
981
      environment
982
  @type override: dict
983
  @param override: dictionary with key/values that will override
984
      our values
985
  @rtype: dict
986
  @return: the hook environment dictionary
987

988
  """
989
  cluster = lu.cfg.GetClusterInfo()
990
  bep = cluster.FillBE(instance)
991
  hvp = cluster.FillHV(instance)
992
  args = {
993
    'name': instance.name,
994
    'primary_node': instance.primary_node,
995
    'secondary_nodes': instance.secondary_nodes,
996
    'os_type': instance.os,
997
    'status': instance.admin_up,
998
    'memory': bep[constants.BE_MEMORY],
999
    'vcpus': bep[constants.BE_VCPUS],
1000
    'nics': _NICListToTuple(lu, instance.nics),
1001
    'disk_template': instance.disk_template,
1002
    'disks': [(disk.size, disk.mode) for disk in instance.disks],
1003
    'bep': bep,
1004
    'hvp': hvp,
1005
    'hypervisor_name': instance.hypervisor,
1006
  }
1007
  if override:
1008
    args.update(override)
1009
  return _BuildInstanceHookEnv(**args) # pylint: disable-msg=W0142
1010

    
1011

    
1012
def _AdjustCandidatePool(lu, exceptions):
1013
  """Adjust the candidate pool after node operations.
1014

1015
  """
1016
  mod_list = lu.cfg.MaintainCandidatePool(exceptions)
1017
  if mod_list:
1018
    lu.LogInfo("Promoted nodes to master candidate role: %s",
1019
               utils.CommaJoin(node.name for node in mod_list))
1020
    for name in mod_list:
1021
      lu.context.ReaddNode(name)
1022
  mc_now, mc_max, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1023
  if mc_now > mc_max:
1024
    lu.LogInfo("Note: more nodes are candidates (%d) than desired (%d)" %
1025
               (mc_now, mc_max))
1026

    
1027

    
1028
def _DecideSelfPromotion(lu, exceptions=None):
1029
  """Decide whether I should promote myself as a master candidate.
1030

1031
  """
1032
  cp_size = lu.cfg.GetClusterInfo().candidate_pool_size
1033
  mc_now, mc_should, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1034
  # the new node will increase mc_max with one, so:
1035
  mc_should = min(mc_should + 1, cp_size)
1036
  return mc_now < mc_should
1037

    
1038

    
1039
def _CheckNicsBridgesExist(lu, target_nics, target_node):
1040
  """Check that the brigdes needed by a list of nics exist.
1041

1042
  """
1043
  cluster = lu.cfg.GetClusterInfo()
1044
  paramslist = [cluster.SimpleFillNIC(nic.nicparams) for nic in target_nics]
1045
  brlist = [params[constants.NIC_LINK] for params in paramslist
1046
            if params[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED]
1047
  if brlist:
1048
    result = lu.rpc.call_bridges_exist(target_node, brlist)
1049
    result.Raise("Error checking bridges on destination node '%s'" %
1050
                 target_node, prereq=True, ecode=errors.ECODE_ENVIRON)
1051

    
1052

    
1053
def _CheckInstanceBridgesExist(lu, instance, node=None):
1054
  """Check that the brigdes needed by an instance exist.
1055

1056
  """
1057
  if node is None:
1058
    node = instance.primary_node
1059
  _CheckNicsBridgesExist(lu, instance.nics, node)
1060

    
1061

    
1062
def _CheckOSVariant(os_obj, name):
1063
  """Check whether an OS name conforms to the os variants specification.
1064

1065
  @type os_obj: L{objects.OS}
1066
  @param os_obj: OS object to check
1067
  @type name: string
1068
  @param name: OS name passed by the user, to check for validity
1069

1070
  """
1071
  if not os_obj.supported_variants:
1072
    return
1073
  variant = objects.OS.GetVariant(name)
1074
  if not variant:
1075
    raise errors.OpPrereqError("OS name must include a variant",
1076
                               errors.ECODE_INVAL)
1077

    
1078
  if variant not in os_obj.supported_variants:
1079
    raise errors.OpPrereqError("Unsupported OS variant", errors.ECODE_INVAL)
1080

    
1081

    
1082
def _GetNodeInstancesInner(cfg, fn):
1083
  return [i for i in cfg.GetAllInstancesInfo().values() if fn(i)]
1084

    
1085

    
1086
def _GetNodeInstances(cfg, node_name):
1087
  """Returns a list of all primary and secondary instances on a node.
1088

1089
  """
1090

    
1091
  return _GetNodeInstancesInner(cfg, lambda inst: node_name in inst.all_nodes)
1092

    
1093

    
1094
def _GetNodePrimaryInstances(cfg, node_name):
1095
  """Returns primary instances on a node.
1096

1097
  """
1098
  return _GetNodeInstancesInner(cfg,
1099
                                lambda inst: node_name == inst.primary_node)
1100

    
1101

    
1102
def _GetNodeSecondaryInstances(cfg, node_name):
1103
  """Returns secondary instances on a node.
1104

1105
  """
1106
  return _GetNodeInstancesInner(cfg,
1107
                                lambda inst: node_name in inst.secondary_nodes)
1108

    
1109

    
1110
def _GetStorageTypeArgs(cfg, storage_type):
1111
  """Returns the arguments for a storage type.
1112

1113
  """
1114
  # Special case for file storage
1115
  if storage_type == constants.ST_FILE:
1116
    # storage.FileStorage wants a list of storage directories
1117
    return [[cfg.GetFileStorageDir(), cfg.GetSharedFileStorageDir()]]
1118

    
1119
  return []
1120

    
1121

    
1122
def _FindFaultyInstanceDisks(cfg, rpc, instance, node_name, prereq):
1123
  faulty = []
1124

    
1125
  for dev in instance.disks:
1126
    cfg.SetDiskID(dev, node_name)
1127

    
1128
  result = rpc.call_blockdev_getmirrorstatus(node_name, instance.disks)
1129
  result.Raise("Failed to get disk status from node %s" % node_name,
1130
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
1131

    
1132
  for idx, bdev_status in enumerate(result.payload):
1133
    if bdev_status and bdev_status.ldisk_status == constants.LDS_FAULTY:
1134
      faulty.append(idx)
1135

    
1136
  return faulty
1137

    
1138

    
1139
def _CheckIAllocatorOrNode(lu, iallocator_slot, node_slot):
1140
  """Check the sanity of iallocator and node arguments and use the
1141
  cluster-wide iallocator if appropriate.
1142

1143
  Check that at most one of (iallocator, node) is specified. If none is
1144
  specified, then the LU's opcode's iallocator slot is filled with the
1145
  cluster-wide default iallocator.
1146

1147
  @type iallocator_slot: string
1148
  @param iallocator_slot: the name of the opcode iallocator slot
1149
  @type node_slot: string
1150
  @param node_slot: the name of the opcode target node slot
1151

1152
  """
1153
  node = getattr(lu.op, node_slot, None)
1154
  iallocator = getattr(lu.op, iallocator_slot, None)
1155

    
1156
  if node is not None and iallocator is not None:
1157
    raise errors.OpPrereqError("Do not specify both, iallocator and node",
1158
                               errors.ECODE_INVAL)
1159
  elif node is None and iallocator is None:
1160
    default_iallocator = lu.cfg.GetDefaultIAllocator()
1161
    if default_iallocator:
1162
      setattr(lu.op, iallocator_slot, default_iallocator)
1163
    else:
1164
      raise errors.OpPrereqError("No iallocator or node given and no"
1165
                                 " cluster-wide default iallocator found;"
1166
                                 " please specify either an iallocator or a"
1167
                                 " node, or set a cluster-wide default"
1168
                                 " iallocator")
1169

    
1170

    
1171
class LUClusterPostInit(LogicalUnit):
1172
  """Logical unit for running hooks after cluster initialization.
1173

1174
  """
1175
  HPATH = "cluster-init"
1176
  HTYPE = constants.HTYPE_CLUSTER
1177

    
1178
  def BuildHooksEnv(self):
1179
    """Build hooks env.
1180

1181
    """
1182
    return {
1183
      "OP_TARGET": self.cfg.GetClusterName(),
1184
      }
1185

    
1186
  def BuildHooksNodes(self):
1187
    """Build hooks nodes.
1188

1189
    """
1190
    return ([], [self.cfg.GetMasterNode()])
1191

    
1192
  def Exec(self, feedback_fn):
1193
    """Nothing to do.
1194

1195
    """
1196
    return True
1197

    
1198

    
1199
class LUClusterDestroy(LogicalUnit):
1200
  """Logical unit for destroying the cluster.
1201

1202
  """
1203
  HPATH = "cluster-destroy"
1204
  HTYPE = constants.HTYPE_CLUSTER
1205

    
1206
  def BuildHooksEnv(self):
1207
    """Build hooks env.
1208

1209
    """
1210
    return {
1211
      "OP_TARGET": self.cfg.GetClusterName(),
1212
      }
1213

    
1214
  def BuildHooksNodes(self):
1215
    """Build hooks nodes.
1216

1217
    """
1218
    return ([], [])
1219

    
1220
  def CheckPrereq(self):
1221
    """Check prerequisites.
1222

1223
    This checks whether the cluster is empty.
1224

1225
    Any errors are signaled by raising errors.OpPrereqError.
1226

1227
    """
1228
    master = self.cfg.GetMasterNode()
1229

    
1230
    nodelist = self.cfg.GetNodeList()
1231
    if len(nodelist) != 1 or nodelist[0] != master:
1232
      raise errors.OpPrereqError("There are still %d node(s) in"
1233
                                 " this cluster." % (len(nodelist) - 1),
1234
                                 errors.ECODE_INVAL)
1235
    instancelist = self.cfg.GetInstanceList()
1236
    if instancelist:
1237
      raise errors.OpPrereqError("There are still %d instance(s) in"
1238
                                 " this cluster." % len(instancelist),
1239
                                 errors.ECODE_INVAL)
1240

    
1241
  def Exec(self, feedback_fn):
1242
    """Destroys the cluster.
1243

1244
    """
1245
    master = self.cfg.GetMasterNode()
1246

    
1247
    # Run post hooks on master node before it's removed
1248
    _RunPostHook(self, master)
1249

    
1250
    result = self.rpc.call_node_stop_master(master, False)
1251
    result.Raise("Could not disable the master role")
1252

    
1253
    return master
1254

    
1255

    
1256
def _VerifyCertificate(filename):
1257
  """Verifies a certificate for LUClusterVerifyConfig.
1258

1259
  @type filename: string
1260
  @param filename: Path to PEM file
1261

1262
  """
1263
  try:
1264
    cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
1265
                                           utils.ReadFile(filename))
1266
  except Exception, err: # pylint: disable-msg=W0703
1267
    return (LUClusterVerifyConfig.ETYPE_ERROR,
1268
            "Failed to load X509 certificate %s: %s" % (filename, err))
1269

    
1270
  (errcode, msg) = \
1271
    utils.VerifyX509Certificate(cert, constants.SSL_CERT_EXPIRATION_WARN,
1272
                                constants.SSL_CERT_EXPIRATION_ERROR)
1273

    
1274
  if msg:
1275
    fnamemsg = "While verifying %s: %s" % (filename, msg)
1276
  else:
1277
    fnamemsg = None
1278

    
1279
  if errcode is None:
1280
    return (None, fnamemsg)
1281
  elif errcode == utils.CERT_WARNING:
1282
    return (LUClusterVerifyConfig.ETYPE_WARNING, fnamemsg)
1283
  elif errcode == utils.CERT_ERROR:
1284
    return (LUClusterVerifyConfig.ETYPE_ERROR, fnamemsg)
1285

    
1286
  raise errors.ProgrammerError("Unhandled certificate error code %r" % errcode)
1287

    
1288

    
1289
def _GetAllHypervisorParameters(cluster, instances):
1290
  """Compute the set of all hypervisor parameters.
1291

1292
  @type cluster: L{objects.Cluster}
1293
  @param cluster: the cluster object
1294
  @param instances: list of L{objects.Instance}
1295
  @param instances: additional instances from which to obtain parameters
1296
  @rtype: list of (origin, hypervisor, parameters)
1297
  @return: a list with all parameters found, indicating the hypervisor they
1298
       apply to, and the origin (can be "cluster", "os X", or "instance Y")
1299

1300
  """
1301
  hvp_data = []
1302

    
1303
  for hv_name in cluster.enabled_hypervisors:
1304
    hvp_data.append(("cluster", hv_name, cluster.GetHVDefaults(hv_name)))
1305

    
1306
  for os_name, os_hvp in cluster.os_hvp.items():
1307
    for hv_name, hv_params in os_hvp.items():
1308
      if hv_params:
1309
        full_params = cluster.GetHVDefaults(hv_name, os_name=os_name)
1310
        hvp_data.append(("os %s" % os_name, hv_name, full_params))
1311

    
1312
  # TODO: collapse identical parameter values in a single one
1313
  for instance in instances:
1314
    if instance.hvparams:
1315
      hvp_data.append(("instance %s" % instance.name, instance.hypervisor,
1316
                       cluster.FillHV(instance)))
1317

    
1318
  return hvp_data
1319

    
1320

    
1321
class _VerifyErrors(object):
1322
  """Mix-in for cluster/group verify LUs.
1323

1324
  It provides _Error and _ErrorIf, and updates the self.bad boolean. (Expects
1325
  self.op and self._feedback_fn to be available.)
1326

1327
  """
1328
  TCLUSTER = "cluster"
1329
  TNODE = "node"
1330
  TINSTANCE = "instance"
1331

    
1332
  ECLUSTERCFG = (TCLUSTER, "ECLUSTERCFG")
1333
  ECLUSTERCERT = (TCLUSTER, "ECLUSTERCERT")
1334
  ECLUSTERFILECHECK = (TCLUSTER, "ECLUSTERFILECHECK")
1335
  ECLUSTERDANGLINGNODES = (TNODE, "ECLUSTERDANGLINGNODES")
1336
  ECLUSTERDANGLINGINST = (TNODE, "ECLUSTERDANGLINGINST")
1337
  EINSTANCEBADNODE = (TINSTANCE, "EINSTANCEBADNODE")
1338
  EINSTANCEDOWN = (TINSTANCE, "EINSTANCEDOWN")
1339
  EINSTANCELAYOUT = (TINSTANCE, "EINSTANCELAYOUT")
1340
  EINSTANCEMISSINGDISK = (TINSTANCE, "EINSTANCEMISSINGDISK")
1341
  EINSTANCEFAULTYDISK = (TINSTANCE, "EINSTANCEFAULTYDISK")
1342
  EINSTANCEWRONGNODE = (TINSTANCE, "EINSTANCEWRONGNODE")
1343
  EINSTANCESPLITGROUPS = (TINSTANCE, "EINSTANCESPLITGROUPS")
1344
  ENODEDRBD = (TNODE, "ENODEDRBD")
1345
  ENODEDRBDHELPER = (TNODE, "ENODEDRBDHELPER")
1346
  ENODEFILECHECK = (TNODE, "ENODEFILECHECK")
1347
  ENODEHOOKS = (TNODE, "ENODEHOOKS")
1348
  ENODEHV = (TNODE, "ENODEHV")
1349
  ENODELVM = (TNODE, "ENODELVM")
1350
  ENODEN1 = (TNODE, "ENODEN1")
1351
  ENODENET = (TNODE, "ENODENET")
1352
  ENODEOS = (TNODE, "ENODEOS")
1353
  ENODEORPHANINSTANCE = (TNODE, "ENODEORPHANINSTANCE")
1354
  ENODEORPHANLV = (TNODE, "ENODEORPHANLV")
1355
  ENODERPC = (TNODE, "ENODERPC")
1356
  ENODESSH = (TNODE, "ENODESSH")
1357
  ENODEVERSION = (TNODE, "ENODEVERSION")
1358
  ENODESETUP = (TNODE, "ENODESETUP")
1359
  ENODETIME = (TNODE, "ENODETIME")
1360
  ENODEOOBPATH = (TNODE, "ENODEOOBPATH")
1361

    
1362
  ETYPE_FIELD = "code"
1363
  ETYPE_ERROR = "ERROR"
1364
  ETYPE_WARNING = "WARNING"
1365

    
1366
  def _Error(self, ecode, item, msg, *args, **kwargs):
1367
    """Format an error message.
1368

1369
    Based on the opcode's error_codes parameter, either format a
1370
    parseable error code, or a simpler error string.
1371

1372
    This must be called only from Exec and functions called from Exec.
1373

1374
    """
1375
    ltype = kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR)
1376
    itype, etxt = ecode
1377
    # first complete the msg
1378
    if args:
1379
      msg = msg % args
1380
    # then format the whole message
1381
    if self.op.error_codes: # This is a mix-in. pylint: disable-msg=E1101
1382
      msg = "%s:%s:%s:%s:%s" % (ltype, etxt, itype, item, msg)
1383
    else:
1384
      if item:
1385
        item = " " + item
1386
      else:
1387
        item = ""
1388
      msg = "%s: %s%s: %s" % (ltype, itype, item, msg)
1389
    # and finally report it via the feedback_fn
1390
    self._feedback_fn("  - %s" % msg) # Mix-in. pylint: disable-msg=E1101
1391

    
1392
  def _ErrorIf(self, cond, *args, **kwargs):
1393
    """Log an error message if the passed condition is True.
1394

1395
    """
1396
    cond = (bool(cond)
1397
            or self.op.debug_simulate_errors) # pylint: disable-msg=E1101
1398
    if cond:
1399
      self._Error(*args, **kwargs)
1400
    # do not mark the operation as failed for WARN cases only
1401
    if kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR) == self.ETYPE_ERROR:
1402
      self.bad = self.bad or cond
1403

    
1404

    
1405
class LUClusterVerifyConfig(NoHooksLU, _VerifyErrors):
1406
  """Verifies the cluster config.
1407

1408
  """
1409

    
1410
  REQ_BGL = False
1411

    
1412
  def _VerifyHVP(self, hvp_data):
1413
    """Verifies locally the syntax of the hypervisor parameters.
1414

1415
    """
1416
    for item, hv_name, hv_params in hvp_data:
1417
      msg = ("hypervisor %s parameters syntax check (source %s): %%s" %
1418
             (item, hv_name))
1419
      try:
1420
        hv_class = hypervisor.GetHypervisor(hv_name)
1421
        utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
1422
        hv_class.CheckParameterSyntax(hv_params)
1423
      except errors.GenericError, err:
1424
        self._ErrorIf(True, self.ECLUSTERCFG, None, msg % str(err))
1425

    
1426
  def ExpandNames(self):
1427
    self.all_group_info = self.cfg.GetAllNodeGroupsInfo()
1428
    self.all_node_info = self.cfg.GetAllNodesInfo()
1429
    self.all_inst_info = self.cfg.GetAllInstancesInfo()
1430
    self.needed_locks = {}
1431

    
1432
  def Exec(self, feedback_fn):
1433
    """Verify integrity of cluster, performing various test on nodes.
1434

1435
    """
1436
    self.bad = False
1437
    self._feedback_fn = feedback_fn
1438

    
1439
    feedback_fn("* Verifying cluster config")
1440

    
1441
    for msg in self.cfg.VerifyConfig():
1442
      self._ErrorIf(True, self.ECLUSTERCFG, None, msg)
1443

    
1444
    feedback_fn("* Verifying cluster certificate files")
1445

    
1446
    for cert_filename in constants.ALL_CERT_FILES:
1447
      (errcode, msg) = _VerifyCertificate(cert_filename)
1448
      self._ErrorIf(errcode, self.ECLUSTERCERT, None, msg, code=errcode)
1449

    
1450
    feedback_fn("* Verifying hypervisor parameters")
1451

    
1452
    self._VerifyHVP(_GetAllHypervisorParameters(self.cfg.GetClusterInfo(),
1453
                                                self.all_inst_info.values()))
1454

    
1455
    feedback_fn("* Verifying all nodes belong to an existing group")
1456

    
1457
    # We do this verification here because, should this bogus circumstance
1458
    # occur, it would never be catched by VerifyGroup, which only acts on
1459
    # nodes/instances reachable from existing node groups.
1460

    
1461
    dangling_nodes = set(node.name for node in self.all_node_info.values()
1462
                         if node.group not in self.all_group_info)
1463

    
1464
    dangling_instances = {}
1465
    no_node_instances = []
1466

    
1467
    for inst in self.all_inst_info.values():
1468
      if inst.primary_node in dangling_nodes:
1469
        dangling_instances.setdefault(inst.primary_node, []).append(inst.name)
1470
      elif inst.primary_node not in self.all_node_info:
1471
        no_node_instances.append(inst.name)
1472

    
1473
    pretty_dangling = [
1474
        "%s (%s)" %
1475
        (node.name,
1476
         utils.CommaJoin(dangling_instances.get(node.name,
1477
                                                ["no instances"])))
1478
        for node in dangling_nodes]
1479

    
1480
    self._ErrorIf(bool(dangling_nodes), self.ECLUSTERDANGLINGNODES, None,
1481
                  "the following nodes (and their instances) belong to a non"
1482
                  " existing group: %s", utils.CommaJoin(pretty_dangling))
1483

    
1484
    self._ErrorIf(bool(no_node_instances), self.ECLUSTERDANGLINGINST, None,
1485
                  "the following instances have a non-existing primary-node:"
1486
                  " %s", utils.CommaJoin(no_node_instances))
1487

    
1488
    return (not self.bad, [g.name for g in self.all_group_info.values()])
1489

    
1490

    
1491
class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors):
1492
  """Verifies the status of a node group.
1493

1494
  """
1495

    
1496
  HPATH = "cluster-verify"
1497
  HTYPE = constants.HTYPE_CLUSTER
1498
  REQ_BGL = False
1499

    
1500
  _HOOKS_INDENT_RE = re.compile("^", re.M)
1501

    
1502
  class NodeImage(object):
1503
    """A class representing the logical and physical status of a node.
1504

1505
    @type name: string
1506
    @ivar name: the node name to which this object refers
1507
    @ivar volumes: a structure as returned from
1508
        L{ganeti.backend.GetVolumeList} (runtime)
1509
    @ivar instances: a list of running instances (runtime)
1510
    @ivar pinst: list of configured primary instances (config)
1511
    @ivar sinst: list of configured secondary instances (config)
1512
    @ivar sbp: dictionary of {primary-node: list of instances} for all
1513
        instances for which this node is secondary (config)
1514
    @ivar mfree: free memory, as reported by hypervisor (runtime)
1515
    @ivar dfree: free disk, as reported by the node (runtime)
1516
    @ivar offline: the offline status (config)
1517
    @type rpc_fail: boolean
1518
    @ivar rpc_fail: whether the RPC verify call was successfull (overall,
1519
        not whether the individual keys were correct) (runtime)
1520
    @type lvm_fail: boolean
1521
    @ivar lvm_fail: whether the RPC call didn't return valid LVM data
1522
    @type hyp_fail: boolean
1523
    @ivar hyp_fail: whether the RPC call didn't return the instance list
1524
    @type ghost: boolean
1525
    @ivar ghost: whether this is a known node or not (config)
1526
    @type os_fail: boolean
1527
    @ivar os_fail: whether the RPC call didn't return valid OS data
1528
    @type oslist: list
1529
    @ivar oslist: list of OSes as diagnosed by DiagnoseOS
1530
    @type vm_capable: boolean
1531
    @ivar vm_capable: whether the node can host instances
1532

1533
    """
1534
    def __init__(self, offline=False, name=None, vm_capable=True):
1535
      self.name = name
1536
      self.volumes = {}
1537
      self.instances = []
1538
      self.pinst = []
1539
      self.sinst = []
1540
      self.sbp = {}
1541
      self.mfree = 0
1542
      self.dfree = 0
1543
      self.offline = offline
1544
      self.vm_capable = vm_capable
1545
      self.rpc_fail = False
1546
      self.lvm_fail = False
1547
      self.hyp_fail = False
1548
      self.ghost = False
1549
      self.os_fail = False
1550
      self.oslist = {}
1551

    
1552
  def ExpandNames(self):
1553
    # This raises errors.OpPrereqError on its own:
1554
    self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name)
1555

    
1556
    all_node_info = self.cfg.GetAllNodesInfo()
1557
    all_inst_info = self.cfg.GetAllInstancesInfo()
1558

    
1559
    node_names = set(node.name
1560
                     for node in all_node_info.values()
1561
                     if node.group == self.group_uuid)
1562

    
1563
    inst_names = [inst.name
1564
                  for inst in all_inst_info.values()
1565
                  if inst.primary_node in node_names]
1566

    
1567
    # In Exec(), we warn about mirrored instances that have primary and
1568
    # secondary living in separate node groups. To fully verify that
1569
    # volumes for these instances are healthy, we will need to do an
1570
    # extra call to their secondaries. We ensure here those nodes will
1571
    # be locked.
1572
    for inst in inst_names:
1573
      if all_inst_info[inst].disk_template in constants.DTS_INT_MIRROR:
1574
        node_names.update(all_inst_info[inst].secondary_nodes)
1575

    
1576
    self.needed_locks = {
1577
      locking.LEVEL_NODEGROUP: [self.group_uuid],
1578
      locking.LEVEL_NODE: list(node_names),
1579
      locking.LEVEL_INSTANCE: inst_names,
1580
    }
1581

    
1582
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
1583

    
1584
  def CheckPrereq(self):
1585
    self.all_node_info = self.cfg.GetAllNodesInfo()
1586
    self.all_inst_info = self.cfg.GetAllInstancesInfo()
1587

    
1588
    group_nodes = set(node.name
1589
                      for node in self.all_node_info.values()
1590
                      if node.group == self.group_uuid)
1591

    
1592
    group_instances = set(inst.name
1593
                          for inst in self.all_inst_info.values()
1594
                          if inst.primary_node in group_nodes)
1595

    
1596
    unlocked_nodes = \
1597
        group_nodes.difference(self.glm.list_owned(locking.LEVEL_NODE))
1598

    
1599
    unlocked_instances = \
1600
        group_instances.difference(self.glm.list_owned(locking.LEVEL_INSTANCE))
1601

    
1602
    if unlocked_nodes:
1603
      raise errors.OpPrereqError("missing lock for nodes: %s" %
1604
                                 utils.CommaJoin(unlocked_nodes))
1605

    
1606
    if unlocked_instances:
1607
      raise errors.OpPrereqError("missing lock for instances: %s" %
1608
                                 utils.CommaJoin(unlocked_instances))
1609

    
1610
    self.my_node_names = utils.NiceSort(group_nodes)
1611
    self.my_inst_names = utils.NiceSort(group_instances)
1612

    
1613
    self.my_node_info = dict((name, self.all_node_info[name])
1614
                             for name in self.my_node_names)
1615

    
1616
    self.my_inst_info = dict((name, self.all_inst_info[name])
1617
                             for name in self.my_inst_names)
1618

    
1619
    # We detect here the nodes that will need the extra RPC calls for verifying
1620
    # split LV volumes; they should be locked.
1621
    extra_lv_nodes = set()
1622

    
1623
    for inst in self.my_inst_info.values():
1624
      if inst.disk_template in constants.DTS_INT_MIRROR:
1625
        group = self.my_node_info[inst.primary_node].group
1626
        for nname in inst.secondary_nodes:
1627
          if self.all_node_info[nname].group != group:
1628
            extra_lv_nodes.add(nname)
1629

    
1630
    unlocked_lv_nodes = \
1631
        extra_lv_nodes.difference(self.glm.list_owned(locking.LEVEL_NODE))
1632

    
1633
    if unlocked_lv_nodes:
1634
      raise errors.OpPrereqError("these nodes could be locked: %s" %
1635
                                 utils.CommaJoin(unlocked_lv_nodes))
1636
    self.extra_lv_nodes = list(extra_lv_nodes)
1637

    
1638
  def _VerifyNode(self, ninfo, nresult):
1639
    """Perform some basic validation on data returned from a node.
1640

1641
      - check the result data structure is well formed and has all the
1642
        mandatory fields
1643
      - check ganeti version
1644

1645
    @type ninfo: L{objects.Node}
1646
    @param ninfo: the node to check
1647
    @param nresult: the results from the node
1648
    @rtype: boolean
1649
    @return: whether overall this call was successful (and we can expect
1650
         reasonable values in the respose)
1651

1652
    """
1653
    node = ninfo.name
1654
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1655

    
1656
    # main result, nresult should be a non-empty dict
1657
    test = not nresult or not isinstance(nresult, dict)
1658
    _ErrorIf(test, self.ENODERPC, node,
1659
                  "unable to verify node: no data returned")
1660
    if test:
1661
      return False
1662

    
1663
    # compares ganeti version
1664
    local_version = constants.PROTOCOL_VERSION
1665
    remote_version = nresult.get("version", None)
1666
    test = not (remote_version and
1667
                isinstance(remote_version, (list, tuple)) and
1668
                len(remote_version) == 2)
1669
    _ErrorIf(test, self.ENODERPC, node,
1670
             "connection to node returned invalid data")
1671
    if test:
1672
      return False
1673

    
1674
    test = local_version != remote_version[0]
1675
    _ErrorIf(test, self.ENODEVERSION, node,
1676
             "incompatible protocol versions: master %s,"
1677
             " node %s", local_version, remote_version[0])
1678
    if test:
1679
      return False
1680

    
1681
    # node seems compatible, we can actually try to look into its results
1682

    
1683
    # full package version
1684
    self._ErrorIf(constants.RELEASE_VERSION != remote_version[1],
1685
                  self.ENODEVERSION, node,
1686
                  "software version mismatch: master %s, node %s",
1687
                  constants.RELEASE_VERSION, remote_version[1],
1688
                  code=self.ETYPE_WARNING)
1689

    
1690
    hyp_result = nresult.get(constants.NV_HYPERVISOR, None)
1691
    if ninfo.vm_capable and isinstance(hyp_result, dict):
1692
      for hv_name, hv_result in hyp_result.iteritems():
1693
        test = hv_result is not None
1694
        _ErrorIf(test, self.ENODEHV, node,
1695
                 "hypervisor %s verify failure: '%s'", hv_name, hv_result)
1696

    
1697
    hvp_result = nresult.get(constants.NV_HVPARAMS, None)
1698
    if ninfo.vm_capable and isinstance(hvp_result, list):
1699
      for item, hv_name, hv_result in hvp_result:
1700
        _ErrorIf(True, self.ENODEHV, node,
1701
                 "hypervisor %s parameter verify failure (source %s): %s",
1702
                 hv_name, item, hv_result)
1703

    
1704
    test = nresult.get(constants.NV_NODESETUP,
1705
                       ["Missing NODESETUP results"])
1706
    _ErrorIf(test, self.ENODESETUP, node, "node setup error: %s",
1707
             "; ".join(test))
1708

    
1709
    return True
1710

    
1711
  def _VerifyNodeTime(self, ninfo, nresult,
1712
                      nvinfo_starttime, nvinfo_endtime):
1713
    """Check the node time.
1714

1715
    @type ninfo: L{objects.Node}
1716
    @param ninfo: the node to check
1717
    @param nresult: the remote results for the node
1718
    @param nvinfo_starttime: the start time of the RPC call
1719
    @param nvinfo_endtime: the end time of the RPC call
1720

1721
    """
1722
    node = ninfo.name
1723
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1724

    
1725
    ntime = nresult.get(constants.NV_TIME, None)
1726
    try:
1727
      ntime_merged = utils.MergeTime(ntime)
1728
    except (ValueError, TypeError):
1729
      _ErrorIf(True, self.ENODETIME, node, "Node returned invalid time")
1730
      return
1731

    
1732
    if ntime_merged < (nvinfo_starttime - constants.NODE_MAX_CLOCK_SKEW):
1733
      ntime_diff = "%.01fs" % abs(nvinfo_starttime - ntime_merged)
1734
    elif ntime_merged > (nvinfo_endtime + constants.NODE_MAX_CLOCK_SKEW):
1735
      ntime_diff = "%.01fs" % abs(ntime_merged - nvinfo_endtime)
1736
    else:
1737
      ntime_diff = None
1738

    
1739
    _ErrorIf(ntime_diff is not None, self.ENODETIME, node,
1740
             "Node time diverges by at least %s from master node time",
1741
             ntime_diff)
1742

    
1743
  def _VerifyNodeLVM(self, ninfo, nresult, vg_name):
1744
    """Check the node LVM results.
1745

1746
    @type ninfo: L{objects.Node}
1747
    @param ninfo: the node to check
1748
    @param nresult: the remote results for the node
1749
    @param vg_name: the configured VG name
1750

1751
    """
1752
    if vg_name is None:
1753
      return
1754

    
1755
    node = ninfo.name
1756
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1757

    
1758
    # checks vg existence and size > 20G
1759
    vglist = nresult.get(constants.NV_VGLIST, None)
1760
    test = not vglist
1761
    _ErrorIf(test, self.ENODELVM, node, "unable to check volume groups")
1762
    if not test:
1763
      vgstatus = utils.CheckVolumeGroupSize(vglist, vg_name,
1764
                                            constants.MIN_VG_SIZE)
1765
      _ErrorIf(vgstatus, self.ENODELVM, node, vgstatus)
1766

    
1767
    # check pv names
1768
    pvlist = nresult.get(constants.NV_PVLIST, None)
1769
    test = pvlist is None
1770
    _ErrorIf(test, self.ENODELVM, node, "Can't get PV list from node")
1771
    if not test:
1772
      # check that ':' is not present in PV names, since it's a
1773
      # special character for lvcreate (denotes the range of PEs to
1774
      # use on the PV)
1775
      for _, pvname, owner_vg in pvlist:
1776
        test = ":" in pvname
1777
        _ErrorIf(test, self.ENODELVM, node, "Invalid character ':' in PV"
1778
                 " '%s' of VG '%s'", pvname, owner_vg)
1779

    
1780
  def _VerifyNodeBridges(self, ninfo, nresult, bridges):
1781
    """Check the node bridges.
1782

1783
    @type ninfo: L{objects.Node}
1784
    @param ninfo: the node to check
1785
    @param nresult: the remote results for the node
1786
    @param bridges: the expected list of bridges
1787

1788
    """
1789
    if not bridges:
1790
      return
1791

    
1792
    node = ninfo.name
1793
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1794

    
1795
    missing = nresult.get(constants.NV_BRIDGES, None)
1796
    test = not isinstance(missing, list)
1797
    _ErrorIf(test, self.ENODENET, node,
1798
             "did not return valid bridge information")
1799
    if not test:
1800
      _ErrorIf(bool(missing), self.ENODENET, node, "missing bridges: %s" %
1801
               utils.CommaJoin(sorted(missing)))
1802

    
1803
  def _VerifyNodeNetwork(self, ninfo, nresult):
1804
    """Check the node network connectivity results.
1805

1806
    @type ninfo: L{objects.Node}
1807
    @param ninfo: the node to check
1808
    @param nresult: the remote results for the node
1809

1810
    """
1811
    node = ninfo.name
1812
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1813

    
1814
    test = constants.NV_NODELIST not in nresult
1815
    _ErrorIf(test, self.ENODESSH, node,
1816
             "node hasn't returned node ssh connectivity data")
1817
    if not test:
1818
      if nresult[constants.NV_NODELIST]:
1819
        for a_node, a_msg in nresult[constants.NV_NODELIST].items():
1820
          _ErrorIf(True, self.ENODESSH, node,
1821
                   "ssh communication with node '%s': %s", a_node, a_msg)
1822

    
1823
    test = constants.NV_NODENETTEST not in nresult
1824
    _ErrorIf(test, self.ENODENET, node,
1825
             "node hasn't returned node tcp connectivity data")
1826
    if not test:
1827
      if nresult[constants.NV_NODENETTEST]:
1828
        nlist = utils.NiceSort(nresult[constants.NV_NODENETTEST].keys())
1829
        for anode in nlist:
1830
          _ErrorIf(True, self.ENODENET, node,
1831
                   "tcp communication with node '%s': %s",
1832
                   anode, nresult[constants.NV_NODENETTEST][anode])
1833

    
1834
    test = constants.NV_MASTERIP not in nresult
1835
    _ErrorIf(test, self.ENODENET, node,
1836
             "node hasn't returned node master IP reachability data")
1837
    if not test:
1838
      if not nresult[constants.NV_MASTERIP]:
1839
        if node == self.master_node:
1840
          msg = "the master node cannot reach the master IP (not configured?)"
1841
        else:
1842
          msg = "cannot reach the master IP"
1843
        _ErrorIf(True, self.ENODENET, node, msg)
1844

    
1845
  def _VerifyInstance(self, instance, instanceconfig, node_image,
1846
                      diskstatus):
1847
    """Verify an instance.
1848

1849
    This function checks to see if the required block devices are
1850
    available on the instance's node.
1851

1852
    """
1853
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1854
    node_current = instanceconfig.primary_node
1855

    
1856
    node_vol_should = {}
1857
    instanceconfig.MapLVsByNode(node_vol_should)
1858

    
1859
    for node in node_vol_should:
1860
      n_img = node_image[node]
1861
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1862
        # ignore missing volumes on offline or broken nodes
1863
        continue
1864
      for volume in node_vol_should[node]:
1865
        test = volume not in n_img.volumes
1866
        _ErrorIf(test, self.EINSTANCEMISSINGDISK, instance,
1867
                 "volume %s missing on node %s", volume, node)
1868

    
1869
    if instanceconfig.admin_up:
1870
      pri_img = node_image[node_current]
1871
      test = instance not in pri_img.instances and not pri_img.offline
1872
      _ErrorIf(test, self.EINSTANCEDOWN, instance,
1873
               "instance not running on its primary node %s",
1874
               node_current)
1875

    
1876
    diskdata = [(nname, success, status, idx)
1877
                for (nname, disks) in diskstatus.items()
1878
                for idx, (success, status) in enumerate(disks)]
1879

    
1880
    for nname, success, bdev_status, idx in diskdata:
1881
      # the 'ghost node' construction in Exec() ensures that we have a
1882
      # node here
1883
      snode = node_image[nname]
1884
      bad_snode = snode.ghost or snode.offline
1885
      _ErrorIf(instanceconfig.admin_up and not success and not bad_snode,
1886
               self.EINSTANCEFAULTYDISK, instance,
1887
               "couldn't retrieve status for disk/%s on %s: %s",
1888
               idx, nname, bdev_status)
1889
      _ErrorIf((instanceconfig.admin_up and success and
1890
                bdev_status.ldisk_status == constants.LDS_FAULTY),
1891
               self.EINSTANCEFAULTYDISK, instance,
1892
               "disk/%s on %s is faulty", idx, nname)
1893

    
1894
  def _VerifyOrphanVolumes(self, node_vol_should, node_image, reserved):
1895
    """Verify if there are any unknown volumes in the cluster.
1896

1897
    The .os, .swap and backup volumes are ignored. All other volumes are
1898
    reported as unknown.
1899

1900
    @type reserved: L{ganeti.utils.FieldSet}
1901
    @param reserved: a FieldSet of reserved volume names
1902

1903
    """
1904
    for node, n_img in node_image.items():
1905
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1906
        # skip non-healthy nodes
1907
        continue
1908
      for volume in n_img.volumes:
1909
        test = ((node not in node_vol_should or
1910
                volume not in node_vol_should[node]) and
1911
                not reserved.Matches(volume))
1912
        self._ErrorIf(test, self.ENODEORPHANLV, node,
1913
                      "volume %s is unknown", volume)
1914

    
1915
  def _VerifyNPlusOneMemory(self, node_image, instance_cfg):
1916
    """Verify N+1 Memory Resilience.
1917

1918
    Check that if one single node dies we can still start all the
1919
    instances it was primary for.
1920

1921
    """
1922
    cluster_info = self.cfg.GetClusterInfo()
1923
    for node, n_img in node_image.items():
1924
      # This code checks that every node which is now listed as
1925
      # secondary has enough memory to host all instances it is
1926
      # supposed to should a single other node in the cluster fail.
1927
      # FIXME: not ready for failover to an arbitrary node
1928
      # FIXME: does not support file-backed instances
1929
      # WARNING: we currently take into account down instances as well
1930
      # as up ones, considering that even if they're down someone
1931
      # might want to start them even in the event of a node failure.
1932
      if n_img.offline:
1933
        # we're skipping offline nodes from the N+1 warning, since
1934
        # most likely we don't have good memory infromation from them;
1935
        # we already list instances living on such nodes, and that's
1936
        # enough warning
1937
        continue
1938
      for prinode, instances in n_img.sbp.items():
1939
        needed_mem = 0
1940
        for instance in instances:
1941
          bep = cluster_info.FillBE(instance_cfg[instance])
1942
          if bep[constants.BE_AUTO_BALANCE]:
1943
            needed_mem += bep[constants.BE_MEMORY]
1944
        test = n_img.mfree < needed_mem
1945
        self._ErrorIf(test, self.ENODEN1, node,
1946
                      "not enough memory to accomodate instance failovers"
1947
                      " should node %s fail (%dMiB needed, %dMiB available)",
1948
                      prinode, needed_mem, n_img.mfree)
1949

    
1950
  @classmethod
1951
  def _VerifyFiles(cls, errorif, nodeinfo, master_node, all_nvinfo,
1952
                   (files_all, files_all_opt, files_mc, files_vm)):
1953
    """Verifies file checksums collected from all nodes.
1954

1955
    @param errorif: Callback for reporting errors
1956
    @param nodeinfo: List of L{objects.Node} objects
1957
    @param master_node: Name of master node
1958
    @param all_nvinfo: RPC results
1959

1960
    """
1961
    node_names = frozenset(node.name for node in nodeinfo)
1962

    
1963
    assert master_node in node_names
1964
    assert (len(files_all | files_all_opt | files_mc | files_vm) ==
1965
            sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
1966
           "Found file listed in more than one file list"
1967

    
1968
    # Define functions determining which nodes to consider for a file
1969
    file2nodefn = dict([(filename, fn)
1970
      for (files, fn) in [(files_all, None),
1971
                          (files_all_opt, None),
1972
                          (files_mc, lambda node: (node.master_candidate or
1973
                                                   node.name == master_node)),
1974
                          (files_vm, lambda node: node.vm_capable)]
1975
      for filename in files])
1976

    
1977
    fileinfo = dict((filename, {}) for filename in file2nodefn.keys())
1978

    
1979
    for node in nodeinfo:
1980
      nresult = all_nvinfo[node.name]
1981

    
1982
      if nresult.fail_msg or not nresult.payload:
1983
        node_files = None
1984
      else:
1985
        node_files = nresult.payload.get(constants.NV_FILELIST, None)
1986

    
1987
      test = not (node_files and isinstance(node_files, dict))
1988
      errorif(test, cls.ENODEFILECHECK, node.name,
1989
              "Node did not return file checksum data")
1990
      if test:
1991
        continue
1992

    
1993
      for (filename, checksum) in node_files.items():
1994
        # Check if the file should be considered for a node
1995
        fn = file2nodefn[filename]
1996
        if fn is None or fn(node):
1997
          fileinfo[filename].setdefault(checksum, set()).add(node.name)
1998

    
1999
    for (filename, checksums) in fileinfo.items():
2000
      assert compat.all(len(i) > 10 for i in checksums), "Invalid checksum"
2001

    
2002
      # Nodes having the file
2003
      with_file = frozenset(node_name
2004
                            for nodes in fileinfo[filename].values()
2005
                            for node_name in nodes)
2006

    
2007
      # Nodes missing file
2008
      missing_file = node_names - with_file
2009

    
2010
      if filename in files_all_opt:
2011
        # All or no nodes
2012
        errorif(missing_file and missing_file != node_names,
2013
                cls.ECLUSTERFILECHECK, None,
2014
                "File %s is optional, but it must exist on all or no nodes (not"
2015
                " found on %s)",
2016
                filename, utils.CommaJoin(utils.NiceSort(missing_file)))
2017
      else:
2018
        errorif(missing_file, cls.ECLUSTERFILECHECK, None,
2019
                "File %s is missing from node(s) %s", filename,
2020
                utils.CommaJoin(utils.NiceSort(missing_file)))
2021

    
2022
      # See if there are multiple versions of the file
2023
      test = len(checksums) > 1
2024
      if test:
2025
        variants = ["variant %s on %s" %
2026
                    (idx + 1, utils.CommaJoin(utils.NiceSort(nodes)))
2027
                    for (idx, (checksum, nodes)) in
2028
                      enumerate(sorted(checksums.items()))]
2029
      else:
2030
        variants = []
2031

    
2032
      errorif(test, cls.ECLUSTERFILECHECK, None,
2033
              "File %s found with %s different checksums (%s)",
2034
              filename, len(checksums), "; ".join(variants))
2035

    
2036
  def _VerifyNodeDrbd(self, ninfo, nresult, instanceinfo, drbd_helper,
2037
                      drbd_map):
2038
    """Verifies and the node DRBD status.
2039

2040
    @type ninfo: L{objects.Node}
2041
    @param ninfo: the node to check
2042
    @param nresult: the remote results for the node
2043
    @param instanceinfo: the dict of instances
2044
    @param drbd_helper: the configured DRBD usermode helper
2045
    @param drbd_map: the DRBD map as returned by
2046
        L{ganeti.config.ConfigWriter.ComputeDRBDMap}
2047

2048
    """
2049
    node = ninfo.name
2050
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2051

    
2052
    if drbd_helper:
2053
      helper_result = nresult.get(constants.NV_DRBDHELPER, None)
2054
      test = (helper_result == None)
2055
      _ErrorIf(test, self.ENODEDRBDHELPER, node,
2056
               "no drbd usermode helper returned")
2057
      if helper_result:
2058
        status, payload = helper_result
2059
        test = not status
2060
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
2061
                 "drbd usermode helper check unsuccessful: %s", payload)
2062
        test = status and (payload != drbd_helper)
2063
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
2064
                 "wrong drbd usermode helper: %s", payload)
2065

    
2066
    # compute the DRBD minors
2067
    node_drbd = {}
2068
    for minor, instance in drbd_map[node].items():
2069
      test = instance not in instanceinfo
2070
      _ErrorIf(test, self.ECLUSTERCFG, None,
2071
               "ghost instance '%s' in temporary DRBD map", instance)
2072
        # ghost instance should not be running, but otherwise we
2073
        # don't give double warnings (both ghost instance and
2074
        # unallocated minor in use)
2075
      if test:
2076
        node_drbd[minor] = (instance, False)
2077
      else:
2078
        instance = instanceinfo[instance]
2079
        node_drbd[minor] = (instance.name, instance.admin_up)
2080

    
2081
    # and now check them
2082
    used_minors = nresult.get(constants.NV_DRBDLIST, [])
2083
    test = not isinstance(used_minors, (tuple, list))
2084
    _ErrorIf(test, self.ENODEDRBD, node,
2085
             "cannot parse drbd status file: %s", str(used_minors))
2086
    if test:
2087
      # we cannot check drbd status
2088
      return
2089

    
2090
    for minor, (iname, must_exist) in node_drbd.items():
2091
      test = minor not in used_minors and must_exist
2092
      _ErrorIf(test, self.ENODEDRBD, node,
2093
               "drbd minor %d of instance %s is not active", minor, iname)
2094
    for minor in used_minors:
2095
      test = minor not in node_drbd
2096
      _ErrorIf(test, self.ENODEDRBD, node,
2097
               "unallocated drbd minor %d is in use", minor)
2098

    
2099
  def _UpdateNodeOS(self, ninfo, nresult, nimg):
2100
    """Builds the node OS structures.
2101

2102
    @type ninfo: L{objects.Node}
2103
    @param ninfo: the node to check
2104
    @param nresult: the remote results for the node
2105
    @param nimg: the node image object
2106

2107
    """
2108
    node = ninfo.name
2109
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2110

    
2111
    remote_os = nresult.get(constants.NV_OSLIST, None)
2112
    test = (not isinstance(remote_os, list) or
2113
            not compat.all(isinstance(v, list) and len(v) == 7
2114
                           for v in remote_os))
2115

    
2116
    _ErrorIf(test, self.ENODEOS, node,
2117
             "node hasn't returned valid OS data")
2118

    
2119
    nimg.os_fail = test
2120

    
2121
    if test:
2122
      return
2123

    
2124
    os_dict = {}
2125

    
2126
    for (name, os_path, status, diagnose,
2127
         variants, parameters, api_ver) in nresult[constants.NV_OSLIST]:
2128

    
2129
      if name not in os_dict:
2130
        os_dict[name] = []
2131

    
2132
      # parameters is a list of lists instead of list of tuples due to
2133
      # JSON lacking a real tuple type, fix it:
2134
      parameters = [tuple(v) for v in parameters]
2135
      os_dict[name].append((os_path, status, diagnose,
2136
                            set(variants), set(parameters), set(api_ver)))
2137

    
2138
    nimg.oslist = os_dict
2139

    
2140
  def _VerifyNodeOS(self, ninfo, nimg, base):
2141
    """Verifies the node OS list.
2142

2143
    @type ninfo: L{objects.Node}
2144
    @param ninfo: the node to check
2145
    @param nimg: the node image object
2146
    @param base: the 'template' node we match against (e.g. from the master)
2147

2148
    """
2149
    node = ninfo.name
2150
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2151

    
2152
    assert not nimg.os_fail, "Entered _VerifyNodeOS with failed OS rpc?"
2153

    
2154
    beautify_params = lambda l: ["%s: %s" % (k, v) for (k, v) in l]
2155
    for os_name, os_data in nimg.oslist.items():
2156
      assert os_data, "Empty OS status for OS %s?!" % os_name
2157
      f_path, f_status, f_diag, f_var, f_param, f_api = os_data[0]
2158
      _ErrorIf(not f_status, self.ENODEOS, node,
2159
               "Invalid OS %s (located at %s): %s", os_name, f_path, f_diag)
2160
      _ErrorIf(len(os_data) > 1, self.ENODEOS, node,
2161
               "OS '%s' has multiple entries (first one shadows the rest): %s",
2162
               os_name, utils.CommaJoin([v[0] for v in os_data]))
2163
      # this will catched in backend too
2164
      _ErrorIf(compat.any(v >= constants.OS_API_V15 for v in f_api)
2165
               and not f_var, self.ENODEOS, node,
2166
               "OS %s with API at least %d does not declare any variant",
2167
               os_name, constants.OS_API_V15)
2168
      # comparisons with the 'base' image
2169
      test = os_name not in base.oslist
2170
      _ErrorIf(test, self.ENODEOS, node,
2171
               "Extra OS %s not present on reference node (%s)",
2172
               os_name, base.name)
2173
      if test:
2174
        continue
2175
      assert base.oslist[os_name], "Base node has empty OS status?"
2176
      _, b_status, _, b_var, b_param, b_api = base.oslist[os_name][0]
2177
      if not b_status:
2178
        # base OS is invalid, skipping
2179
        continue
2180
      for kind, a, b in [("API version", f_api, b_api),
2181
                         ("variants list", f_var, b_var),
2182
                         ("parameters", beautify_params(f_param),
2183
                          beautify_params(b_param))]:
2184
        _ErrorIf(a != b, self.ENODEOS, node,
2185
                 "OS %s for %s differs from reference node %s: [%s] vs. [%s]",
2186
                 kind, os_name, base.name,
2187
                 utils.CommaJoin(sorted(a)), utils.CommaJoin(sorted(b)))
2188

    
2189
    # check any missing OSes
2190
    missing = set(base.oslist.keys()).difference(nimg.oslist.keys())
2191
    _ErrorIf(missing, self.ENODEOS, node,
2192
             "OSes present on reference node %s but missing on this node: %s",
2193
             base.name, utils.CommaJoin(missing))
2194

    
2195
  def _VerifyOob(self, ninfo, nresult):
2196
    """Verifies out of band functionality of a node.
2197

2198
    @type ninfo: L{objects.Node}
2199
    @param ninfo: the node to check
2200
    @param nresult: the remote results for the node
2201

2202
    """
2203
    node = ninfo.name
2204
    # We just have to verify the paths on master and/or master candidates
2205
    # as the oob helper is invoked on the master
2206
    if ((ninfo.master_candidate or ninfo.master_capable) and
2207
        constants.NV_OOB_PATHS in nresult):
2208
      for path_result in nresult[constants.NV_OOB_PATHS]:
2209
        self._ErrorIf(path_result, self.ENODEOOBPATH, node, path_result)
2210

    
2211
  def _UpdateNodeVolumes(self, ninfo, nresult, nimg, vg_name):
2212
    """Verifies and updates the node volume data.
2213

2214
    This function will update a L{NodeImage}'s internal structures
2215
    with data from the remote call.
2216

2217
    @type ninfo: L{objects.Node}
2218
    @param ninfo: the node to check
2219
    @param nresult: the remote results for the node
2220
    @param nimg: the node image object
2221
    @param vg_name: the configured VG name
2222

2223
    """
2224
    node = ninfo.name
2225
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2226

    
2227
    nimg.lvm_fail = True
2228
    lvdata = nresult.get(constants.NV_LVLIST, "Missing LV data")
2229
    if vg_name is None:
2230
      pass
2231
    elif isinstance(lvdata, basestring):
2232
      _ErrorIf(True, self.ENODELVM, node, "LVM problem on node: %s",
2233
               utils.SafeEncode(lvdata))
2234
    elif not isinstance(lvdata, dict):
2235
      _ErrorIf(True, self.ENODELVM, node, "rpc call to node failed (lvlist)")
2236
    else:
2237
      nimg.volumes = lvdata
2238
      nimg.lvm_fail = False
2239

    
2240
  def _UpdateNodeInstances(self, ninfo, nresult, nimg):
2241
    """Verifies and updates the node instance list.
2242

2243
    If the listing was successful, then updates this node's instance
2244
    list. Otherwise, it marks the RPC call as failed for the instance
2245
    list key.
2246

2247
    @type ninfo: L{objects.Node}
2248
    @param ninfo: the node to check
2249
    @param nresult: the remote results for the node
2250
    @param nimg: the node image object
2251

2252
    """
2253
    idata = nresult.get(constants.NV_INSTANCELIST, None)
2254
    test = not isinstance(idata, list)
2255
    self._ErrorIf(test, self.ENODEHV, ninfo.name, "rpc call to node failed"
2256
                  " (instancelist): %s", utils.SafeEncode(str(idata)))
2257
    if test:
2258
      nimg.hyp_fail = True
2259
    else:
2260
      nimg.instances = idata
2261

    
2262
  def _UpdateNodeInfo(self, ninfo, nresult, nimg, vg_name):
2263
    """Verifies and computes a node information map
2264

2265
    @type ninfo: L{objects.Node}
2266
    @param ninfo: the node to check
2267
    @param nresult: the remote results for the node
2268
    @param nimg: the node image object
2269
    @param vg_name: the configured VG name
2270

2271
    """
2272
    node = ninfo.name
2273
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2274

    
2275
    # try to read free memory (from the hypervisor)
2276
    hv_info = nresult.get(constants.NV_HVINFO, None)
2277
    test = not isinstance(hv_info, dict) or "memory_free" not in hv_info
2278
    _ErrorIf(test, self.ENODEHV, node, "rpc call to node failed (hvinfo)")
2279
    if not test:
2280
      try:
2281
        nimg.mfree = int(hv_info["memory_free"])
2282
      except (ValueError, TypeError):
2283
        _ErrorIf(True, self.ENODERPC, node,
2284
                 "node returned invalid nodeinfo, check hypervisor")
2285

    
2286
    # FIXME: devise a free space model for file based instances as well
2287
    if vg_name is not None:
2288
      test = (constants.NV_VGLIST not in nresult or
2289
              vg_name not in nresult[constants.NV_VGLIST])
2290
      _ErrorIf(test, self.ENODELVM, node,
2291
               "node didn't return data for the volume group '%s'"
2292
               " - it is either missing or broken", vg_name)
2293
      if not test:
2294
        try:
2295
          nimg.dfree = int(nresult[constants.NV_VGLIST][vg_name])
2296
        except (ValueError, TypeError):
2297
          _ErrorIf(True, self.ENODERPC, node,
2298
                   "node returned invalid LVM info, check LVM status")
2299

    
2300
  def _CollectDiskInfo(self, nodelist, node_image, instanceinfo):
2301
    """Gets per-disk status information for all instances.
2302

2303
    @type nodelist: list of strings
2304
    @param nodelist: Node names
2305
    @type node_image: dict of (name, L{objects.Node})
2306
    @param node_image: Node objects
2307
    @type instanceinfo: dict of (name, L{objects.Instance})
2308
    @param instanceinfo: Instance objects
2309
    @rtype: {instance: {node: [(succes, payload)]}}
2310
    @return: a dictionary of per-instance dictionaries with nodes as
2311
        keys and disk information as values; the disk information is a
2312
        list of tuples (success, payload)
2313

2314
    """
2315
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2316

    
2317
    node_disks = {}
2318
    node_disks_devonly = {}
2319
    diskless_instances = set()
2320
    diskless = constants.DT_DISKLESS
2321

    
2322
    for nname in nodelist:
2323
      node_instances = list(itertools.chain(node_image[nname].pinst,
2324
                                            node_image[nname].sinst))
2325
      diskless_instances.update(inst for inst in node_instances
2326
                                if instanceinfo[inst].disk_template == diskless)
2327
      disks = [(inst, disk)
2328
               for inst in node_instances
2329
               for disk in instanceinfo[inst].disks]
2330

    
2331
      if not disks:
2332
        # No need to collect data
2333
        continue
2334

    
2335
      node_disks[nname] = disks
2336

    
2337
      # Creating copies as SetDiskID below will modify the objects and that can
2338
      # lead to incorrect data returned from nodes
2339
      devonly = [dev.Copy() for (_, dev) in disks]
2340

    
2341
      for dev in devonly:
2342
        self.cfg.SetDiskID(dev, nname)
2343

    
2344
      node_disks_devonly[nname] = devonly
2345

    
2346
    assert len(node_disks) == len(node_disks_devonly)
2347

    
2348
    # Collect data from all nodes with disks
2349
    result = self.rpc.call_blockdev_getmirrorstatus_multi(node_disks.keys(),
2350
                                                          node_disks_devonly)
2351

    
2352
    assert len(result) == len(node_disks)
2353

    
2354
    instdisk = {}
2355

    
2356
    for (nname, nres) in result.items():
2357
      disks = node_disks[nname]
2358

    
2359
      if nres.offline:
2360
        # No data from this node
2361
        data = len(disks) * [(False, "node offline")]
2362
      else:
2363
        msg = nres.fail_msg
2364
        _ErrorIf(msg, self.ENODERPC, nname,
2365
                 "while getting disk information: %s", msg)
2366
        if msg:
2367
          # No data from this node
2368
          data = len(disks) * [(False, msg)]
2369
        else:
2370
          data = []
2371
          for idx, i in enumerate(nres.payload):
2372
            if isinstance(i, (tuple, list)) and len(i) == 2:
2373
              data.append(i)
2374
            else:
2375
              logging.warning("Invalid result from node %s, entry %d: %s",
2376
                              nname, idx, i)
2377
              data.append((False, "Invalid result from the remote node"))
2378

    
2379
      for ((inst, _), status) in zip(disks, data):
2380
        instdisk.setdefault(inst, {}).setdefault(nname, []).append(status)
2381

    
2382
    # Add empty entries for diskless instances.
2383
    for inst in diskless_instances:
2384
      assert inst not in instdisk
2385
      instdisk[inst] = {}
2386

    
2387
    assert compat.all(len(statuses) == len(instanceinfo[inst].disks) and
2388
                      len(nnames) <= len(instanceinfo[inst].all_nodes) and
2389
                      compat.all(isinstance(s, (tuple, list)) and
2390
                                 len(s) == 2 for s in statuses)
2391
                      for inst, nnames in instdisk.items()
2392
                      for nname, statuses in nnames.items())
2393
    assert set(instdisk) == set(instanceinfo), "instdisk consistency failure"
2394

    
2395
    return instdisk
2396

    
2397
  def BuildHooksEnv(self):
2398
    """Build hooks env.
2399

2400
    Cluster-Verify hooks just ran in the post phase and their failure makes
2401
    the output be logged in the verify output and the verification to fail.
2402

2403
    """
2404
    env = {
2405
      "CLUSTER_TAGS": " ".join(self.cfg.GetClusterInfo().GetTags())
2406
      }
2407

    
2408
    env.update(("NODE_TAGS_%s" % node.name, " ".join(node.GetTags()))
2409
               for node in self.my_node_info.values())
2410

    
2411
    return env
2412

    
2413
  def BuildHooksNodes(self):
2414
    """Build hooks nodes.
2415

2416
    """
2417
    assert self.my_node_names, ("Node list not gathered,"
2418
      " has CheckPrereq been executed?")
2419
    return ([], self.my_node_names)
2420

    
2421
  def Exec(self, feedback_fn):
2422
    """Verify integrity of the node group, performing various test on nodes.
2423

2424
    """
2425
    # This method has too many local variables. pylint: disable-msg=R0914
2426
    self.bad = False
2427
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2428
    verbose = self.op.verbose
2429
    self._feedback_fn = feedback_fn
2430

    
2431
    vg_name = self.cfg.GetVGName()
2432
    drbd_helper = self.cfg.GetDRBDHelper()
2433
    cluster = self.cfg.GetClusterInfo()
2434
    groupinfo = self.cfg.GetAllNodeGroupsInfo()
2435
    hypervisors = cluster.enabled_hypervisors
2436
    node_data_list = [self.my_node_info[name] for name in self.my_node_names]
2437

    
2438
    i_non_redundant = [] # Non redundant instances
2439
    i_non_a_balanced = [] # Non auto-balanced instances
2440
    n_offline = 0 # Count of offline nodes
2441
    n_drained = 0 # Count of nodes being drained
2442
    node_vol_should = {}
2443

    
2444
    # FIXME: verify OS list
2445

    
2446
    # File verification
2447
    filemap = _ComputeAncillaryFiles(cluster, False)
2448

    
2449
    # do local checksums
2450
    master_node = self.master_node = self.cfg.GetMasterNode()
2451
    master_ip = self.cfg.GetMasterIP()
2452

    
2453
    feedback_fn("* Gathering data (%d nodes)" % len(self.my_node_names))
2454

    
2455
    # We will make nodes contact all nodes in their group, and one node from
2456
    # every other group.
2457
    # TODO: should it be a *random* node, different every time?
2458
    online_nodes = [node.name for node in node_data_list if not node.offline]
2459
    other_group_nodes = {}
2460

    
2461
    for name in sorted(self.all_node_info):
2462
      node = self.all_node_info[name]
2463
      if (node.group not in other_group_nodes
2464
          and node.group != self.group_uuid
2465
          and not node.offline):
2466
        other_group_nodes[node.group] = node.name
2467

    
2468
    node_verify_param = {
2469
      constants.NV_FILELIST:
2470
        utils.UniqueSequence(filename
2471
                             for files in filemap
2472
                             for filename in files),
2473
      constants.NV_NODELIST: online_nodes + other_group_nodes.values(),
2474
      constants.NV_HYPERVISOR: hypervisors,
2475
      constants.NV_HVPARAMS:
2476
        _GetAllHypervisorParameters(cluster, self.all_inst_info.values()),
2477
      constants.NV_NODENETTEST: [(node.name, node.primary_ip, node.secondary_ip)
2478
                                 for node in node_data_list
2479
                                 if not node.offline],
2480
      constants.NV_INSTANCELIST: hypervisors,
2481
      constants.NV_VERSION: None,
2482
      constants.NV_HVINFO: self.cfg.GetHypervisorType(),
2483
      constants.NV_NODESETUP: None,
2484
      constants.NV_TIME: None,
2485
      constants.NV_MASTERIP: (master_node, master_ip),
2486
      constants.NV_OSLIST: None,
2487
      constants.NV_VMNODES: self.cfg.GetNonVmCapableNodeList(),
2488
      }
2489

    
2490
    if vg_name is not None:
2491
      node_verify_param[constants.NV_VGLIST] = None
2492
      node_verify_param[constants.NV_LVLIST] = vg_name
2493
      node_verify_param[constants.NV_PVLIST] = [vg_name]
2494
      node_verify_param[constants.NV_DRBDLIST] = None
2495

    
2496
    if drbd_helper:
2497
      node_verify_param[constants.NV_DRBDHELPER] = drbd_helper
2498

    
2499
    # bridge checks
2500
    # FIXME: this needs to be changed per node-group, not cluster-wide
2501
    bridges = set()
2502
    default_nicpp = cluster.nicparams[constants.PP_DEFAULT]
2503
    if default_nicpp[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2504
      bridges.add(default_nicpp[constants.NIC_LINK])
2505
    for instance in self.my_inst_info.values():
2506
      for nic in instance.nics:
2507
        full_nic = cluster.SimpleFillNIC(nic.nicparams)
2508
        if full_nic[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2509
          bridges.add(full_nic[constants.NIC_LINK])
2510

    
2511
    if bridges:
2512
      node_verify_param[constants.NV_BRIDGES] = list(bridges)
2513

    
2514
    # Build our expected cluster state
2515
    node_image = dict((node.name, self.NodeImage(offline=node.offline,
2516
                                                 name=node.name,
2517
                                                 vm_capable=node.vm_capable))
2518
                      for node in node_data_list)
2519

    
2520
    # Gather OOB paths
2521
    oob_paths = []
2522
    for node in self.all_node_info.values():
2523
      path = _SupportsOob(self.cfg, node)
2524
      if path and path not in oob_paths:
2525
        oob_paths.append(path)
2526

    
2527
    if oob_paths:
2528
      node_verify_param[constants.NV_OOB_PATHS] = oob_paths
2529

    
2530
    for instance in self.my_inst_names:
2531
      inst_config = self.my_inst_info[instance]
2532

    
2533
      for nname in inst_config.all_nodes:
2534
        if nname not in node_image:
2535
          gnode = self.NodeImage(name=nname)
2536
          gnode.ghost = (nname not in self.all_node_info)
2537
          node_image[nname] = gnode
2538

    
2539
      inst_config.MapLVsByNode(node_vol_should)
2540

    
2541
      pnode = inst_config.primary_node
2542
      node_image[pnode].pinst.append(instance)
2543

    
2544
      for snode in inst_config.secondary_nodes:
2545
        nimg = node_image[snode]
2546
        nimg.sinst.append(instance)
2547
        if pnode not in nimg.sbp:
2548
          nimg.sbp[pnode] = []
2549
        nimg.sbp[pnode].append(instance)
2550

    
2551
    # At this point, we have the in-memory data structures complete,
2552
    # except for the runtime information, which we'll gather next
2553

    
2554
    # Due to the way our RPC system works, exact response times cannot be
2555
    # guaranteed (e.g. a broken node could run into a timeout). By keeping the
2556
    # time before and after executing the request, we can at least have a time
2557
    # window.
2558
    nvinfo_starttime = time.time()
2559
    all_nvinfo = self.rpc.call_node_verify(self.my_node_names,
2560
                                           node_verify_param,
2561
                                           self.cfg.GetClusterName())
2562
    if self.extra_lv_nodes and vg_name is not None:
2563
      extra_lv_nvinfo = \
2564
          self.rpc.call_node_verify(self.extra_lv_nodes,
2565
                                    {constants.NV_LVLIST: vg_name},
2566
                                    self.cfg.GetClusterName())
2567
    else:
2568
      extra_lv_nvinfo = {}
2569
    nvinfo_endtime = time.time()
2570

    
2571
    all_drbd_map = self.cfg.ComputeDRBDMap()
2572

    
2573
    feedback_fn("* Gathering disk information (%s nodes)" %
2574
                len(self.my_node_names))
2575
    instdisk = self._CollectDiskInfo(self.my_node_names, node_image,
2576
                                     self.my_inst_info)
2577

    
2578
    feedback_fn("* Verifying configuration file consistency")
2579

    
2580
    # If not all nodes are being checked, we need to make sure the master node
2581
    # and a non-checked vm_capable node are in the list.
2582
    absent_nodes = set(self.all_node_info).difference(self.my_node_info)
2583
    if absent_nodes:
2584
      vf_nvinfo = all_nvinfo.copy()
2585
      vf_node_info = list(self.my_node_info.values())
2586
      additional_nodes = []
2587
      if master_node not in self.my_node_info:
2588
        additional_nodes.append(master_node)
2589
        vf_node_info.append(self.all_node_info[master_node])
2590
      # Add the first vm_capable node we find which is not included
2591
      for node in absent_nodes:
2592
        nodeinfo = self.all_node_info[node]
2593
        if nodeinfo.vm_capable and not nodeinfo.offline:
2594
          additional_nodes.append(node)
2595
          vf_node_info.append(self.all_node_info[node])
2596
          break
2597
      key = constants.NV_FILELIST
2598
      vf_nvinfo.update(self.rpc.call_node_verify(additional_nodes,
2599
                                                 {key: node_verify_param[key]},
2600
                                                 self.cfg.GetClusterName()))
2601
    else:
2602
      vf_nvinfo = all_nvinfo
2603
      vf_node_info = self.my_node_info.values()
2604

    
2605
    self._VerifyFiles(_ErrorIf, vf_node_info, master_node, vf_nvinfo, filemap)
2606

    
2607
    feedback_fn("* Verifying node status")
2608

    
2609
    refos_img = None
2610

    
2611
    for node_i in node_data_list:
2612
      node = node_i.name
2613
      nimg = node_image[node]
2614

    
2615
      if node_i.offline:
2616
        if verbose:
2617
          feedback_fn("* Skipping offline node %s" % (node,))
2618
        n_offline += 1
2619
        continue
2620

    
2621
      if node == master_node:
2622
        ntype = "master"
2623
      elif node_i.master_candidate:
2624
        ntype = "master candidate"
2625
      elif node_i.drained:
2626
        ntype = "drained"
2627
        n_drained += 1
2628
      else:
2629
        ntype = "regular"
2630
      if verbose:
2631
        feedback_fn("* Verifying node %s (%s)" % (node, ntype))
2632

    
2633
      msg = all_nvinfo[node].fail_msg
2634
      _ErrorIf(msg, self.ENODERPC, node, "while contacting node: %s", msg)
2635
      if msg:
2636
        nimg.rpc_fail = True
2637
        continue
2638

    
2639
      nresult = all_nvinfo[node].payload
2640

    
2641
      nimg.call_ok = self._VerifyNode(node_i, nresult)
2642
      self._VerifyNodeTime(node_i, nresult, nvinfo_starttime, nvinfo_endtime)
2643
      self._VerifyNodeNetwork(node_i, nresult)
2644
      self._VerifyOob(node_i, nresult)
2645

    
2646
      if nimg.vm_capable:
2647
        self._VerifyNodeLVM(node_i, nresult, vg_name)
2648
        self._VerifyNodeDrbd(node_i, nresult, self.all_inst_info, drbd_helper,
2649
                             all_drbd_map)
2650

    
2651
        self._UpdateNodeVolumes(node_i, nresult, nimg, vg_name)
2652
        self._UpdateNodeInstances(node_i, nresult, nimg)
2653
        self._UpdateNodeInfo(node_i, nresult, nimg, vg_name)
2654
        self._UpdateNodeOS(node_i, nresult, nimg)
2655

    
2656
        if not nimg.os_fail:
2657
          if refos_img is None:
2658
            refos_img = nimg
2659
          self._VerifyNodeOS(node_i, nimg, refos_img)
2660
        self._VerifyNodeBridges(node_i, nresult, bridges)
2661

    
2662
        # Check whether all running instancies are primary for the node. (This
2663
        # can no longer be done from _VerifyInstance below, since some of the
2664
        # wrong instances could be from other node groups.)
2665
        non_primary_inst = set(nimg.instances).difference(nimg.pinst)
2666

    
2667
        for inst in non_primary_inst:
2668
          test = inst in self.all_inst_info
2669
          _ErrorIf(test, self.EINSTANCEWRONGNODE, inst,
2670
                   "instance should not run on node %s", node_i.name)
2671
          _ErrorIf(not test, self.ENODEORPHANINSTANCE, node_i.name,
2672
                   "node is running unknown instance %s", inst)
2673

    
2674
    for node, result in extra_lv_nvinfo.items():
2675
      self._UpdateNodeVolumes(self.all_node_info[node], result.payload,
2676
                              node_image[node], vg_name)
2677

    
2678
    feedback_fn("* Verifying instance status")
2679
    for instance in self.my_inst_names:
2680
      if verbose:
2681
        feedback_fn("* Verifying instance %s" % instance)
2682
      inst_config = self.my_inst_info[instance]
2683
      self._VerifyInstance(instance, inst_config, node_image,
2684
                           instdisk[instance])
2685
      inst_nodes_offline = []
2686

    
2687
      pnode = inst_config.primary_node
2688
      pnode_img = node_image[pnode]
2689
      _ErrorIf(pnode_img.rpc_fail and not pnode_img.offline,
2690
               self.ENODERPC, pnode, "instance %s, connection to"
2691
               " primary node failed", instance)
2692

    
2693
      _ErrorIf(inst_config.admin_up and pnode_img.offline,
2694
               self.EINSTANCEBADNODE, instance,
2695
               "instance is marked as running and lives on offline node %s",
2696
               inst_config.primary_node)
2697

    
2698
      # If the instance is non-redundant we cannot survive losing its primary
2699
      # node, so we are not N+1 compliant. On the other hand we have no disk
2700
      # templates with more than one secondary so that situation is not well
2701
      # supported either.
2702
      # FIXME: does not support file-backed instances
2703
      if not inst_config.secondary_nodes:
2704
        i_non_redundant.append(instance)
2705

    
2706
      _ErrorIf(len(inst_config.secondary_nodes) > 1, self.EINSTANCELAYOUT,
2707
               instance, "instance has multiple secondary nodes: %s",
2708
               utils.CommaJoin(inst_config.secondary_nodes),
2709
               code=self.ETYPE_WARNING)
2710

    
2711
      if inst_config.disk_template in constants.DTS_INT_MIRROR:
2712
        pnode = inst_config.primary_node
2713
        instance_nodes = utils.NiceSort(inst_config.all_nodes)
2714
        instance_groups = {}
2715

    
2716
        for node in instance_nodes:
2717
          instance_groups.setdefault(self.all_node_info[node].group,
2718
                                     []).append(node)
2719

    
2720
        pretty_list = [
2721
          "%s (group %s)" % (utils.CommaJoin(nodes), groupinfo[group].name)
2722
          # Sort so that we always list the primary node first.
2723
          for group, nodes in sorted(instance_groups.items(),
2724
                                     key=lambda (_, nodes): pnode in nodes,
2725
                                     reverse=True)]
2726

    
2727
        self._ErrorIf(len(instance_groups) > 1, self.EINSTANCESPLITGROUPS,
2728
                      instance, "instance has primary and secondary nodes in"
2729
                      " different groups: %s", utils.CommaJoin(pretty_list),
2730
                      code=self.ETYPE_WARNING)
2731

    
2732
      if not cluster.FillBE(inst_config)[constants.BE_AUTO_BALANCE]:
2733
        i_non_a_balanced.append(instance)
2734

    
2735
      for snode in inst_config.secondary_nodes:
2736
        s_img = node_image[snode]
2737
        _ErrorIf(s_img.rpc_fail and not s_img.offline, self.ENODERPC, snode,
2738
                 "instance %s, connection to secondary node failed", instance)
2739

    
2740
        if s_img.offline:
2741
          inst_nodes_offline.append(snode)
2742

    
2743
      # warn that the instance lives on offline nodes
2744
      _ErrorIf(inst_nodes_offline, self.EINSTANCEBADNODE, instance,
2745
               "instance has offline secondary node(s) %s",
2746
               utils.CommaJoin(inst_nodes_offline))
2747
      # ... or ghost/non-vm_capable nodes
2748
      for node in inst_config.all_nodes:
2749
        _ErrorIf(node_image[node].ghost, self.EINSTANCEBADNODE, instance,
2750
                 "instance lives on ghost node %s", node)
2751
        _ErrorIf(not node_image[node].vm_capable, self.EINSTANCEBADNODE,
2752
                 instance, "instance lives on non-vm_capable node %s", node)
2753

    
2754
    feedback_fn("* Verifying orphan volumes")
2755
    reserved = utils.FieldSet(*cluster.reserved_lvs)
2756

    
2757
    # We will get spurious "unknown volume" warnings if any node of this group
2758
    # is secondary for an instance whose primary is in another group. To avoid
2759
    # them, we find these instances and add their volumes to node_vol_should.
2760
    for inst in self.all_inst_info.values():
2761
      for secondary in inst.secondary_nodes:
2762
        if (secondary in self.my_node_info
2763
            and inst.name not in self.my_inst_info):
2764
          inst.MapLVsByNode(node_vol_should)
2765
          break
2766

    
2767
    self._VerifyOrphanVolumes(node_vol_should, node_image, reserved)
2768

    
2769
    if constants.VERIFY_NPLUSONE_MEM not in self.op.skip_checks:
2770
      feedback_fn("* Verifying N+1 Memory redundancy")
2771
      self._VerifyNPlusOneMemory(node_image, self.my_inst_info)
2772

    
2773
    feedback_fn("* Other Notes")
2774
    if i_non_redundant:
2775
      feedback_fn("  - NOTICE: %d non-redundant instance(s) found."
2776
                  % len(i_non_redundant))
2777

    
2778
    if i_non_a_balanced:
2779
      feedback_fn("  - NOTICE: %d non-auto-balanced instance(s) found."
2780
                  % len(i_non_a_balanced))
2781

    
2782
    if n_offline:
2783
      feedback_fn("  - NOTICE: %d offline node(s) found." % n_offline)
2784

    
2785
    if n_drained:
2786
      feedback_fn("  - NOTICE: %d drained node(s) found." % n_drained)
2787

    
2788
    return not self.bad
2789

    
2790
  def HooksCallBack(self, phase, hooks_results, feedback_fn, lu_result):
2791
    """Analyze the post-hooks' result
2792

2793
    This method analyses the hook result, handles it, and sends some
2794
    nicely-formatted feedback back to the user.
2795

2796
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
2797
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
2798
    @param hooks_results: the results of the multi-node hooks rpc call
2799
    @param feedback_fn: function used send feedback back to the caller
2800
    @param lu_result: previous Exec result
2801
    @return: the new Exec result, based on the previous result
2802
        and hook results
2803

2804
    """
2805
    # We only really run POST phase hooks, and are only interested in
2806
    # their results
2807
    if phase == constants.HOOKS_PHASE_POST:
2808
      # Used to change hooks' output to proper indentation
2809
      feedback_fn("* Hooks Results")
2810
      assert hooks_results, "invalid result from hooks"
2811

    
2812
      for node_name in hooks_results:
2813
        res = hooks_results[node_name]
2814
        msg = res.fail_msg
2815
        test = msg and not res.offline
2816
        self._ErrorIf(test, self.ENODEHOOKS, node_name,
2817
                      "Communication failure in hooks execution: %s", msg)
2818
        if res.offline or msg:
2819
          # No need to investigate payload if node is offline or gave an error.
2820
          # override manually lu_result here as _ErrorIf only
2821
          # overrides self.bad
2822
          lu_result = 1
2823
          continue
2824
        for script, hkr, output in res.payload:
2825
          test = hkr == constants.HKR_FAIL
2826
          self._ErrorIf(test, self.ENODEHOOKS, node_name,
2827
                        "Script %s failed, output:", script)
2828
          if test:
2829
            output = self._HOOKS_INDENT_RE.sub('      ', output)
2830
            feedback_fn("%s" % output)
2831
            lu_result = 0
2832

    
2833
      return lu_result
2834

    
2835

    
2836
class LUClusterVerifyDisks(NoHooksLU):
2837
  """Verifies the cluster disks status.
2838

2839
  """
2840
  REQ_BGL = False
2841

    
2842
  def ExpandNames(self):
2843
    self.needed_locks = {
2844
      locking.LEVEL_NODE: locking.ALL_SET,
2845
      locking.LEVEL_INSTANCE: locking.ALL_SET,
2846
    }
2847
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2848

    
2849
  def Exec(self, feedback_fn):
2850
    """Verify integrity of cluster disks.
2851

2852
    @rtype: tuple of three items
2853
    @return: a tuple of (dict of node-to-node_error, list of instances
2854
        which need activate-disks, dict of instance: (node, volume) for
2855
        missing volumes
2856

2857
    """
2858
    result = res_nodes, res_instances, res_missing = {}, [], {}
2859

    
2860
    nodes = utils.NiceSort(self.cfg.GetVmCapableNodeList())
2861
    instances = self.cfg.GetAllInstancesInfo().values()
2862

    
2863
    nv_dict = {}
2864
    for inst in instances:
2865
      inst_lvs = {}
2866
      if not inst.admin_up:
2867
        continue
2868
      inst.MapLVsByNode(inst_lvs)
2869
      # transform { iname: {node: [vol,],},} to {(node, vol): iname}
2870
      for node, vol_list in inst_lvs.iteritems():
2871
        for vol in vol_list:
2872
          nv_dict[(node, vol)] = inst
2873

    
2874
    if not nv_dict:
2875
      return result
2876

    
2877
    node_lvs = self.rpc.call_lv_list(nodes, [])
2878
    for node, node_res in node_lvs.items():
2879
      if node_res.offline:
2880
        continue
2881
      msg = node_res.fail_msg
2882
      if msg:
2883
        logging.warning("Error enumerating LVs on node %s: %s", node, msg)
2884
        res_nodes[node] = msg
2885
        continue
2886

    
2887
      lvs = node_res.payload
2888
      for lv_name, (_, _, lv_online) in lvs.items():
2889
        inst = nv_dict.pop((node, lv_name), None)
2890
        if (not lv_online and inst is not None
2891
            and inst.name not in res_instances):
2892
          res_instances.append(inst.name)
2893

    
2894
    # any leftover items in nv_dict are missing LVs, let's arrange the
2895
    # data better
2896
    for key, inst in nv_dict.iteritems():
2897
      if inst.name not in res_missing:
2898
        res_missing[inst.name] = []
2899
      res_missing[inst.name].append(key)
2900

    
2901
    return result
2902

    
2903

    
2904
class LUClusterRepairDiskSizes(NoHooksLU):
2905
  """Verifies the cluster disks sizes.
2906

2907
  """
2908
  REQ_BGL = False
2909

    
2910
  def ExpandNames(self):
2911
    if self.op.instances:
2912
      self.wanted_names = _GetWantedInstances(self, self.op.instances)
2913
      self.needed_locks = {
2914
        locking.LEVEL_NODE: [],
2915
        locking.LEVEL_INSTANCE: self.wanted_names,
2916
        }
2917
      self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE
2918
    else:
2919
      self.wanted_names = None
2920
      self.needed_locks = {
2921
        locking.LEVEL_NODE: locking.ALL_SET,
2922
        locking.LEVEL_INSTANCE: locking.ALL_SET,
2923
        }
2924
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2925

    
2926
  def DeclareLocks(self, level):
2927
    if level == locking.LEVEL_NODE and self.wanted_names is not None:
2928
      self._LockInstancesNodes(primary_only=True)
2929

    
2930
  def CheckPrereq(self):
2931
    """Check prerequisites.
2932

2933
    This only checks the optional instance list against the existing names.
2934

2935
    """
2936
    if self.wanted_names is None:
2937
      self.wanted_names = self.glm.list_owned(locking.LEVEL_INSTANCE)
2938

    
2939
    self.wanted_instances = [self.cfg.GetInstanceInfo(name) for name
2940
                             in self.wanted_names]
2941

    
2942
  def _EnsureChildSizes(self, disk):
2943
    """Ensure children of the disk have the needed disk size.
2944

2945
    This is valid mainly for DRBD8 and fixes an issue where the
2946
    children have smaller disk size.
2947

2948
    @param disk: an L{ganeti.objects.Disk} object
2949

2950
    """
2951
    if disk.dev_type == constants.LD_DRBD8:
2952
      assert disk.children, "Empty children for DRBD8?"
2953
      fchild = disk.children[0]
2954
      mismatch = fchild.size < disk.size
2955
      if mismatch:
2956
        self.LogInfo("Child disk has size %d, parent %d, fixing",
2957
                     fchild.size, disk.size)
2958
        fchild.size = disk.size
2959

    
2960
      # and we recurse on this child only, not on the metadev
2961
      return self._EnsureChildSizes(fchild) or mismatch
2962
    else:
2963
      return False
2964

    
2965
  def Exec(self, feedback_fn):
2966
    """Verify the size of cluster disks.
2967

2968
    """
2969
    # TODO: check child disks too
2970
    # TODO: check differences in size between primary/secondary nodes
2971
    per_node_disks = {}
2972
    for instance in self.wanted_instances:
2973
      pnode = instance.primary_node
2974
      if pnode not in per_node_disks:
2975
        per_node_disks[pnode] = []
2976
      for idx, disk in enumerate(instance.disks):
2977
        per_node_disks[pnode].append((instance, idx, disk))
2978

    
2979
    changed = []
2980
    for node, dskl in per_node_disks.items():
2981
      newl = [v[2].Copy() for v in dskl]
2982
      for dsk in newl:
2983
        self.cfg.SetDiskID(dsk, node)
2984
      result = self.rpc.call_blockdev_getsize(node, newl)
2985
      if result.fail_msg:
2986
        self.LogWarning("Failure in blockdev_getsize call to node"
2987
                        " %s, ignoring", node)
2988
        continue
2989
      if len(result.payload) != len(dskl):
2990
        logging.warning("Invalid result from node %s: len(dksl)=%d,"
2991
                        " result.payload=%s", node, len(dskl), result.payload)
2992
        self.LogWarning("Invalid result from node %s, ignoring node results",
2993
                        node)
2994
        continue
2995
      for ((instance, idx, disk), size) in zip(dskl, result.payload):
2996
        if size is None:
2997
          self.LogWarning("Disk %d of instance %s did not return size"
2998
                          " information, ignoring", idx, instance.name)
2999
          continue
3000
        if not isinstance(size, (int, long)):
3001
          self.LogWarning("Disk %d of instance %s did not return valid"
3002
                          " size information, ignoring", idx, instance.name)
3003
          continue
3004
        size = size >> 20
3005
        if size != disk.size:
3006
          self.LogInfo("Disk %d of instance %s has mismatched size,"
3007
                       " correcting: recorded %d, actual %d", idx,
3008
                       instance.name, disk.size, size)
3009
          disk.size = size
3010
          self.cfg.Update(instance, feedback_fn)
3011
          changed.append((instance.name, idx, size))
3012
        if self._EnsureChildSizes(disk):
3013
          self.cfg.Update(instance, feedback_fn)
3014
          changed.append((instance.name, idx, disk.size))
3015
    return changed
3016

    
3017

    
3018
class LUClusterRename(LogicalUnit):
3019
  """Rename the cluster.
3020

3021
  """
3022
  HPATH = "cluster-rename"
3023
  HTYPE = constants.HTYPE_CLUSTER
3024

    
3025
  def BuildHooksEnv(self):
3026
    """Build hooks env.
3027

3028
    """
3029
    return {
3030
      "OP_TARGET": self.cfg.GetClusterName(),
3031
      "NEW_NAME": self.op.name,
3032
      }
3033

    
3034
  def BuildHooksNodes(self):
3035
    """Build hooks nodes.
3036

3037
    """
3038
    return ([self.cfg.GetMasterNode()], self.cfg.GetNodeList())
3039

    
3040
  def CheckPrereq(self):
3041
    """Verify that the passed name is a valid one.
3042

3043
    """
3044
    hostname = netutils.GetHostname(name=self.op.name,
3045
                                    family=self.cfg.GetPrimaryIPFamily())
3046

    
3047
    new_name = hostname.name
3048
    self.ip = new_ip = hostname.ip
3049
    old_name = self.cfg.GetClusterName()
3050
    old_ip = self.cfg.GetMasterIP()
3051
    if new_name == old_name and new_ip == old_ip:
3052
      raise errors.OpPrereqError("Neither the name nor the IP address of the"
3053
                                 " cluster has changed",
3054
                                 errors.ECODE_INVAL)
3055
    if new_ip != old_ip:
3056
      if netutils.TcpPing(new_ip, constants.DEFAULT_NODED_PORT):
3057
        raise errors.OpPrereqError("The given cluster IP address (%s) is"
3058
                                   " reachable on the network" %
3059
                                   new_ip, errors.ECODE_NOTUNIQUE)
3060

    
3061
    self.op.name = new_name
3062

    
3063
  def Exec(self, feedback_fn):
3064
    """Rename the cluster.
3065

3066
    """
3067
    clustername = self.op.name
3068
    ip = self.ip
3069

    
3070
    # shutdown the master IP
3071
    master = self.cfg.GetMasterNode()
3072
    result = self.rpc.call_node_stop_master(master, False)
3073
    result.Raise("Could not disable the master role")
3074

    
3075
    try:
3076
      cluster = self.cfg.GetClusterInfo()
3077
      cluster.cluster_name = clustername
3078
      cluster.master_ip = ip
3079
      self.cfg.Update(cluster, feedback_fn)
3080

    
3081
      # update the known hosts file
3082
      ssh.WriteKnownHostsFile(self.cfg, constants.SSH_KNOWN_HOSTS_FILE)
3083
      node_list = self.cfg.GetOnlineNodeList()
3084
      try:
3085
        node_list.remove(master)
3086
      except ValueError:
3087
        pass
3088
      _UploadHelper(self, node_list, constants.SSH_KNOWN_HOSTS_FILE)
3089
    finally:
3090
      result = self.rpc.call_node_start_master(master, False, False)
3091
      msg = result.fail_msg
3092
      if msg:
3093
        self.LogWarning("Could not re-enable the master role on"
3094
                        " the master, please restart manually: %s", msg)
3095

    
3096
    return clustername
3097

    
3098

    
3099
class LUClusterSetParams(LogicalUnit):
3100
  """Change the parameters of the cluster.
3101

3102
  """
3103
  HPATH = "cluster-modify"
3104
  HTYPE = constants.HTYPE_CLUSTER
3105
  REQ_BGL = False
3106

    
3107
  def CheckArguments(self):
3108
    """Check parameters
3109

3110
    """
3111
    if self.op.uid_pool:
3112
      uidpool.CheckUidPool(self.op.uid_pool)
3113

    
3114
    if self.op.add_uids:
3115
      uidpool.CheckUidPool(self.op.add_uids)
3116

    
3117
    if self.op.remove_uids:
3118
      uidpool.CheckUidPool(self.op.remove_uids)
3119

    
3120
  def ExpandNames(self):
3121
    # FIXME: in the future maybe other cluster params won't require checking on
3122
    # all nodes to be modified.
3123
    self.needed_locks = {
3124
      locking.LEVEL_NODE: locking.ALL_SET,
3125
    }
3126
    self.share_locks[locking.LEVEL_NODE] = 1
3127

    
3128
  def BuildHooksEnv(self):
3129
    """Build hooks env.
3130

3131
    """
3132
    return {
3133
      "OP_TARGET": self.cfg.GetClusterName(),
3134
      "NEW_VG_NAME": self.op.vg_name,
3135
      }
3136

    
3137
  def BuildHooksNodes(self):
3138
    """Build hooks nodes.
3139

3140
    """
3141
    mn = self.cfg.GetMasterNode()
3142
    return ([mn], [mn])
3143

    
3144
  def CheckPrereq(self):
3145
    """Check prerequisites.
3146

3147
    This checks whether the given params don't conflict and
3148
    if the given volume group is valid.
3149

3150
    """
3151
    if self.op.vg_name is not None and not self.op.vg_name:
3152
      if self.cfg.HasAnyDiskOfType(constants.LD_LV):
3153
        raise errors.OpPrereqError("Cannot disable lvm storage while lvm-based"
3154
                                   " instances exist", errors.ECODE_INVAL)
3155

    
3156
    if self.op.drbd_helper is not None and not self.op.drbd_helper:
3157
      if self.cfg.HasAnyDiskOfType(constants.LD_DRBD8):
3158
        raise errors.OpPrereqError("Cannot disable drbd helper while"
3159
                                   " drbd-based instances exist",
3160
                                   errors.ECODE_INVAL)
3161

    
3162
    node_list = self.glm.list_owned(locking.LEVEL_NODE)
3163

    
3164
    # if vg_name not None, checks given volume group on all nodes
3165
    if self.op.vg_name:
3166
      vglist = self.rpc.call_vg_list(node_list)
3167
      for node in node_list:
3168
        msg = vglist[node].fail_msg
3169
        if msg:
3170
          # ignoring down node
3171
          self.LogWarning("Error while gathering data on node %s"
3172
                          " (ignoring node): %s", node, msg)
3173
          continue
3174
        vgstatus = utils.CheckVolumeGroupSize(vglist[node].payload,
3175
                                              self.op.vg_name,
3176
                                              constants.MIN_VG_SIZE)
3177
        if vgstatus:
3178
          raise errors.OpPrereqError("Error on node '%s': %s" %
3179
                                     (node, vgstatus), errors.ECODE_ENVIRON)
3180

    
3181
    if self.op.drbd_helper:
3182
      # checks given drbd helper on all nodes
3183
      helpers = self.rpc.call_drbd_helper(node_list)
3184
      for node in node_list:
3185
        ninfo = self.cfg.GetNodeInfo(node)
3186
        if ninfo.offline:
3187
          self.LogInfo("Not checking drbd helper on offline node %s", node)
3188
          continue
3189
        msg = helpers[node].fail_msg
3190
        if msg:
3191
          raise errors.OpPrereqError("Error checking drbd helper on node"
3192
                                     " '%s': %s" % (node, msg),
3193
                                     errors.ECODE_ENVIRON)
3194
        node_helper = helpers[node].payload
3195
        if node_helper != self.op.drbd_helper:
3196
          raise errors.OpPrereqError("Error on node '%s': drbd helper is %s" %
3197
                                     (node, node_helper), errors.ECODE_ENVIRON)
3198

    
3199
    self.cluster = cluster = self.cfg.GetClusterInfo()
3200
    # validate params changes
3201
    if self.op.beparams:
3202
      utils.ForceDictType(self.op.beparams, constants.BES_PARAMETER_TYPES)
3203
      self.new_beparams = cluster.SimpleFillBE(self.op.beparams)
3204

    
3205
    if self.op.ndparams:
3206
      utils.ForceDictType(self.op.ndparams, constants.NDS_PARAMETER_TYPES)
3207
      self.new_ndparams = cluster.SimpleFillND(self.op.ndparams)
3208

    
3209
      # TODO: we need a more general way to handle resetting
3210
      # cluster-level parameters to default values
3211
      if self.new_ndparams["oob_program"] == "":
3212
        self.new_ndparams["oob_program"] = \
3213
            constants.NDC_DEFAULTS[constants.ND_OOB_PROGRAM]
3214

    
3215
    if self.op.nicparams:
3216
      utils.ForceDictType(self.op.nicparams, constants.NICS_PARAMETER_TYPES)
3217
      self.new_nicparams = cluster.SimpleFillNIC(self.op.nicparams)
3218
      objects.NIC.CheckParameterSyntax(self.new_nicparams)
3219
      nic_errors = []
3220

    
3221
      # check all instances for consistency
3222
      for instance in self.cfg.GetAllInstancesInfo().values():
3223
        for nic_idx, nic in enumerate(instance.nics):
3224
          params_copy = copy.deepcopy(nic.nicparams)
3225
          params_filled = objects.FillDict(self.new_nicparams, params_copy)
3226

    
3227
          # check parameter syntax
3228
          try:
3229
            objects.NIC.CheckParameterSyntax(params_filled)
3230
          except errors.ConfigurationError, err:
3231
            nic_errors.append("Instance %s, nic/%d: %s" %
3232
                              (instance.name, nic_idx, err))
3233

    
3234
          # if we're moving instances to routed, check that they have an ip
3235
          target_mode = params_filled[constants.NIC_MODE]
3236
          if target_mode == constants.NIC_MODE_ROUTED and not nic.ip:
3237
            nic_errors.append("Instance %s, nic/%d: routed NIC with no ip"
3238
                              " address" % (instance.name, nic_idx))
3239
      if nic_errors:
3240
        raise errors.OpPrereqError("Cannot apply the change, errors:\n%s" %
3241
                                   "\n".join(nic_errors))
3242

    
3243
    # hypervisor list/parameters
3244
    self.new_hvparams = new_hvp = objects.FillDict(cluster.hvparams, {})
3245
    if self.op.hvparams:
3246
      for hv_name, hv_dict in self.op.hvparams.items():
3247
        if hv_name not in self.new_hvparams:
3248
          self.new_hvparams[hv_name] = hv_dict
3249
        else:
3250
          self.new_hvparams[hv_name].update(hv_dict)
3251

    
3252
    # os hypervisor parameters
3253
    self.new_os_hvp = objects.FillDict(cluster.os_hvp, {})
3254
    if self.op.os_hvp:
3255
      for os_name, hvs in self.op.os_hvp.items():
3256
        if os_name not in self.new_os_hvp:
3257
          self.new_os_hvp[os_name] = hvs
3258
        else:
3259
          for hv_name, hv_dict in hvs.items():
3260
            if hv_name not in self.new_os_hvp[os_name]:
3261
              self.new_os_hvp[os_name][hv_name] = hv_dict
3262
            else:
3263
              self.new_os_hvp[os_name][hv_name].update(hv_dict)
3264

    
3265
    # os parameters
3266
    self.new_osp = objects.FillDict(cluster.osparams, {})
3267
    if self.op.osparams:
3268
      for os_name, osp in self.op.osparams.items():
3269
        if os_name not in self.new_osp:
3270
          self.new_osp[os_name] = {}
3271

    
3272
        self.new_osp[os_name] = _GetUpdatedParams(self.new_osp[os_name], osp,
3273
                                                  use_none=True)
3274

    
3275
        if not self.new_osp[os_name]:
3276
          # we removed all parameters
3277
          del self.new_osp[os_name]
3278
        else:
3279
          # check the parameter validity (remote check)
3280
          _CheckOSParams(self, False, [self.cfg.GetMasterNode()],
3281
                         os_name, self.new_osp[os_name])
3282

    
3283
    # changes to the hypervisor list
3284
    if self.op.enabled_hypervisors is not None:
3285
      self.hv_list = self.op.enabled_hypervisors
3286
      for hv in self.hv_list:
3287
        # if the hypervisor doesn't already exist in the cluster
3288
        # hvparams, we initialize it to empty, and then (in both
3289
        # cases) we make sure to fill the defaults, as we might not
3290
        # have a complete defaults list if the hypervisor wasn't
3291
        # enabled before
3292
        if hv not in new_hvp:
3293
          new_hvp[hv] = {}
3294
        new_hvp[hv] = objects.FillDict(constants.HVC_DEFAULTS[hv], new_hvp[hv])
3295
        utils.ForceDictType(new_hvp[hv], constants.HVS_PARAMETER_TYPES)
3296
    else:
3297
      self.hv_list = cluster.enabled_hypervisors
3298

    
3299
    if self.op.hvparams or self.op.enabled_hypervisors is not None:
3300
      # either the enabled list has changed, or the parameters have, validate
3301
      for hv_name, hv_params in self.new_hvparams.items():
3302
        if ((self.op.hvparams and hv_name in self.op.hvparams) or
3303
            (self.op.enabled_hypervisors and
3304
             hv_name in self.op.enabled_hypervisors)):
3305
          # either this is a new hypervisor, or its parameters have changed
3306
          hv_class = hypervisor.GetHypervisor(hv_name)
3307
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3308
          hv_class.CheckParameterSyntax(hv_params)
3309
          _CheckHVParams(self, node_list, hv_name, hv_params)
3310

    
3311
    if self.op.os_hvp:
3312
      # no need to check any newly-enabled hypervisors, since the
3313
      # defaults have already been checked in the above code-block
3314
      for os_name, os_hvp in self.new_os_hvp.items():
3315
        for hv_name, hv_params in os_hvp.items():
3316
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3317
          # we need to fill in the new os_hvp on top of the actual hv_p
3318
          cluster_defaults = self.new_hvparams.get(hv_name, {})
3319
          new_osp = objects.FillDict(cluster_defaults, hv_params)
3320
          hv_class = hypervisor.GetHypervisor(hv_name)
3321
          hv_class.CheckParameterSyntax(new_osp)
3322
          _CheckHVParams(self, node_list, hv_name, new_osp)
3323

    
3324
    if self.op.default_iallocator:
3325
      alloc_script = utils.FindFile(self.op.default_iallocator,
3326
                                    constants.IALLOCATOR_SEARCH_PATH,
3327
                                    os.path.isfile)
3328
      if alloc_script is None:
3329
        raise errors.OpPrereqError("Invalid default iallocator script '%s'"
3330
                                   " specified" % self.op.default_iallocator,
3331
                                   errors.ECODE_INVAL)
3332

    
3333
  def Exec(self, feedback_fn):
3334
    """Change the parameters of the cluster.
3335

3336
    """
3337
    if self.op.vg_name is not None:
3338
      new_volume = self.op.vg_name
3339
      if not new_volume:
3340
        new_volume = None
3341
      if new_volume != self.cfg.GetVGName():
3342
        self.cfg.SetVGName(new_volume)
3343
      else:
3344
        feedback_fn("Cluster LVM configuration already in desired"
3345
                    " state, not changing")
3346
    if self.op.drbd_helper is not None:
3347
      new_helper = self.op.drbd_helper
3348
      if not new_helper:
3349
        new_helper = None
3350
      if new_helper != self.cfg.GetDRBDHelper():
3351
        self.cfg.SetDRBDHelper(new_helper)
3352
      else:
3353
        feedback_fn("Cluster DRBD helper already in desired state,"
3354
                    " not changing")
3355
    if self.op.hvparams:
3356
      self.cluster.hvparams = self.new_hvparams
3357
    if self.op.os_hvp:
3358
      self.cluster.os_hvp = self.new_os_hvp
3359
    if self.op.enabled_hypervisors is not None:
3360
      self.cluster.hvparams = self.new_hvparams
3361
      self.cluster.enabled_hypervisors = self.op.enabled_hypervisors
3362
    if self.op.beparams:
3363
      self.cluster.beparams[constants.PP_DEFAULT] = self.new_beparams
3364
    if self.op.nicparams:
3365
      self.cluster.nicparams[constants.PP_DEFAULT] = self.new_nicparams
3366
    if self.op.osparams:
3367
      self.cluster.osparams = self.new_osp
3368
    if self.op.ndparams:
3369
      self.cluster.ndparams = self.new_ndparams
3370

    
3371
    if self.op.candidate_pool_size is not None:
3372
      self.cluster.candidate_pool_size = self.op.candidate_pool_size
3373
      # we need to update the pool size here, otherwise the save will fail
3374
      _AdjustCandidatePool(self, [])
3375

    
3376
    if self.op.maintain_node_health is not None:
3377
      self.cluster.maintain_node_health = self.op.maintain_node_health
3378

    
3379
    if self.op.prealloc_wipe_disks is not None:
3380
      self.cluster.prealloc_wipe_disks = self.op.prealloc_wipe_disks
3381

    
3382
    if self.op.add_uids is not None:
3383
      uidpool.AddToUidPool(self.cluster.uid_pool, self.op.add_uids)
3384

    
3385
    if self.op.remove_uids is not None:
3386
      uidpool.RemoveFromUidPool(self.cluster.uid_pool, self.op.remove_uids)
3387

    
3388
    if self.op.uid_pool is not None:
3389
      self.cluster.uid_pool = self.op.uid_pool
3390

    
3391
    if self.op.default_iallocator is not None:
3392
      self.cluster.default_iallocator = self.op.default_iallocator
3393

    
3394
    if self.op.reserved_lvs is not None:
3395
      self.cluster.reserved_lvs = self.op.reserved_lvs
3396

    
3397
    def helper_os(aname, mods, desc):
3398
      desc += " OS list"
3399
      lst = getattr(self.cluster, aname)
3400
      for key, val in mods:
3401
        if key == constants.DDM_ADD:
3402
          if val in lst:
3403
            feedback_fn("OS %s already in %s, ignoring" % (val, desc))
3404
          else:
3405
            lst.append(val)
3406
        elif key == constants.DDM_REMOVE:
3407
          if val in lst:
3408
            lst.remove(val)
3409
          else:
3410
            feedback_fn("OS %s not found in %s, ignoring" % (val, desc))
3411
        else:
3412
          raise errors.ProgrammerError("Invalid modification '%s'" % key)
3413

    
3414
    if self.op.hidden_os:
3415
      helper_os("hidden_os", self.op.hidden_os, "hidden")
3416

    
3417
    if self.op.blacklisted_os:
3418
      helper_os("blacklisted_os", self.op.blacklisted_os, "blacklisted")
3419

    
3420
    if self.op.master_netdev:
3421
      master = self.cfg.GetMasterNode()
3422
      feedback_fn("Shutting down master ip on the current netdev (%s)" %
3423
                  self.cluster.master_netdev)
3424
      result = self.rpc.call_node_stop_master(master, False)
3425
      result.Raise("Could not disable the master ip")
3426
      feedback_fn("Changing master_netdev from %s to %s" %
3427
                  (self.cluster.master_netdev, self.op.master_netdev))
3428
      self.cluster.master_netdev = self.op.master_netdev
3429

    
3430
    self.cfg.Update(self.cluster, feedback_fn)
3431

    
3432
    if self.op.master_netdev:
3433
      feedback_fn("Starting the master ip on the new master netdev (%s)" %
3434
                  self.op.master_netdev)
3435
      result = self.rpc.call_node_start_master(master, False, False)
3436
      if result.fail_msg:
3437
        self.LogWarning("Could not re-enable the master ip on"
3438
                        " the master, please restart manually: %s",
3439
                        result.fail_msg)
3440

    
3441

    
3442
def _UploadHelper(lu, nodes, fname):
3443
  """Helper for uploading a file and showing warnings.
3444

3445
  """
3446
  if os.path.exists(fname):
3447
    result = lu.rpc.call_upload_file(nodes, fname)
3448
    for to_node, to_result in result.items():
3449
      msg = to_result.fail_msg
3450
      if msg:
3451
        msg = ("Copy of file %s to node %s failed: %s" %
3452
               (fname, to_node, msg))
3453
        lu.proc.LogWarning(msg)
3454

    
3455

    
3456
def _ComputeAncillaryFiles(cluster, redist):
3457
  """Compute files external to Ganeti which need to be consistent.
3458

3459
  @type redist: boolean
3460
  @param redist: Whether to include files which need to be redistributed
3461

3462
  """
3463
  # Compute files for all nodes
3464
  files_all = set([
3465
    constants.SSH_KNOWN_HOSTS_FILE,
3466
    constants.CONFD_HMAC_KEY,
3467
    constants.CLUSTER_DOMAIN_SECRET_FILE,
3468
    ])
3469

    
3470
  if not redist:
3471
    files_all.update(constants.ALL_CERT_FILES)
3472
    files_all.update(ssconf.SimpleStore().GetFileList())
3473

    
3474
  if cluster.modify_etc_hosts:
3475
    files_all.add(constants.ETC_HOSTS)
3476

    
3477
  # Files which must either exist on all nodes or on none
3478
  files_all_opt = set([
3479
    constants.RAPI_USERS_FILE,
3480
    ])
3481

    
3482
  # Files which should only be on master candidates
3483
  files_mc = set()
3484
  if not redist:
3485
    files_mc.add(constants.CLUSTER_CONF_FILE)
3486

    
3487
  # Files which should only be on VM-capable nodes
3488
  files_vm = set(filename
3489
    for hv_name in cluster.enabled_hypervisors
3490
    for filename in hypervisor.GetHypervisor(hv_name).GetAncillaryFiles())
3491

    
3492
  # Filenames must be unique
3493
  assert (len(files_all | files_all_opt | files_mc | files_vm) ==
3494
          sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
3495
         "Found file listed in more than one file list"
3496

    
3497
  return (files_all, files_all_opt, files_mc, files_vm)
3498

    
3499

    
3500
def _RedistributeAncillaryFiles(lu, additional_nodes=None, additional_vm=True):
3501
  """Distribute additional files which are part of the cluster configuration.
3502

3503
  ConfigWriter takes care of distributing the config and ssconf files, but
3504
  there are more files which should be distributed to all nodes. This function
3505
  makes sure those are copied.
3506

3507
  @param lu: calling logical unit
3508
  @param additional_nodes: list of nodes not in the config to distribute to
3509
  @type additional_vm: boolean
3510
  @param additional_vm: whether the additional nodes are vm-capable or not
3511

3512
  """
3513
  # Gather target nodes
3514
  cluster = lu.cfg.GetClusterInfo()
3515
  master_info = lu.cfg.GetNodeInfo(lu.cfg.GetMasterNode())
3516

    
3517
  online_nodes = lu.cfg.GetOnlineNodeList()
3518
  vm_nodes = lu.cfg.GetVmCapableNodeList()
3519

    
3520
  if additional_nodes is not None:
3521
    online_nodes.extend(additional_nodes)
3522
    if additional_vm:
3523
      vm_nodes.extend(additional_nodes)
3524

    
3525
  # Never distribute to master node
3526
  for nodelist in [online_nodes, vm_nodes]:
3527
    if master_info.name in nodelist:
3528
      nodelist.remove(master_info.name)
3529

    
3530
  # Gather file lists
3531
  (files_all, files_all_opt, files_mc, files_vm) = \
3532
    _ComputeAncillaryFiles(cluster, True)
3533

    
3534
  # Never re-distribute configuration file from here
3535
  assert not (constants.CLUSTER_CONF_FILE in files_all or
3536
              constants.CLUSTER_CONF_FILE in files_vm)
3537
  assert not files_mc, "Master candidates not handled in this function"
3538

    
3539
  filemap = [
3540
    (online_nodes, files_all),
3541
    (online_nodes, files_all_opt),
3542
    (vm_nodes, files_vm),
3543
    ]
3544

    
3545
  # Upload the files
3546
  for (node_list, files) in filemap:
3547
    for fname in files:
3548
      _UploadHelper(lu, node_list, fname)
3549

    
3550

    
3551
class LUClusterRedistConf(NoHooksLU):
3552
  """Force the redistribution of cluster configuration.
3553

3554
  This is a very simple LU.
3555

3556
  """
3557
  REQ_BGL = False
3558

    
3559
  def ExpandNames(self):
3560
    self.needed_locks = {
3561
      locking.LEVEL_NODE: locking.ALL_SET,
3562
    }
3563
    self.share_locks[locking.LEVEL_NODE] = 1
3564

    
3565
  def Exec(self, feedback_fn):
3566
    """Redistribute the configuration.
3567

3568
    """
3569
    self.cfg.Update(self.cfg.GetClusterInfo(), feedback_fn)
3570
    _RedistributeAncillaryFiles(self)
3571

    
3572

    
3573
def _WaitForSync(lu, instance, disks=None, oneshot=False):
3574
  """Sleep and poll for an instance's disk to sync.
3575

3576
  """
3577
  if not instance.disks or disks is not None and not disks:
3578
    return True
3579

    
3580
  disks = _ExpandCheckDisks(instance, disks)
3581

    
3582
  if not oneshot:
3583
    lu.proc.LogInfo("Waiting for instance %s to sync disks." % instance.name)
3584

    
3585
  node = instance.primary_node
3586

    
3587
  for dev in disks:
3588
    lu.cfg.SetDiskID(dev, node)
3589

    
3590
  # TODO: Convert to utils.Retry
3591

    
3592
  retries = 0
3593
  degr_retries = 10 # in seconds, as we sleep 1 second each time
3594
  while True:
3595
    max_time = 0
3596
    done = True
3597
    cumul_degraded = False
3598
    rstats = lu.rpc.call_blockdev_getmirrorstatus(node, disks)
3599
    msg = rstats.fail_msg
3600
    if msg:
3601
      lu.LogWarning("Can't get any data from node %s: %s", node, msg)
3602
      retries += 1
3603
      if retries >= 10:
3604
        raise errors.RemoteError("Can't contact node %s for mirror data,"
3605
                                 " aborting." % node)
3606
      time.sleep(6)
3607
      continue
3608
    rstats = rstats.payload
3609
    retries = 0
3610
    for i, mstat in enumerate(rstats):
3611
      if mstat is None:
3612
        lu.LogWarning("Can't compute data for node %s/%s",
3613
                           node, disks[i].iv_name)
3614
        continue
3615

    
3616
      cumul_degraded = (cumul_degraded or
3617
                        (mstat.is_degraded and mstat.sync_percent is None))
3618
      if mstat.sync_percent is not None:
3619
        done = False
3620
        if mstat.estimated_time is not None:
3621
          rem_time = ("%s remaining (estimated)" %
3622
                      utils.FormatSeconds(mstat.estimated_time))
3623
          max_time = mstat.estimated_time
3624
        else:
3625
          rem_time = "no time estimate"
3626
        lu.proc.LogInfo("- device %s: %5.2f%% done, %s" %
3627
                        (disks[i].iv_name, mstat.sync_percent, rem_time))
3628

    
3629
    # if we're done but degraded, let's do a few small retries, to
3630
    # make sure we see a stable and not transient situation; therefore
3631
    # we force restart of the loop
3632
    if (done or oneshot) and cumul_degraded and degr_retries > 0:
3633
      logging.info("Degraded disks found, %d retries left", degr_retries)
3634
      degr_retries -= 1
3635
      time.sleep(1)
3636
      continue
3637

    
3638
    if done or oneshot:
3639
      break
3640

    
3641
    time.sleep(min(60, max_time))
3642

    
3643
  if done:
3644
    lu.proc.LogInfo("Instance %s's disks are in sync." % instance.name)
3645
  return not cumul_degraded
3646

    
3647

    
3648
def _CheckDiskConsistency(lu, dev, node, on_primary, ldisk=False):
3649
  """Check that mirrors are not degraded.
3650

3651
  The ldisk parameter, if True, will change the test from the
3652
  is_degraded attribute (which represents overall non-ok status for
3653
  the device(s)) to the ldisk (representing the local storage status).
3654

3655
  """
3656
  lu.cfg.SetDiskID(dev, node)
3657

    
3658
  result = True
3659

    
3660
  if on_primary or dev.AssembleOnSecondary():
3661
    rstats = lu.rpc.call_blockdev_find(node, dev)
3662
    msg = rstats.fail_msg
3663
    if msg:
3664
      lu.LogWarning("Can't find disk on node %s: %s", node, msg)
3665
      result = False
3666
    elif not rstats.payload:
3667
      lu.LogWarning("Can't find disk on node %s", node)
3668
      result = False
3669
    else:
3670
      if ldisk:
3671
        result = result and rstats.payload.ldisk_status == constants.LDS_OKAY
3672
      else:
3673
        result = result and not rstats.payload.is_degraded
3674

    
3675
  if dev.children:
3676
    for child in dev.children:
3677
      result = result and _CheckDiskConsistency(lu, child, node, on_primary)
3678

    
3679
  return result
3680

    
3681

    
3682
class LUOobCommand(NoHooksLU):
3683
  """Logical unit for OOB handling.
3684

3685
  """
3686
  REG_BGL = False
3687
  _SKIP_MASTER = (constants.OOB_POWER_OFF, constants.OOB_POWER_CYCLE)
3688

    
3689
  def ExpandNames(self):
3690
    """Gather locks we need.
3691

3692
    """
3693
    if self.op.node_names:
3694
      self.op.node_names = _GetWantedNodes(self, self.op.node_names)
3695
      lock_names = self.op.node_names
3696
    else:
3697
      lock_names = locking.ALL_SET
3698

    
3699
    self.needed_locks = {
3700
      locking.LEVEL_NODE: lock_names,
3701
      }
3702

    
3703
  def CheckPrereq(self):
3704
    """Check prerequisites.
3705

3706
    This checks:
3707
     - the node exists in the configuration
3708
     - OOB is supported
3709

3710
    Any errors are signaled by raising errors.OpPrereqError.
3711

3712
    """
3713
    self.nodes = []
3714
    self.master_node = self.cfg.GetMasterNode()
3715

    
3716
    assert self.op.power_delay >= 0.0
3717

    
3718
    if self.op.node_names:
3719
      if (self.op.command in self._SKIP_MASTER and
3720
          self.master_node in self.op.node_names):
3721
        master_node_obj = self.cfg.GetNodeInfo(self.master_node)
3722
        master_oob_handler = _SupportsOob(self.cfg, master_node_obj)
3723

    
3724
        if master_oob_handler:
3725
          additional_text = ("run '%s %s %s' if you want to operate on the"
3726
                             " master regardless") % (master_oob_handler,
3727
                                                      self.op.command,
3728
                                                      self.master_node)
3729
        else:
3730
          additional_text = "it does not support out-of-band operations"
3731

    
3732
        raise errors.OpPrereqError(("Operating on the master node %s is not"
3733
                                    " allowed for %s; %s") %
3734
                                   (self.master_node, self.op.command,
3735
                                    additional_text), errors.ECODE_INVAL)
3736
    else:
3737
      self.op.node_names = self.cfg.GetNodeList()
3738
      if self.op.command in self._SKIP_MASTER:
3739
        self.op.node_names.remove(self.master_node)
3740

    
3741
    if self.op.command in self._SKIP_MASTER:
3742
      assert self.master_node not in self.op.node_names
3743

    
3744
    for node_name in self.op.node_names:
3745
      node = self.cfg.GetNodeInfo(node_name)
3746

    
3747
      if node is None:
3748
        raise errors.OpPrereqError("Node %s not found" % node_name,
3749
                                   errors.ECODE_NOENT)
3750
      else:
3751
        self.nodes.append(node)
3752

    
3753
      if (not self.op.ignore_status and
3754
          (self.op.command == constants.OOB_POWER_OFF and not node.offline)):
3755
        raise errors.OpPrereqError(("Cannot power off node %s because it is"
3756
                                    " not marked offline") % node_name,
3757
                                   errors.ECODE_STATE)
3758

    
3759
  def Exec(self, feedback_fn):
3760
    """Execute OOB and return result if we expect any.
3761

3762
    """
3763
    master_node = self.master_node
3764
    ret = []
3765

    
3766
    for idx, node in enumerate(utils.NiceSort(self.nodes,
3767
                                              key=lambda node: node.name)):
3768
      node_entry = [(constants.RS_NORMAL, node.name)]
3769
      ret.append(node_entry)
3770

    
3771
      oob_program = _SupportsOob(self.cfg, node)
3772

    
3773
      if not oob_program:
3774
        node_entry.append((constants.RS_UNAVAIL, None))
3775
        continue
3776

    
3777
      logging.info("Executing out-of-band command '%s' using '%s' on %s",
3778
                   self.op.command, oob_program, node.name)
3779
      result = self.rpc.call_run_oob(master_node, oob_program,
3780
                                     self.op.command, node.name,
3781
                                     self.op.timeout)
3782

    
3783
      if result.fail_msg:
3784
        self.LogWarning("Out-of-band RPC failed on node '%s': %s",
3785
                        node.name, result.fail_msg)
3786
        node_entry.append((constants.RS_NODATA, None))
3787
      else:
3788
        try:
3789
          self._CheckPayload(result)
3790
        except errors.OpExecError, err:
3791
          self.LogWarning("Payload returned by node '%s' is not valid: %s",
3792
                          node.name, err)
3793
          node_entry.append((constants.RS_NODATA, None))
3794
        else:
3795
          if self.op.command == constants.OOB_HEALTH:
3796
            # For health we should log important events
3797
            for item, status in result.payload:
3798
              if status in [constants.OOB_STATUS_WARNING,
3799
                            constants.OOB_STATUS_CRITICAL]:
3800
                self.LogWarning("Item '%s' on node '%s' has status '%s'",
3801
                                item, node.name, status)
3802

    
3803
          if self.op.command == constants.OOB_POWER_ON:
3804
            node.powered = True
3805
          elif self.op.command == constants.OOB_POWER_OFF:
3806
            node.powered = False
3807
          elif self.op.command == constants.OOB_POWER_STATUS:
3808
            powered = result.payload[constants.OOB_POWER_STATUS_POWERED]
3809
            if powered != node.powered:
3810
              logging.warning(("Recorded power state (%s) of node '%s' does not"
3811
                               " match actual power state (%s)"), node.powered,
3812
                              node.name, powered)
3813

    
3814
          # For configuration changing commands we should update the node
3815
          if self.op.command in (constants.OOB_POWER_ON,
3816
                                 constants.OOB_POWER_OFF):
3817
            self.cfg.Update(node, feedback_fn)
3818

    
3819
          node_entry.append((constants.RS_NORMAL, result.payload))
3820

    
3821
          if (self.op.command == constants.OOB_POWER_ON and
3822
              idx < len(self.nodes) - 1):
3823
            time.sleep(self.op.power_delay)
3824

    
3825
    return ret
3826

    
3827
  def _CheckPayload(self, result):
3828
    """Checks if the payload is valid.
3829

3830
    @param result: RPC result
3831
    @raises errors.OpExecError: If payload is not valid
3832

3833
    """
3834
    errs = []
3835
    if self.op.command == constants.OOB_HEALTH:
3836
      if not isinstance(result.payload, list):
3837
        errs.append("command 'health' is expected to return a list but got %s" %
3838
                    type(result.payload))
3839
      else:
3840
        for item, status in result.payload:
3841
          if status not in constants.OOB_STATUSES:
3842
            errs.append("health item '%s' has invalid status '%s'" %
3843
                        (item, status))
3844

    
3845
    if self.op.command == constants.OOB_POWER_STATUS:
3846
      if not isinstance(result.payload, dict):
3847
        errs.append("power-status is expected to return a dict but got %s" %
3848
                    type(result.payload))
3849

    
3850
    if self.op.command in [
3851
        constants.OOB_POWER_ON,
3852
        constants.OOB_POWER_OFF,
3853
        constants.OOB_POWER_CYCLE,
3854
        ]:
3855
      if result.payload is not None:
3856
        errs.append("%s is expected to not return payload but got '%s'" %
3857
                    (self.op.command, result.payload))
3858

    
3859
    if errs:
3860
      raise errors.OpExecError("Check of out-of-band payload failed due to %s" %
3861
                               utils.CommaJoin(errs))
3862

    
3863
class _OsQuery(_QueryBase):
3864
  FIELDS = query.OS_FIELDS
3865

    
3866
  def ExpandNames(self, lu):
3867
    # Lock all nodes in shared mode
3868
    # Temporary removal of locks, should be reverted later
3869
    # TODO: reintroduce locks when they are lighter-weight
3870
    lu.needed_locks = {}
3871
    #self.share_locks[locking.LEVEL_NODE] = 1
3872
    #self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET
3873

    
3874
    # The following variables interact with _QueryBase._GetNames
3875
    if self.names:
3876
      self.wanted = self.names
3877
    else:
3878
      self.wanted = locking.ALL_SET
3879

    
3880
    self.do_locking = self.use_locking
3881

    
3882
  def DeclareLocks(self, lu, level):
3883
    pass
3884

    
3885
  @staticmethod
3886
  def _DiagnoseByOS(rlist):
3887
    """Remaps a per-node return list into an a per-os per-node dictionary
3888

3889
    @param rlist: a map with node names as keys and OS objects as values
3890

3891
    @rtype: dict
3892
    @return: a dictionary with osnames as keys and as value another
3893
        map, with nodes as keys and tuples of (path, status, diagnose,
3894
        variants, parameters, api_versions) as values, eg::
3895

3896
          {"debian-etch": {"node1": [(/usr/lib/..., True, "", [], []),
3897
                                     (/srv/..., False, "invalid api")],
3898
                           "node2": [(/srv/..., True, "", [], [])]}
3899
          }
3900

3901
    """
3902
    all_os = {}
3903
    # we build here the list of nodes that didn't fail the RPC (at RPC
3904
    # level), so that nodes with a non-responding node daemon don't
3905
    # make all OSes invalid
3906
    good_nodes = [node_name for node_name in rlist
3907
                  if not rlist[node_name].fail_msg]
3908
    for node_name, nr in rlist.items():
3909
      if nr.fail_msg or not nr.payload:
3910
        continue
3911
      for (name, path, status, diagnose, variants,
3912
           params, api_versions) in nr.payload:
3913
        if name not in all_os:
3914
          # build a list of nodes for this os containing empty lists
3915
          # for each node in node_list
3916
          all_os[name] = {}
3917
          for nname in good_nodes:
3918
            all_os[name][nname] = []
3919
        # convert params from [name, help] to (name, help)
3920
        params = [tuple(v) for v in params]
3921
        all_os[name][node_name].append((path, status, diagnose,
3922
                                        variants, params, api_versions))
3923
    return all_os
3924

    
3925
  def _GetQueryData(self, lu):
3926
    """Computes the list of nodes and their attributes.
3927

3928
    """
3929
    # Locking is not used
3930
    assert not (compat.any(lu.glm.is_owned(level)
3931
                           for level in locking.LEVELS
3932
                           if level != locking.LEVEL_CLUSTER) or
3933
                self.do_locking or self.use_locking)
3934

    
3935
    valid_nodes = [node.name
3936
                   for node in lu.cfg.GetAllNodesInfo().values()
3937
                   if not node.offline and node.vm_capable]
3938
    pol = self._DiagnoseByOS(lu.rpc.call_os_diagnose(valid_nodes))
3939
    cluster = lu.cfg.GetClusterInfo()
3940

    
3941
    data = {}
3942

    
3943
    for (os_name, os_data) in pol.items():
3944
      info = query.OsInfo(name=os_name, valid=True, node_status=os_data,
3945
                          hidden=(os_name in cluster.hidden_os),
3946
                          blacklisted=(os_name in cluster.blacklisted_os))
3947

    
3948
      variants = set()
3949
      parameters = set()
3950
      api_versions = set()
3951

    
3952
      for idx, osl in enumerate(os_data.values()):
3953
        info.valid = bool(info.valid and osl and osl[0][1])
3954
        if not info.valid:
3955
          break
3956

    
3957
        (node_variants, node_params, node_api) = osl[0][3:6]
3958
        if idx == 0:
3959
          # First entry
3960
          variants.update(node_variants)
3961
          parameters.update(node_params)
3962
          api_versions.update(node_api)
3963
        else:
3964
          # Filter out inconsistent values
3965
          variants.intersection_update(node_variants)
3966
          parameters.intersection_update(node_params)
3967
          api_versions.intersection_update(node_api)
3968

    
3969
      info.variants = list(variants)
3970
      info.parameters = list(parameters)
3971
      info.api_versions = list(api_versions)
3972

    
3973
      data[os_name] = info
3974

    
3975
    # Prepare data in requested order
3976
    return [data[name] for name in self._GetNames(lu, pol.keys(), None)
3977
            if name in data]
3978

    
3979

    
3980
class LUOsDiagnose(NoHooksLU):
3981
  """Logical unit for OS diagnose/query.
3982

3983
  """
3984
  REQ_BGL = False
3985

    
3986
  @staticmethod
3987
  def _BuildFilter(fields, names):
3988
    """Builds a filter for querying OSes.
3989

3990
    """
3991
    name_filter = qlang.MakeSimpleFilter("name", names)
3992

    
3993
    # Legacy behaviour: Hide hidden, blacklisted or invalid OSes if the
3994
    # respective field is not requested
3995
    status_filter = [[qlang.OP_NOT, [qlang.OP_TRUE, fname]]
3996
                     for fname in ["hidden", "blacklisted"]
3997
                     if fname not in fields]
3998
    if "valid" not in fields:
3999
      status_filter.append([qlang.OP_TRUE, "valid"])
4000

    
4001
    if status_filter:
4002
      status_filter.insert(0, qlang.OP_AND)
4003
    else:
4004
      status_filter = None
4005

    
4006
    if name_filter and status_filter:
4007
      return [qlang.OP_AND, name_filter, status_filter]
4008
    elif name_filter:
4009
      return name_filter
4010
    else:
4011
      return status_filter
4012

    
4013
  def CheckArguments(self):
4014
    self.oq = _OsQuery(self._BuildFilter(self.op.output_fields, self.op.names),
4015
                       self.op.output_fields, False)
4016

    
4017
  def ExpandNames(self):
4018
    self.oq.ExpandNames(self)
4019

    
4020
  def Exec(self, feedback_fn):
4021
    return self.oq.OldStyleQuery(self)
4022

    
4023

    
4024
class LUNodeRemove(LogicalUnit):
4025
  """Logical unit for removing a node.
4026

4027
  """
4028
  HPATH = "node-remove"
4029
  HTYPE = constants.HTYPE_NODE
4030

    
4031
  def BuildHooksEnv(self):
4032
    """Build hooks env.
4033

4034
    This doesn't run on the target node in the pre phase as a failed
4035
    node would then be impossible to remove.
4036

4037
    """
4038
    return {
4039
      "OP_TARGET": self.op.node_name,
4040
      "NODE_NAME": self.op.node_name,
4041
      }
4042

    
4043
  def BuildHooksNodes(self):
4044
    """Build hooks nodes.
4045

4046
    """
4047
    all_nodes = self.cfg.GetNodeList()
4048
    try:
4049
      all_nodes.remove(self.op.node_name)
4050
    except ValueError:
4051
      logging.warning("Node '%s', which is about to be removed, was not found"
4052
                      " in the list of all nodes", self.op.node_name)
4053
    return (all_nodes, all_nodes)
4054

    
4055
  def CheckPrereq(self):
4056
    """Check prerequisites.
4057

4058
    This checks:
4059
     - the node exists in the configuration
4060
     - it does not have primary or secondary instances
4061
     - it's not the master
4062

4063
    Any errors are signaled by raising errors.OpPrereqError.
4064

4065
    """
4066
    self.op.node_name = _ExpandNodeName(self.cfg, self.op.node_name)
4067
    node = self.cfg.GetNodeInfo(self.op.node_name)
4068
    assert node is not None
4069

    
4070
    instance_list = self.cfg.GetInstanceList()
4071

    
4072
    masternode = self.cfg.GetMasterNode()
4073
    if node.name == masternode:
4074
      raise errors.OpPrereqError("Node is the master node, failover to another"
4075
                                 " node is required", errors.ECODE_INVAL)
4076

    
4077
    for instance_name in instance_list:
4078
      instance = self.cfg.GetInstanceInfo(instance_name)
4079
      if node.name in instance.all_nodes:
4080
        raise errors.OpPrereqError("Instance %s is still running on the node,"
4081
                                   " please remove first" % instance_name,
4082
                                   errors.ECODE_INVAL)
4083
    self.op.node_name = node.name
4084
    self.node = node
4085

    
4086
  def Exec(self, feedback_fn):
4087
    """Removes the node from the cluster.
4088

4089
    """
4090
    node = self.node
4091
    logging.info("Stopp