Statistics
| Branch: | Tag: | Revision:

root / lib / cmdlib.py @ 0f8810df

History | View | Annotate | Download (444.2 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Module implementing the master-side code."""
23

    
24
# pylint: disable-msg=W0201,C0302
25

    
26
# W0201 since most LU attributes are defined in CheckPrereq or similar
27
# functions
28

    
29
# C0302: since we have waaaay to many lines in this module
30

    
31
import os
32
import os.path
33
import time
34
import re
35
import platform
36
import logging
37
import copy
38
import OpenSSL
39
import socket
40
import tempfile
41
import shutil
42
import itertools
43

    
44
from ganeti import ssh
45
from ganeti import utils
46
from ganeti import errors
47
from ganeti import hypervisor
48
from ganeti import locking
49
from ganeti import constants
50
from ganeti import objects
51
from ganeti import serializer
52
from ganeti import ssconf
53
from ganeti import uidpool
54
from ganeti import compat
55
from ganeti import masterd
56
from ganeti import netutils
57
from ganeti import query
58
from ganeti import qlang
59
from ganeti import opcodes
60
from ganeti import ht
61

    
62
import ganeti.masterd.instance # pylint: disable-msg=W0611
63

    
64

    
65
def _SupportsOob(cfg, node):
66
  """Tells if node supports OOB.
67

68
  @type cfg: L{config.ConfigWriter}
69
  @param cfg: The cluster configuration
70
  @type node: L{objects.Node}
71
  @param node: The node
72
  @return: The OOB script if supported or an empty string otherwise
73

74
  """
75
  return cfg.GetNdParams(node)[constants.ND_OOB_PROGRAM]
76

    
77

    
78
class ResultWithJobs:
79
  """Data container for LU results with jobs.
80

81
  Instances of this class returned from L{LogicalUnit.Exec} will be recognized
82
  by L{mcpu.Processor._ProcessResult}. The latter will then submit the jobs
83
  contained in the C{jobs} attribute and include the job IDs in the opcode
84
  result.
85

86
  """
87
  def __init__(self, jobs, **kwargs):
88
    """Initializes this class.
89

90
    Additional return values can be specified as keyword arguments.
91

92
    @type jobs: list of lists of L{opcode.OpCode}
93
    @param jobs: A list of lists of opcode objects
94

95
    """
96
    self.jobs = jobs
97
    self.other = kwargs
98

    
99

    
100
class LogicalUnit(object):
101
  """Logical Unit base class.
102

103
  Subclasses must follow these rules:
104
    - implement ExpandNames
105
    - implement CheckPrereq (except when tasklets are used)
106
    - implement Exec (except when tasklets are used)
107
    - implement BuildHooksEnv
108
    - implement BuildHooksNodes
109
    - redefine HPATH and HTYPE
110
    - optionally redefine their run requirements:
111
        REQ_BGL: the LU needs to hold the Big Ganeti Lock exclusively
112

113
  Note that all commands require root permissions.
114

115
  @ivar dry_run_result: the value (if any) that will be returned to the caller
116
      in dry-run mode (signalled by opcode dry_run parameter)
117

118
  """
119
  HPATH = None
120
  HTYPE = None
121
  REQ_BGL = True
122

    
123
  def __init__(self, processor, op, context, rpc):
124
    """Constructor for LogicalUnit.
125

126
    This needs to be overridden in derived classes in order to check op
127
    validity.
128

129
    """
130
    self.proc = processor
131
    self.op = op
132
    self.cfg = context.cfg
133
    self.glm = context.glm
134
    self.context = context
135
    self.rpc = rpc
136
    # Dicts used to declare locking needs to mcpu
137
    self.needed_locks = None
138
    self.share_locks = dict.fromkeys(locking.LEVELS, 0)
139
    self.add_locks = {}
140
    self.remove_locks = {}
141
    # Used to force good behavior when calling helper functions
142
    self.recalculate_locks = {}
143
    # logging
144
    self.Log = processor.Log # pylint: disable-msg=C0103
145
    self.LogWarning = processor.LogWarning # pylint: disable-msg=C0103
146
    self.LogInfo = processor.LogInfo # pylint: disable-msg=C0103
147
    self.LogStep = processor.LogStep # pylint: disable-msg=C0103
148
    # support for dry-run
149
    self.dry_run_result = None
150
    # support for generic debug attribute
151
    if (not hasattr(self.op, "debug_level") or
152
        not isinstance(self.op.debug_level, int)):
153
      self.op.debug_level = 0
154

    
155
    # Tasklets
156
    self.tasklets = None
157

    
158
    # Validate opcode parameters and set defaults
159
    self.op.Validate(True)
160

    
161
    self.CheckArguments()
162

    
163
  def CheckArguments(self):
164
    """Check syntactic validity for the opcode arguments.
165

166
    This method is for doing a simple syntactic check and ensure
167
    validity of opcode parameters, without any cluster-related
168
    checks. While the same can be accomplished in ExpandNames and/or
169
    CheckPrereq, doing these separate is better because:
170

171
      - ExpandNames is left as as purely a lock-related function
172
      - CheckPrereq is run after we have acquired locks (and possible
173
        waited for them)
174

175
    The function is allowed to change the self.op attribute so that
176
    later methods can no longer worry about missing parameters.
177

178
    """
179
    pass
180

    
181
  def ExpandNames(self):
182
    """Expand names for this LU.
183

184
    This method is called before starting to execute the opcode, and it should
185
    update all the parameters of the opcode to their canonical form (e.g. a
186
    short node name must be fully expanded after this method has successfully
187
    completed). This way locking, hooks, logging, etc. can work correctly.
188

189
    LUs which implement this method must also populate the self.needed_locks
190
    member, as a dict with lock levels as keys, and a list of needed lock names
191
    as values. Rules:
192

193
      - use an empty dict if you don't need any lock
194
      - if you don't need any lock at a particular level omit that level
195
      - don't put anything for the BGL level
196
      - if you want all locks at a level use locking.ALL_SET as a value
197

198
    If you need to share locks (rather than acquire them exclusively) at one
199
    level you can modify self.share_locks, setting a true value (usually 1) for
200
    that level. By default locks are not shared.
201

202
    This function can also define a list of tasklets, which then will be
203
    executed in order instead of the usual LU-level CheckPrereq and Exec
204
    functions, if those are not defined by the LU.
205

206
    Examples::
207

208
      # Acquire all nodes and one instance
209
      self.needed_locks = {
210
        locking.LEVEL_NODE: locking.ALL_SET,
211
        locking.LEVEL_INSTANCE: ['instance1.example.com'],
212
      }
213
      # Acquire just two nodes
214
      self.needed_locks = {
215
        locking.LEVEL_NODE: ['node1.example.com', 'node2.example.com'],
216
      }
217
      # Acquire no locks
218
      self.needed_locks = {} # No, you can't leave it to the default value None
219

220
    """
221
    # The implementation of this method is mandatory only if the new LU is
222
    # concurrent, so that old LUs don't need to be changed all at the same
223
    # time.
224
    if self.REQ_BGL:
225
      self.needed_locks = {} # Exclusive LUs don't need locks.
226
    else:
227
      raise NotImplementedError
228

    
229
  def DeclareLocks(self, level):
230
    """Declare LU locking needs for a level
231

232
    While most LUs can just declare their locking needs at ExpandNames time,
233
    sometimes there's the need to calculate some locks after having acquired
234
    the ones before. This function is called just before acquiring locks at a
235
    particular level, but after acquiring the ones at lower levels, and permits
236
    such calculations. It can be used to modify self.needed_locks, and by
237
    default it does nothing.
238

239
    This function is only called if you have something already set in
240
    self.needed_locks for the level.
241

242
    @param level: Locking level which is going to be locked
243
    @type level: member of ganeti.locking.LEVELS
244

245
    """
246

    
247
  def CheckPrereq(self):
248
    """Check prerequisites for this LU.
249

250
    This method should check that the prerequisites for the execution
251
    of this LU are fulfilled. It can do internode communication, but
252
    it should be idempotent - no cluster or system changes are
253
    allowed.
254

255
    The method should raise errors.OpPrereqError in case something is
256
    not fulfilled. Its return value is ignored.
257

258
    This method should also update all the parameters of the opcode to
259
    their canonical form if it hasn't been done by ExpandNames before.
260

261
    """
262
    if self.tasklets is not None:
263
      for (idx, tl) in enumerate(self.tasklets):
264
        logging.debug("Checking prerequisites for tasklet %s/%s",
265
                      idx + 1, len(self.tasklets))
266
        tl.CheckPrereq()
267
    else:
268
      pass
269

    
270
  def Exec(self, feedback_fn):
271
    """Execute the LU.
272

273
    This method should implement the actual work. It should raise
274
    errors.OpExecError for failures that are somewhat dealt with in
275
    code, or expected.
276

277
    """
278
    if self.tasklets is not None:
279
      for (idx, tl) in enumerate(self.tasklets):
280
        logging.debug("Executing tasklet %s/%s", idx + 1, len(self.tasklets))
281
        tl.Exec(feedback_fn)
282
    else:
283
      raise NotImplementedError
284

    
285
  def BuildHooksEnv(self):
286
    """Build hooks environment for this LU.
287

288
    @rtype: dict
289
    @return: Dictionary containing the environment that will be used for
290
      running the hooks for this LU. The keys of the dict must not be prefixed
291
      with "GANETI_"--that'll be added by the hooks runner. The hooks runner
292
      will extend the environment with additional variables. If no environment
293
      should be defined, an empty dictionary should be returned (not C{None}).
294
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
295
      will not be called.
296

297
    """
298
    raise NotImplementedError
299

    
300
  def BuildHooksNodes(self):
301
    """Build list of nodes to run LU's hooks.
302

303
    @rtype: tuple; (list, list)
304
    @return: Tuple containing a list of node names on which the hook
305
      should run before the execution and a list of node names on which the
306
      hook should run after the execution. No nodes should be returned as an
307
      empty list (and not None).
308
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
309
      will not be called.
310

311
    """
312
    raise NotImplementedError
313

    
314
  def HooksCallBack(self, phase, hook_results, feedback_fn, lu_result):
315
    """Notify the LU about the results of its hooks.
316

317
    This method is called every time a hooks phase is executed, and notifies
318
    the Logical Unit about the hooks' result. The LU can then use it to alter
319
    its result based on the hooks.  By default the method does nothing and the
320
    previous result is passed back unchanged but any LU can define it if it
321
    wants to use the local cluster hook-scripts somehow.
322

323
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
324
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
325
    @param hook_results: the results of the multi-node hooks rpc call
326
    @param feedback_fn: function used send feedback back to the caller
327
    @param lu_result: the previous Exec result this LU had, or None
328
        in the PRE phase
329
    @return: the new Exec result, based on the previous result
330
        and hook results
331

332
    """
333
    # API must be kept, thus we ignore the unused argument and could
334
    # be a function warnings
335
    # pylint: disable-msg=W0613,R0201
336
    return lu_result
337

    
338
  def _ExpandAndLockInstance(self):
339
    """Helper function to expand and lock an instance.
340

341
    Many LUs that work on an instance take its name in self.op.instance_name
342
    and need to expand it and then declare the expanded name for locking. This
343
    function does it, and then updates self.op.instance_name to the expanded
344
    name. It also initializes needed_locks as a dict, if this hasn't been done
345
    before.
346

347
    """
348
    if self.needed_locks is None:
349
      self.needed_locks = {}
350
    else:
351
      assert locking.LEVEL_INSTANCE not in self.needed_locks, \
352
        "_ExpandAndLockInstance called with instance-level locks set"
353
    self.op.instance_name = _ExpandInstanceName(self.cfg,
354
                                                self.op.instance_name)
355
    self.needed_locks[locking.LEVEL_INSTANCE] = self.op.instance_name
356

    
357
  def _LockInstancesNodes(self, primary_only=False):
358
    """Helper function to declare instances' nodes for locking.
359

360
    This function should be called after locking one or more instances to lock
361
    their nodes. Its effect is populating self.needed_locks[locking.LEVEL_NODE]
362
    with all primary or secondary nodes for instances already locked and
363
    present in self.needed_locks[locking.LEVEL_INSTANCE].
364

365
    It should be called from DeclareLocks, and for safety only works if
366
    self.recalculate_locks[locking.LEVEL_NODE] is set.
367

368
    In the future it may grow parameters to just lock some instance's nodes, or
369
    to just lock primaries or secondary nodes, if needed.
370

371
    If should be called in DeclareLocks in a way similar to::
372

373
      if level == locking.LEVEL_NODE:
374
        self._LockInstancesNodes()
375

376
    @type primary_only: boolean
377
    @param primary_only: only lock primary nodes of locked instances
378

379
    """
380
    assert locking.LEVEL_NODE in self.recalculate_locks, \
381
      "_LockInstancesNodes helper function called with no nodes to recalculate"
382

    
383
    # TODO: check if we're really been called with the instance locks held
384

    
385
    # For now we'll replace self.needed_locks[locking.LEVEL_NODE], but in the
386
    # future we might want to have different behaviors depending on the value
387
    # of self.recalculate_locks[locking.LEVEL_NODE]
388
    wanted_nodes = []
389
    for instance_name in self.glm.list_owned(locking.LEVEL_INSTANCE):
390
      instance = self.context.cfg.GetInstanceInfo(instance_name)
391
      wanted_nodes.append(instance.primary_node)
392
      if not primary_only:
393
        wanted_nodes.extend(instance.secondary_nodes)
394

    
395
    if self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_REPLACE:
396
      self.needed_locks[locking.LEVEL_NODE] = wanted_nodes
397
    elif self.recalculate_locks[locking.LEVEL_NODE] == constants.LOCKS_APPEND:
398
      self.needed_locks[locking.LEVEL_NODE].extend(wanted_nodes)
399

    
400
    del self.recalculate_locks[locking.LEVEL_NODE]
401

    
402

    
403
class NoHooksLU(LogicalUnit): # pylint: disable-msg=W0223
404
  """Simple LU which runs no hooks.
405

406
  This LU is intended as a parent for other LogicalUnits which will
407
  run no hooks, in order to reduce duplicate code.
408

409
  """
410
  HPATH = None
411
  HTYPE = None
412

    
413
  def BuildHooksEnv(self):
414
    """Empty BuildHooksEnv for NoHooksLu.
415

416
    This just raises an error.
417

418
    """
419
    raise AssertionError("BuildHooksEnv called for NoHooksLUs")
420

    
421
  def BuildHooksNodes(self):
422
    """Empty BuildHooksNodes for NoHooksLU.
423

424
    """
425
    raise AssertionError("BuildHooksNodes called for NoHooksLU")
426

    
427

    
428
class Tasklet:
429
  """Tasklet base class.
430

431
  Tasklets are subcomponents for LUs. LUs can consist entirely of tasklets or
432
  they can mix legacy code with tasklets. Locking needs to be done in the LU,
433
  tasklets know nothing about locks.
434

435
  Subclasses must follow these rules:
436
    - Implement CheckPrereq
437
    - Implement Exec
438

439
  """
440
  def __init__(self, lu):
441
    self.lu = lu
442

    
443
    # Shortcuts
444
    self.cfg = lu.cfg
445
    self.rpc = lu.rpc
446

    
447
  def CheckPrereq(self):
448
    """Check prerequisites for this tasklets.
449

450
    This method should check whether the prerequisites for the execution of
451
    this tasklet are fulfilled. It can do internode communication, but it
452
    should be idempotent - no cluster or system changes are allowed.
453

454
    The method should raise errors.OpPrereqError in case something is not
455
    fulfilled. Its return value is ignored.
456

457
    This method should also update all parameters to their canonical form if it
458
    hasn't been done before.
459

460
    """
461
    pass
462

    
463
  def Exec(self, feedback_fn):
464
    """Execute the tasklet.
465

466
    This method should implement the actual work. It should raise
467
    errors.OpExecError for failures that are somewhat dealt with in code, or
468
    expected.
469

470
    """
471
    raise NotImplementedError
472

    
473

    
474
class _QueryBase:
475
  """Base for query utility classes.
476

477
  """
478
  #: Attribute holding field definitions
479
  FIELDS = None
480

    
481
  def __init__(self, filter_, fields, use_locking):
482
    """Initializes this class.
483

484
    """
485
    self.use_locking = use_locking
486

    
487
    self.query = query.Query(self.FIELDS, fields, filter_=filter_,
488
                             namefield="name")
489
    self.requested_data = self.query.RequestedData()
490
    self.names = self.query.RequestedNames()
491

    
492
    # Sort only if no names were requested
493
    self.sort_by_name = not self.names
494

    
495
    self.do_locking = None
496
    self.wanted = None
497

    
498
  def _GetNames(self, lu, all_names, lock_level):
499
    """Helper function to determine names asked for in the query.
500

501
    """
502
    if self.do_locking:
503
      names = lu.glm.list_owned(lock_level)
504
    else:
505
      names = all_names
506

    
507
    if self.wanted == locking.ALL_SET:
508
      assert not self.names
509
      # caller didn't specify names, so ordering is not important
510
      return utils.NiceSort(names)
511

    
512
    # caller specified names and we must keep the same order
513
    assert self.names
514
    assert not self.do_locking or lu.glm.is_owned(lock_level)
515

    
516
    missing = set(self.wanted).difference(names)
517
    if missing:
518
      raise errors.OpExecError("Some items were removed before retrieving"
519
                               " their data: %s" % missing)
520

    
521
    # Return expanded names
522
    return self.wanted
523

    
524
  def ExpandNames(self, lu):
525
    """Expand names for this query.
526

527
    See L{LogicalUnit.ExpandNames}.
528

529
    """
530
    raise NotImplementedError()
531

    
532
  def DeclareLocks(self, lu, level):
533
    """Declare locks for this query.
534

535
    See L{LogicalUnit.DeclareLocks}.
536

537
    """
538
    raise NotImplementedError()
539

    
540
  def _GetQueryData(self, lu):
541
    """Collects all data for this query.
542

543
    @return: Query data object
544

545
    """
546
    raise NotImplementedError()
547

    
548
  def NewStyleQuery(self, lu):
549
    """Collect data and execute query.
550

551
    """
552
    return query.GetQueryResponse(self.query, self._GetQueryData(lu),
553
                                  sort_by_name=self.sort_by_name)
554

    
555
  def OldStyleQuery(self, lu):
556
    """Collect data and execute query.
557

558
    """
559
    return self.query.OldStyleQuery(self._GetQueryData(lu),
560
                                    sort_by_name=self.sort_by_name)
561

    
562

    
563
def _GetWantedNodes(lu, nodes):
564
  """Returns list of checked and expanded node names.
565

566
  @type lu: L{LogicalUnit}
567
  @param lu: the logical unit on whose behalf we execute
568
  @type nodes: list
569
  @param nodes: list of node names or None for all nodes
570
  @rtype: list
571
  @return: the list of nodes, sorted
572
  @raise errors.ProgrammerError: if the nodes parameter is wrong type
573

574
  """
575
  if nodes:
576
    return [_ExpandNodeName(lu.cfg, name) for name in nodes]
577

    
578
  return utils.NiceSort(lu.cfg.GetNodeList())
579

    
580

    
581
def _GetWantedInstances(lu, instances):
582
  """Returns list of checked and expanded instance names.
583

584
  @type lu: L{LogicalUnit}
585
  @param lu: the logical unit on whose behalf we execute
586
  @type instances: list
587
  @param instances: list of instance names or None for all instances
588
  @rtype: list
589
  @return: the list of instances, sorted
590
  @raise errors.OpPrereqError: if the instances parameter is wrong type
591
  @raise errors.OpPrereqError: if any of the passed instances is not found
592

593
  """
594
  if instances:
595
    wanted = [_ExpandInstanceName(lu.cfg, name) for name in instances]
596
  else:
597
    wanted = utils.NiceSort(lu.cfg.GetInstanceList())
598
  return wanted
599

    
600

    
601
def _GetUpdatedParams(old_params, update_dict,
602
                      use_default=True, use_none=False):
603
  """Return the new version of a parameter dictionary.
604

605
  @type old_params: dict
606
  @param old_params: old parameters
607
  @type update_dict: dict
608
  @param update_dict: dict containing new parameter values, or
609
      constants.VALUE_DEFAULT to reset the parameter to its default
610
      value
611
  @param use_default: boolean
612
  @type use_default: whether to recognise L{constants.VALUE_DEFAULT}
613
      values as 'to be deleted' values
614
  @param use_none: boolean
615
  @type use_none: whether to recognise C{None} values as 'to be
616
      deleted' values
617
  @rtype: dict
618
  @return: the new parameter dictionary
619

620
  """
621
  params_copy = copy.deepcopy(old_params)
622
  for key, val in update_dict.iteritems():
623
    if ((use_default and val == constants.VALUE_DEFAULT) or
624
        (use_none and val is None)):
625
      try:
626
        del params_copy[key]
627
      except KeyError:
628
        pass
629
    else:
630
      params_copy[key] = val
631
  return params_copy
632

    
633

    
634
def _ReleaseLocks(lu, level, names=None, keep=None):
635
  """Releases locks owned by an LU.
636

637
  @type lu: L{LogicalUnit}
638
  @param level: Lock level
639
  @type names: list or None
640
  @param names: Names of locks to release
641
  @type keep: list or None
642
  @param keep: Names of locks to retain
643

644
  """
645
  assert not (keep is not None and names is not None), \
646
         "Only one of the 'names' and the 'keep' parameters can be given"
647

    
648
  if names is not None:
649
    should_release = names.__contains__
650
  elif keep:
651
    should_release = lambda name: name not in keep
652
  else:
653
    should_release = None
654

    
655
  if should_release:
656
    retain = []
657
    release = []
658

    
659
    # Determine which locks to release
660
    for name in lu.glm.list_owned(level):
661
      if should_release(name):
662
        release.append(name)
663
      else:
664
        retain.append(name)
665

    
666
    assert len(lu.glm.list_owned(level)) == (len(retain) + len(release))
667

    
668
    # Release just some locks
669
    lu.glm.release(level, names=release)
670

    
671
    assert frozenset(lu.glm.list_owned(level)) == frozenset(retain)
672
  else:
673
    # Release everything
674
    lu.glm.release(level)
675

    
676
    assert not lu.glm.is_owned(level), "No locks should be owned"
677

    
678

    
679
def _RunPostHook(lu, node_name):
680
  """Runs the post-hook for an opcode on a single node.
681

682
  """
683
  hm = lu.proc.hmclass(lu.rpc.call_hooks_runner, lu)
684
  try:
685
    hm.RunPhase(constants.HOOKS_PHASE_POST, nodes=[node_name])
686
  except:
687
    # pylint: disable-msg=W0702
688
    lu.LogWarning("Errors occurred running hooks on %s" % node_name)
689

    
690

    
691
def _CheckOutputFields(static, dynamic, selected):
692
  """Checks whether all selected fields are valid.
693

694
  @type static: L{utils.FieldSet}
695
  @param static: static fields set
696
  @type dynamic: L{utils.FieldSet}
697
  @param dynamic: dynamic fields set
698

699
  """
700
  f = utils.FieldSet()
701
  f.Extend(static)
702
  f.Extend(dynamic)
703

    
704
  delta = f.NonMatching(selected)
705
  if delta:
706
    raise errors.OpPrereqError("Unknown output fields selected: %s"
707
                               % ",".join(delta), errors.ECODE_INVAL)
708

    
709

    
710
def _CheckGlobalHvParams(params):
711
  """Validates that given hypervisor params are not global ones.
712

713
  This will ensure that instances don't get customised versions of
714
  global params.
715

716
  """
717
  used_globals = constants.HVC_GLOBALS.intersection(params)
718
  if used_globals:
719
    msg = ("The following hypervisor parameters are global and cannot"
720
           " be customized at instance level, please modify them at"
721
           " cluster level: %s" % utils.CommaJoin(used_globals))
722
    raise errors.OpPrereqError(msg, errors.ECODE_INVAL)
723

    
724

    
725
def _CheckNodeOnline(lu, node, msg=None):
726
  """Ensure that a given node is online.
727

728
  @param lu: the LU on behalf of which we make the check
729
  @param node: the node to check
730
  @param msg: if passed, should be a message to replace the default one
731
  @raise errors.OpPrereqError: if the node is offline
732

733
  """
734
  if msg is None:
735
    msg = "Can't use offline node"
736
  if lu.cfg.GetNodeInfo(node).offline:
737
    raise errors.OpPrereqError("%s: %s" % (msg, node), errors.ECODE_STATE)
738

    
739

    
740
def _CheckNodeNotDrained(lu, node):
741
  """Ensure that a given node is not drained.
742

743
  @param lu: the LU on behalf of which we make the check
744
  @param node: the node to check
745
  @raise errors.OpPrereqError: if the node is drained
746

747
  """
748
  if lu.cfg.GetNodeInfo(node).drained:
749
    raise errors.OpPrereqError("Can't use drained node %s" % node,
750
                               errors.ECODE_STATE)
751

    
752

    
753
def _CheckNodeVmCapable(lu, node):
754
  """Ensure that a given node is vm capable.
755

756
  @param lu: the LU on behalf of which we make the check
757
  @param node: the node to check
758
  @raise errors.OpPrereqError: if the node is not vm capable
759

760
  """
761
  if not lu.cfg.GetNodeInfo(node).vm_capable:
762
    raise errors.OpPrereqError("Can't use non-vm_capable node %s" % node,
763
                               errors.ECODE_STATE)
764

    
765

    
766
def _CheckNodeHasOS(lu, node, os_name, force_variant):
767
  """Ensure that a node supports a given OS.
768

769
  @param lu: the LU on behalf of which we make the check
770
  @param node: the node to check
771
  @param os_name: the OS to query about
772
  @param force_variant: whether to ignore variant errors
773
  @raise errors.OpPrereqError: if the node is not supporting the OS
774

775
  """
776
  result = lu.rpc.call_os_get(node, os_name)
777
  result.Raise("OS '%s' not in supported OS list for node %s" %
778
               (os_name, node),
779
               prereq=True, ecode=errors.ECODE_INVAL)
780
  if not force_variant:
781
    _CheckOSVariant(result.payload, os_name)
782

    
783

    
784
def _CheckNodeHasSecondaryIP(lu, node, secondary_ip, prereq):
785
  """Ensure that a node has the given secondary ip.
786

787
  @type lu: L{LogicalUnit}
788
  @param lu: the LU on behalf of which we make the check
789
  @type node: string
790
  @param node: the node to check
791
  @type secondary_ip: string
792
  @param secondary_ip: the ip to check
793
  @type prereq: boolean
794
  @param prereq: whether to throw a prerequisite or an execute error
795
  @raise errors.OpPrereqError: if the node doesn't have the ip, and prereq=True
796
  @raise errors.OpExecError: if the node doesn't have the ip, and prereq=False
797

798
  """
799
  result = lu.rpc.call_node_has_ip_address(node, secondary_ip)
800
  result.Raise("Failure checking secondary ip on node %s" % node,
801
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
802
  if not result.payload:
803
    msg = ("Node claims it doesn't have the secondary ip you gave (%s),"
804
           " please fix and re-run this command" % secondary_ip)
805
    if prereq:
806
      raise errors.OpPrereqError(msg, errors.ECODE_ENVIRON)
807
    else:
808
      raise errors.OpExecError(msg)
809

    
810

    
811
def _GetClusterDomainSecret():
812
  """Reads the cluster domain secret.
813

814
  """
815
  return utils.ReadOneLineFile(constants.CLUSTER_DOMAIN_SECRET_FILE,
816
                               strict=True)
817

    
818

    
819
def _CheckInstanceDown(lu, instance, reason):
820
  """Ensure that an instance is not running."""
821
  if instance.admin_up:
822
    raise errors.OpPrereqError("Instance %s is marked to be up, %s" %
823
                               (instance.name, reason), errors.ECODE_STATE)
824

    
825
  pnode = instance.primary_node
826
  ins_l = lu.rpc.call_instance_list([pnode], [instance.hypervisor])[pnode]
827
  ins_l.Raise("Can't contact node %s for instance information" % pnode,
828
              prereq=True, ecode=errors.ECODE_ENVIRON)
829

    
830
  if instance.name in ins_l.payload:
831
    raise errors.OpPrereqError("Instance %s is running, %s" %
832
                               (instance.name, reason), errors.ECODE_STATE)
833

    
834

    
835
def _ExpandItemName(fn, name, kind):
836
  """Expand an item name.
837

838
  @param fn: the function to use for expansion
839
  @param name: requested item name
840
  @param kind: text description ('Node' or 'Instance')
841
  @return: the resolved (full) name
842
  @raise errors.OpPrereqError: if the item is not found
843

844
  """
845
  full_name = fn(name)
846
  if full_name is None:
847
    raise errors.OpPrereqError("%s '%s' not known" % (kind, name),
848
                               errors.ECODE_NOENT)
849
  return full_name
850

    
851

    
852
def _ExpandNodeName(cfg, name):
853
  """Wrapper over L{_ExpandItemName} for nodes."""
854
  return _ExpandItemName(cfg.ExpandNodeName, name, "Node")
855

    
856

    
857
def _ExpandInstanceName(cfg, name):
858
  """Wrapper over L{_ExpandItemName} for instance."""
859
  return _ExpandItemName(cfg.ExpandInstanceName, name, "Instance")
860

    
861

    
862
def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status,
863
                          memory, vcpus, nics, disk_template, disks,
864
                          bep, hvp, hypervisor_name, tags):
865
  """Builds instance related env variables for hooks
866

867
  This builds the hook environment from individual variables.
868

869
  @type name: string
870
  @param name: the name of the instance
871
  @type primary_node: string
872
  @param primary_node: the name of the instance's primary node
873
  @type secondary_nodes: list
874
  @param secondary_nodes: list of secondary nodes as strings
875
  @type os_type: string
876
  @param os_type: the name of the instance's OS
877
  @type status: boolean
878
  @param status: the should_run status of the instance
879
  @type memory: string
880
  @param memory: the memory size of the instance
881
  @type vcpus: string
882
  @param vcpus: the count of VCPUs the instance has
883
  @type nics: list
884
  @param nics: list of tuples (ip, mac, mode, link) representing
885
      the NICs the instance has
886
  @type disk_template: string
887
  @param disk_template: the disk template of the instance
888
  @type disks: list
889
  @param disks: the list of (size, mode) pairs
890
  @type bep: dict
891
  @param bep: the backend parameters for the instance
892
  @type hvp: dict
893
  @param hvp: the hypervisor parameters for the instance
894
  @type hypervisor_name: string
895
  @param hypervisor_name: the hypervisor for the instance
896
  @type tags: list
897
  @param tags: list of instance tags as strings
898
  @rtype: dict
899
  @return: the hook environment for this instance
900

901
  """
902
  if status:
903
    str_status = "up"
904
  else:
905
    str_status = "down"
906
  env = {
907
    "OP_TARGET": name,
908
    "INSTANCE_NAME": name,
909
    "INSTANCE_PRIMARY": primary_node,
910
    "INSTANCE_SECONDARIES": " ".join(secondary_nodes),
911
    "INSTANCE_OS_TYPE": os_type,
912
    "INSTANCE_STATUS": str_status,
913
    "INSTANCE_MEMORY": memory,
914
    "INSTANCE_VCPUS": vcpus,
915
    "INSTANCE_DISK_TEMPLATE": disk_template,
916
    "INSTANCE_HYPERVISOR": hypervisor_name,
917
  }
918

    
919
  if nics:
920
    nic_count = len(nics)
921
    for idx, (ip, mac, mode, link) in enumerate(nics):
922
      if ip is None:
923
        ip = ""
924
      env["INSTANCE_NIC%d_IP" % idx] = ip
925
      env["INSTANCE_NIC%d_MAC" % idx] = mac
926
      env["INSTANCE_NIC%d_MODE" % idx] = mode
927
      env["INSTANCE_NIC%d_LINK" % idx] = link
928
      if mode == constants.NIC_MODE_BRIDGED:
929
        env["INSTANCE_NIC%d_BRIDGE" % idx] = link
930
  else:
931
    nic_count = 0
932

    
933
  env["INSTANCE_NIC_COUNT"] = nic_count
934

    
935
  if disks:
936
    disk_count = len(disks)
937
    for idx, (size, mode) in enumerate(disks):
938
      env["INSTANCE_DISK%d_SIZE" % idx] = size
939
      env["INSTANCE_DISK%d_MODE" % idx] = mode
940
  else:
941
    disk_count = 0
942

    
943
  env["INSTANCE_DISK_COUNT"] = disk_count
944

    
945
  if not tags:
946
    tags = []
947

    
948
  env["INSTANCE_TAGS"] = " ".join(tags)
949

    
950
  for source, kind in [(bep, "BE"), (hvp, "HV")]:
951
    for key, value in source.items():
952
      env["INSTANCE_%s_%s" % (kind, key)] = value
953

    
954
  return env
955

    
956

    
957
def _NICListToTuple(lu, nics):
958
  """Build a list of nic information tuples.
959

960
  This list is suitable to be passed to _BuildInstanceHookEnv or as a return
961
  value in LUInstanceQueryData.
962

963
  @type lu:  L{LogicalUnit}
964
  @param lu: the logical unit on whose behalf we execute
965
  @type nics: list of L{objects.NIC}
966
  @param nics: list of nics to convert to hooks tuples
967

968
  """
969
  hooks_nics = []
970
  cluster = lu.cfg.GetClusterInfo()
971
  for nic in nics:
972
    ip = nic.ip
973
    mac = nic.mac
974
    filled_params = cluster.SimpleFillNIC(nic.nicparams)
975
    mode = filled_params[constants.NIC_MODE]
976
    link = filled_params[constants.NIC_LINK]
977
    hooks_nics.append((ip, mac, mode, link))
978
  return hooks_nics
979

    
980

    
981
def _BuildInstanceHookEnvByObject(lu, instance, override=None):
982
  """Builds instance related env variables for hooks from an object.
983

984
  @type lu: L{LogicalUnit}
985
  @param lu: the logical unit on whose behalf we execute
986
  @type instance: L{objects.Instance}
987
  @param instance: the instance for which we should build the
988
      environment
989
  @type override: dict
990
  @param override: dictionary with key/values that will override
991
      our values
992
  @rtype: dict
993
  @return: the hook environment dictionary
994

995
  """
996
  cluster = lu.cfg.GetClusterInfo()
997
  bep = cluster.FillBE(instance)
998
  hvp = cluster.FillHV(instance)
999
  args = {
1000
    'name': instance.name,
1001
    'primary_node': instance.primary_node,
1002
    'secondary_nodes': instance.secondary_nodes,
1003
    'os_type': instance.os,
1004
    'status': instance.admin_up,
1005
    'memory': bep[constants.BE_MEMORY],
1006
    'vcpus': bep[constants.BE_VCPUS],
1007
    'nics': _NICListToTuple(lu, instance.nics),
1008
    'disk_template': instance.disk_template,
1009
    'disks': [(disk.size, disk.mode) for disk in instance.disks],
1010
    'bep': bep,
1011
    'hvp': hvp,
1012
    'hypervisor_name': instance.hypervisor,
1013
    'tags': instance.tags,
1014
  }
1015
  if override:
1016
    args.update(override)
1017
  return _BuildInstanceHookEnv(**args) # pylint: disable-msg=W0142
1018

    
1019

    
1020
def _AdjustCandidatePool(lu, exceptions):
1021
  """Adjust the candidate pool after node operations.
1022

1023
  """
1024
  mod_list = lu.cfg.MaintainCandidatePool(exceptions)
1025
  if mod_list:
1026
    lu.LogInfo("Promoted nodes to master candidate role: %s",
1027
               utils.CommaJoin(node.name for node in mod_list))
1028
    for name in mod_list:
1029
      lu.context.ReaddNode(name)
1030
  mc_now, mc_max, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1031
  if mc_now > mc_max:
1032
    lu.LogInfo("Note: more nodes are candidates (%d) than desired (%d)" %
1033
               (mc_now, mc_max))
1034

    
1035

    
1036
def _DecideSelfPromotion(lu, exceptions=None):
1037
  """Decide whether I should promote myself as a master candidate.
1038

1039
  """
1040
  cp_size = lu.cfg.GetClusterInfo().candidate_pool_size
1041
  mc_now, mc_should, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1042
  # the new node will increase mc_max with one, so:
1043
  mc_should = min(mc_should + 1, cp_size)
1044
  return mc_now < mc_should
1045

    
1046

    
1047
def _CheckNicsBridgesExist(lu, target_nics, target_node):
1048
  """Check that the brigdes needed by a list of nics exist.
1049

1050
  """
1051
  cluster = lu.cfg.GetClusterInfo()
1052
  paramslist = [cluster.SimpleFillNIC(nic.nicparams) for nic in target_nics]
1053
  brlist = [params[constants.NIC_LINK] for params in paramslist
1054
            if params[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED]
1055
  if brlist:
1056
    result = lu.rpc.call_bridges_exist(target_node, brlist)
1057
    result.Raise("Error checking bridges on destination node '%s'" %
1058
                 target_node, prereq=True, ecode=errors.ECODE_ENVIRON)
1059

    
1060

    
1061
def _CheckInstanceBridgesExist(lu, instance, node=None):
1062
  """Check that the brigdes needed by an instance exist.
1063

1064
  """
1065
  if node is None:
1066
    node = instance.primary_node
1067
  _CheckNicsBridgesExist(lu, instance.nics, node)
1068

    
1069

    
1070
def _CheckOSVariant(os_obj, name):
1071
  """Check whether an OS name conforms to the os variants specification.
1072

1073
  @type os_obj: L{objects.OS}
1074
  @param os_obj: OS object to check
1075
  @type name: string
1076
  @param name: OS name passed by the user, to check for validity
1077

1078
  """
1079
  if not os_obj.supported_variants:
1080
    return
1081
  variant = objects.OS.GetVariant(name)
1082
  if not variant:
1083
    raise errors.OpPrereqError("OS name must include a variant",
1084
                               errors.ECODE_INVAL)
1085

    
1086
  if variant not in os_obj.supported_variants:
1087
    raise errors.OpPrereqError("Unsupported OS variant", errors.ECODE_INVAL)
1088

    
1089

    
1090
def _GetNodeInstancesInner(cfg, fn):
1091
  return [i for i in cfg.GetAllInstancesInfo().values() if fn(i)]
1092

    
1093

    
1094
def _GetNodeInstances(cfg, node_name):
1095
  """Returns a list of all primary and secondary instances on a node.
1096

1097
  """
1098

    
1099
  return _GetNodeInstancesInner(cfg, lambda inst: node_name in inst.all_nodes)
1100

    
1101

    
1102
def _GetNodePrimaryInstances(cfg, node_name):
1103
  """Returns primary instances on a node.
1104

1105
  """
1106
  return _GetNodeInstancesInner(cfg,
1107
                                lambda inst: node_name == inst.primary_node)
1108

    
1109

    
1110
def _GetNodeSecondaryInstances(cfg, node_name):
1111
  """Returns secondary instances on a node.
1112

1113
  """
1114
  return _GetNodeInstancesInner(cfg,
1115
                                lambda inst: node_name in inst.secondary_nodes)
1116

    
1117

    
1118
def _GetStorageTypeArgs(cfg, storage_type):
1119
  """Returns the arguments for a storage type.
1120

1121
  """
1122
  # Special case for file storage
1123
  if storage_type == constants.ST_FILE:
1124
    # storage.FileStorage wants a list of storage directories
1125
    return [[cfg.GetFileStorageDir(), cfg.GetSharedFileStorageDir()]]
1126

    
1127
  return []
1128

    
1129

    
1130
def _FindFaultyInstanceDisks(cfg, rpc, instance, node_name, prereq):
1131
  faulty = []
1132

    
1133
  for dev in instance.disks:
1134
    cfg.SetDiskID(dev, node_name)
1135

    
1136
  result = rpc.call_blockdev_getmirrorstatus(node_name, instance.disks)
1137
  result.Raise("Failed to get disk status from node %s" % node_name,
1138
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
1139

    
1140
  for idx, bdev_status in enumerate(result.payload):
1141
    if bdev_status and bdev_status.ldisk_status == constants.LDS_FAULTY:
1142
      faulty.append(idx)
1143

    
1144
  return faulty
1145

    
1146

    
1147
def _CheckIAllocatorOrNode(lu, iallocator_slot, node_slot):
1148
  """Check the sanity of iallocator and node arguments and use the
1149
  cluster-wide iallocator if appropriate.
1150

1151
  Check that at most one of (iallocator, node) is specified. If none is
1152
  specified, then the LU's opcode's iallocator slot is filled with the
1153
  cluster-wide default iallocator.
1154

1155
  @type iallocator_slot: string
1156
  @param iallocator_slot: the name of the opcode iallocator slot
1157
  @type node_slot: string
1158
  @param node_slot: the name of the opcode target node slot
1159

1160
  """
1161
  node = getattr(lu.op, node_slot, None)
1162
  iallocator = getattr(lu.op, iallocator_slot, None)
1163

    
1164
  if node is not None and iallocator is not None:
1165
    raise errors.OpPrereqError("Do not specify both, iallocator and node",
1166
                               errors.ECODE_INVAL)
1167
  elif node is None and iallocator is None:
1168
    default_iallocator = lu.cfg.GetDefaultIAllocator()
1169
    if default_iallocator:
1170
      setattr(lu.op, iallocator_slot, default_iallocator)
1171
    else:
1172
      raise errors.OpPrereqError("No iallocator or node given and no"
1173
                                 " cluster-wide default iallocator found;"
1174
                                 " please specify either an iallocator or a"
1175
                                 " node, or set a cluster-wide default"
1176
                                 " iallocator")
1177

    
1178

    
1179
class LUClusterPostInit(LogicalUnit):
1180
  """Logical unit for running hooks after cluster initialization.
1181

1182
  """
1183
  HPATH = "cluster-init"
1184
  HTYPE = constants.HTYPE_CLUSTER
1185

    
1186
  def BuildHooksEnv(self):
1187
    """Build hooks env.
1188

1189
    """
1190
    return {
1191
      "OP_TARGET": self.cfg.GetClusterName(),
1192
      }
1193

    
1194
  def BuildHooksNodes(self):
1195
    """Build hooks nodes.
1196

1197
    """
1198
    return ([], [self.cfg.GetMasterNode()])
1199

    
1200
  def Exec(self, feedback_fn):
1201
    """Nothing to do.
1202

1203
    """
1204
    return True
1205

    
1206

    
1207
class LUClusterDestroy(LogicalUnit):
1208
  """Logical unit for destroying the cluster.
1209

1210
  """
1211
  HPATH = "cluster-destroy"
1212
  HTYPE = constants.HTYPE_CLUSTER
1213

    
1214
  def BuildHooksEnv(self):
1215
    """Build hooks env.
1216

1217
    """
1218
    return {
1219
      "OP_TARGET": self.cfg.GetClusterName(),
1220
      }
1221

    
1222
  def BuildHooksNodes(self):
1223
    """Build hooks nodes.
1224

1225
    """
1226
    return ([], [])
1227

    
1228
  def CheckPrereq(self):
1229
    """Check prerequisites.
1230

1231
    This checks whether the cluster is empty.
1232

1233
    Any errors are signaled by raising errors.OpPrereqError.
1234

1235
    """
1236
    master = self.cfg.GetMasterNode()
1237

    
1238
    nodelist = self.cfg.GetNodeList()
1239
    if len(nodelist) != 1 or nodelist[0] != master:
1240
      raise errors.OpPrereqError("There are still %d node(s) in"
1241
                                 " this cluster." % (len(nodelist) - 1),
1242
                                 errors.ECODE_INVAL)
1243
    instancelist = self.cfg.GetInstanceList()
1244
    if instancelist:
1245
      raise errors.OpPrereqError("There are still %d instance(s) in"
1246
                                 " this cluster." % len(instancelist),
1247
                                 errors.ECODE_INVAL)
1248

    
1249
  def Exec(self, feedback_fn):
1250
    """Destroys the cluster.
1251

1252
    """
1253
    master = self.cfg.GetMasterNode()
1254

    
1255
    # Run post hooks on master node before it's removed
1256
    _RunPostHook(self, master)
1257

    
1258
    result = self.rpc.call_node_stop_master(master, False)
1259
    result.Raise("Could not disable the master role")
1260

    
1261
    return master
1262

    
1263

    
1264
def _VerifyCertificate(filename):
1265
  """Verifies a certificate for LUClusterVerifyConfig.
1266

1267
  @type filename: string
1268
  @param filename: Path to PEM file
1269

1270
  """
1271
  try:
1272
    cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
1273
                                           utils.ReadFile(filename))
1274
  except Exception, err: # pylint: disable-msg=W0703
1275
    return (LUClusterVerifyConfig.ETYPE_ERROR,
1276
            "Failed to load X509 certificate %s: %s" % (filename, err))
1277

    
1278
  (errcode, msg) = \
1279
    utils.VerifyX509Certificate(cert, constants.SSL_CERT_EXPIRATION_WARN,
1280
                                constants.SSL_CERT_EXPIRATION_ERROR)
1281

    
1282
  if msg:
1283
    fnamemsg = "While verifying %s: %s" % (filename, msg)
1284
  else:
1285
    fnamemsg = None
1286

    
1287
  if errcode is None:
1288
    return (None, fnamemsg)
1289
  elif errcode == utils.CERT_WARNING:
1290
    return (LUClusterVerifyConfig.ETYPE_WARNING, fnamemsg)
1291
  elif errcode == utils.CERT_ERROR:
1292
    return (LUClusterVerifyConfig.ETYPE_ERROR, fnamemsg)
1293

    
1294
  raise errors.ProgrammerError("Unhandled certificate error code %r" % errcode)
1295

    
1296

    
1297
def _GetAllHypervisorParameters(cluster, instances):
1298
  """Compute the set of all hypervisor parameters.
1299

1300
  @type cluster: L{objects.Cluster}
1301
  @param cluster: the cluster object
1302
  @param instances: list of L{objects.Instance}
1303
  @param instances: additional instances from which to obtain parameters
1304
  @rtype: list of (origin, hypervisor, parameters)
1305
  @return: a list with all parameters found, indicating the hypervisor they
1306
       apply to, and the origin (can be "cluster", "os X", or "instance Y")
1307

1308
  """
1309
  hvp_data = []
1310

    
1311
  for hv_name in cluster.enabled_hypervisors:
1312
    hvp_data.append(("cluster", hv_name, cluster.GetHVDefaults(hv_name)))
1313

    
1314
  for os_name, os_hvp in cluster.os_hvp.items():
1315
    for hv_name, hv_params in os_hvp.items():
1316
      if hv_params:
1317
        full_params = cluster.GetHVDefaults(hv_name, os_name=os_name)
1318
        hvp_data.append(("os %s" % os_name, hv_name, full_params))
1319

    
1320
  # TODO: collapse identical parameter values in a single one
1321
  for instance in instances:
1322
    if instance.hvparams:
1323
      hvp_data.append(("instance %s" % instance.name, instance.hypervisor,
1324
                       cluster.FillHV(instance)))
1325

    
1326
  return hvp_data
1327

    
1328

    
1329
class _VerifyErrors(object):
1330
  """Mix-in for cluster/group verify LUs.
1331

1332
  It provides _Error and _ErrorIf, and updates the self.bad boolean. (Expects
1333
  self.op and self._feedback_fn to be available.)
1334

1335
  """
1336
  TCLUSTER = "cluster"
1337
  TNODE = "node"
1338
  TINSTANCE = "instance"
1339

    
1340
  ECLUSTERCFG = (TCLUSTER, "ECLUSTERCFG")
1341
  ECLUSTERCERT = (TCLUSTER, "ECLUSTERCERT")
1342
  ECLUSTERFILECHECK = (TCLUSTER, "ECLUSTERFILECHECK")
1343
  ECLUSTERDANGLINGNODES = (TNODE, "ECLUSTERDANGLINGNODES")
1344
  ECLUSTERDANGLINGINST = (TNODE, "ECLUSTERDANGLINGINST")
1345
  EINSTANCEBADNODE = (TINSTANCE, "EINSTANCEBADNODE")
1346
  EINSTANCEDOWN = (TINSTANCE, "EINSTANCEDOWN")
1347
  EINSTANCELAYOUT = (TINSTANCE, "EINSTANCELAYOUT")
1348
  EINSTANCEMISSINGDISK = (TINSTANCE, "EINSTANCEMISSINGDISK")
1349
  EINSTANCEFAULTYDISK = (TINSTANCE, "EINSTANCEFAULTYDISK")
1350
  EINSTANCEWRONGNODE = (TINSTANCE, "EINSTANCEWRONGNODE")
1351
  EINSTANCESPLITGROUPS = (TINSTANCE, "EINSTANCESPLITGROUPS")
1352
  ENODEDRBD = (TNODE, "ENODEDRBD")
1353
  ENODEDRBDHELPER = (TNODE, "ENODEDRBDHELPER")
1354
  ENODEFILECHECK = (TNODE, "ENODEFILECHECK")
1355
  ENODEHOOKS = (TNODE, "ENODEHOOKS")
1356
  ENODEHV = (TNODE, "ENODEHV")
1357
  ENODELVM = (TNODE, "ENODELVM")
1358
  ENODEN1 = (TNODE, "ENODEN1")
1359
  ENODENET = (TNODE, "ENODENET")
1360
  ENODEOS = (TNODE, "ENODEOS")
1361
  ENODEORPHANINSTANCE = (TNODE, "ENODEORPHANINSTANCE")
1362
  ENODEORPHANLV = (TNODE, "ENODEORPHANLV")
1363
  ENODERPC = (TNODE, "ENODERPC")
1364
  ENODESSH = (TNODE, "ENODESSH")
1365
  ENODEVERSION = (TNODE, "ENODEVERSION")
1366
  ENODESETUP = (TNODE, "ENODESETUP")
1367
  ENODETIME = (TNODE, "ENODETIME")
1368
  ENODEOOBPATH = (TNODE, "ENODEOOBPATH")
1369

    
1370
  ETYPE_FIELD = "code"
1371
  ETYPE_ERROR = "ERROR"
1372
  ETYPE_WARNING = "WARNING"
1373

    
1374
  def _Error(self, ecode, item, msg, *args, **kwargs):
1375
    """Format an error message.
1376

1377
    Based on the opcode's error_codes parameter, either format a
1378
    parseable error code, or a simpler error string.
1379

1380
    This must be called only from Exec and functions called from Exec.
1381

1382
    """
1383
    ltype = kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR)
1384
    itype, etxt = ecode
1385
    # first complete the msg
1386
    if args:
1387
      msg = msg % args
1388
    # then format the whole message
1389
    if self.op.error_codes: # This is a mix-in. pylint: disable-msg=E1101
1390
      msg = "%s:%s:%s:%s:%s" % (ltype, etxt, itype, item, msg)
1391
    else:
1392
      if item:
1393
        item = " " + item
1394
      else:
1395
        item = ""
1396
      msg = "%s: %s%s: %s" % (ltype, itype, item, msg)
1397
    # and finally report it via the feedback_fn
1398
    self._feedback_fn("  - %s" % msg) # Mix-in. pylint: disable-msg=E1101
1399

    
1400
  def _ErrorIf(self, cond, *args, **kwargs):
1401
    """Log an error message if the passed condition is True.
1402

1403
    """
1404
    cond = (bool(cond)
1405
            or self.op.debug_simulate_errors) # pylint: disable-msg=E1101
1406
    if cond:
1407
      self._Error(*args, **kwargs)
1408
    # do not mark the operation as failed for WARN cases only
1409
    if kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR) == self.ETYPE_ERROR:
1410
      self.bad = self.bad or cond
1411

    
1412

    
1413
class LUClusterVerifyConfig(NoHooksLU, _VerifyErrors):
1414
  """Verifies the cluster config.
1415

1416
  """
1417
  REQ_BGL = False
1418

    
1419
  def _VerifyHVP(self, hvp_data):
1420
    """Verifies locally the syntax of the hypervisor parameters.
1421

1422
    """
1423
    for item, hv_name, hv_params in hvp_data:
1424
      msg = ("hypervisor %s parameters syntax check (source %s): %%s" %
1425
             (item, hv_name))
1426
      try:
1427
        hv_class = hypervisor.GetHypervisor(hv_name)
1428
        utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
1429
        hv_class.CheckParameterSyntax(hv_params)
1430
      except errors.GenericError, err:
1431
        self._ErrorIf(True, self.ECLUSTERCFG, None, msg % str(err))
1432

    
1433
  def ExpandNames(self):
1434
    self.all_group_info = self.cfg.GetAllNodeGroupsInfo()
1435
    self.all_node_info = self.cfg.GetAllNodesInfo()
1436
    self.all_inst_info = self.cfg.GetAllInstancesInfo()
1437
    self.needed_locks = {}
1438

    
1439
  def Exec(self, feedback_fn):
1440
    """Verify integrity of cluster, performing various test on nodes.
1441

1442
    """
1443
    self.bad = False
1444
    self._feedback_fn = feedback_fn
1445

    
1446
    feedback_fn("* Verifying cluster config")
1447

    
1448
    for msg in self.cfg.VerifyConfig():
1449
      self._ErrorIf(True, self.ECLUSTERCFG, None, msg)
1450

    
1451
    feedback_fn("* Verifying cluster certificate files")
1452

    
1453
    for cert_filename in constants.ALL_CERT_FILES:
1454
      (errcode, msg) = _VerifyCertificate(cert_filename)
1455
      self._ErrorIf(errcode, self.ECLUSTERCERT, None, msg, code=errcode)
1456

    
1457
    feedback_fn("* Verifying hypervisor parameters")
1458

    
1459
    self._VerifyHVP(_GetAllHypervisorParameters(self.cfg.GetClusterInfo(),
1460
                                                self.all_inst_info.values()))
1461

    
1462
    feedback_fn("* Verifying all nodes belong to an existing group")
1463

    
1464
    # We do this verification here because, should this bogus circumstance
1465
    # occur, it would never be catched by VerifyGroup, which only acts on
1466
    # nodes/instances reachable from existing node groups.
1467

    
1468
    dangling_nodes = set(node.name for node in self.all_node_info.values()
1469
                         if node.group not in self.all_group_info)
1470

    
1471
    dangling_instances = {}
1472
    no_node_instances = []
1473

    
1474
    for inst in self.all_inst_info.values():
1475
      if inst.primary_node in dangling_nodes:
1476
        dangling_instances.setdefault(inst.primary_node, []).append(inst.name)
1477
      elif inst.primary_node not in self.all_node_info:
1478
        no_node_instances.append(inst.name)
1479

    
1480
    pretty_dangling = [
1481
        "%s (%s)" %
1482
        (node.name,
1483
         utils.CommaJoin(dangling_instances.get(node.name,
1484
                                                ["no instances"])))
1485
        for node in dangling_nodes]
1486

    
1487
    self._ErrorIf(bool(dangling_nodes), self.ECLUSTERDANGLINGNODES, None,
1488
                  "the following nodes (and their instances) belong to a non"
1489
                  " existing group: %s", utils.CommaJoin(pretty_dangling))
1490

    
1491
    self._ErrorIf(bool(no_node_instances), self.ECLUSTERDANGLINGINST, None,
1492
                  "the following instances have a non-existing primary-node:"
1493
                  " %s", utils.CommaJoin(no_node_instances))
1494

    
1495
    return (not self.bad, [g.name for g in self.all_group_info.values()])
1496

    
1497

    
1498
class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors):
1499
  """Verifies the status of a node group.
1500

1501
  """
1502
  HPATH = "cluster-verify"
1503
  HTYPE = constants.HTYPE_CLUSTER
1504
  REQ_BGL = False
1505

    
1506
  _HOOKS_INDENT_RE = re.compile("^", re.M)
1507

    
1508
  class NodeImage(object):
1509
    """A class representing the logical and physical status of a node.
1510

1511
    @type name: string
1512
    @ivar name: the node name to which this object refers
1513
    @ivar volumes: a structure as returned from
1514
        L{ganeti.backend.GetVolumeList} (runtime)
1515
    @ivar instances: a list of running instances (runtime)
1516
    @ivar pinst: list of configured primary instances (config)
1517
    @ivar sinst: list of configured secondary instances (config)
1518
    @ivar sbp: dictionary of {primary-node: list of instances} for all
1519
        instances for which this node is secondary (config)
1520
    @ivar mfree: free memory, as reported by hypervisor (runtime)
1521
    @ivar dfree: free disk, as reported by the node (runtime)
1522
    @ivar offline: the offline status (config)
1523
    @type rpc_fail: boolean
1524
    @ivar rpc_fail: whether the RPC verify call was successfull (overall,
1525
        not whether the individual keys were correct) (runtime)
1526
    @type lvm_fail: boolean
1527
    @ivar lvm_fail: whether the RPC call didn't return valid LVM data
1528
    @type hyp_fail: boolean
1529
    @ivar hyp_fail: whether the RPC call didn't return the instance list
1530
    @type ghost: boolean
1531
    @ivar ghost: whether this is a known node or not (config)
1532
    @type os_fail: boolean
1533
    @ivar os_fail: whether the RPC call didn't return valid OS data
1534
    @type oslist: list
1535
    @ivar oslist: list of OSes as diagnosed by DiagnoseOS
1536
    @type vm_capable: boolean
1537
    @ivar vm_capable: whether the node can host instances
1538

1539
    """
1540
    def __init__(self, offline=False, name=None, vm_capable=True):
1541
      self.name = name
1542
      self.volumes = {}
1543
      self.instances = []
1544
      self.pinst = []
1545
      self.sinst = []
1546
      self.sbp = {}
1547
      self.mfree = 0
1548
      self.dfree = 0
1549
      self.offline = offline
1550
      self.vm_capable = vm_capable
1551
      self.rpc_fail = False
1552
      self.lvm_fail = False
1553
      self.hyp_fail = False
1554
      self.ghost = False
1555
      self.os_fail = False
1556
      self.oslist = {}
1557

    
1558
  def ExpandNames(self):
1559
    # This raises errors.OpPrereqError on its own:
1560
    self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name)
1561

    
1562
    all_node_info = self.cfg.GetAllNodesInfo()
1563
    all_inst_info = self.cfg.GetAllInstancesInfo()
1564

    
1565
    node_names = set(node.name
1566
                     for node in all_node_info.values()
1567
                     if node.group == self.group_uuid)
1568

    
1569
    inst_names = [inst.name
1570
                  for inst in all_inst_info.values()
1571
                  if inst.primary_node in node_names]
1572

    
1573
    # In Exec(), we warn about mirrored instances that have primary and
1574
    # secondary living in separate node groups. To fully verify that
1575
    # volumes for these instances are healthy, we will need to do an
1576
    # extra call to their secondaries. We ensure here those nodes will
1577
    # be locked.
1578
    for inst in inst_names:
1579
      if all_inst_info[inst].disk_template in constants.DTS_INT_MIRROR:
1580
        node_names.update(all_inst_info[inst].secondary_nodes)
1581

    
1582
    self.needed_locks = {
1583
      locking.LEVEL_NODEGROUP: [self.group_uuid],
1584
      locking.LEVEL_NODE: list(node_names),
1585
      locking.LEVEL_INSTANCE: inst_names,
1586
    }
1587

    
1588
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
1589

    
1590
  def CheckPrereq(self):
1591
    self.all_node_info = self.cfg.GetAllNodesInfo()
1592
    self.all_inst_info = self.cfg.GetAllInstancesInfo()
1593

    
1594
    group_nodes = set(node.name
1595
                      for node in self.all_node_info.values()
1596
                      if node.group == self.group_uuid)
1597

    
1598
    group_instances = set(inst.name
1599
                          for inst in self.all_inst_info.values()
1600
                          if inst.primary_node in group_nodes)
1601

    
1602
    unlocked_nodes = \
1603
        group_nodes.difference(self.glm.list_owned(locking.LEVEL_NODE))
1604

    
1605
    unlocked_instances = \
1606
        group_instances.difference(self.glm.list_owned(locking.LEVEL_INSTANCE))
1607

    
1608
    if unlocked_nodes:
1609
      raise errors.OpPrereqError("missing lock for nodes: %s" %
1610
                                 utils.CommaJoin(unlocked_nodes))
1611

    
1612
    if unlocked_instances:
1613
      raise errors.OpPrereqError("missing lock for instances: %s" %
1614
                                 utils.CommaJoin(unlocked_instances))
1615

    
1616
    self.my_node_names = utils.NiceSort(group_nodes)
1617
    self.my_inst_names = utils.NiceSort(group_instances)
1618

    
1619
    self.my_node_info = dict((name, self.all_node_info[name])
1620
                             for name in self.my_node_names)
1621

    
1622
    self.my_inst_info = dict((name, self.all_inst_info[name])
1623
                             for name in self.my_inst_names)
1624

    
1625
    # We detect here the nodes that will need the extra RPC calls for verifying
1626
    # split LV volumes; they should be locked.
1627
    extra_lv_nodes = set()
1628

    
1629
    for inst in self.my_inst_info.values():
1630
      if inst.disk_template in constants.DTS_INT_MIRROR:
1631
        group = self.my_node_info[inst.primary_node].group
1632
        for nname in inst.secondary_nodes:
1633
          if self.all_node_info[nname].group != group:
1634
            extra_lv_nodes.add(nname)
1635

    
1636
    unlocked_lv_nodes = \
1637
        extra_lv_nodes.difference(self.glm.list_owned(locking.LEVEL_NODE))
1638

    
1639
    if unlocked_lv_nodes:
1640
      raise errors.OpPrereqError("these nodes could be locked: %s" %
1641
                                 utils.CommaJoin(unlocked_lv_nodes))
1642
    self.extra_lv_nodes = list(extra_lv_nodes)
1643

    
1644
  def _VerifyNode(self, ninfo, nresult):
1645
    """Perform some basic validation on data returned from a node.
1646

1647
      - check the result data structure is well formed and has all the
1648
        mandatory fields
1649
      - check ganeti version
1650

1651
    @type ninfo: L{objects.Node}
1652
    @param ninfo: the node to check
1653
    @param nresult: the results from the node
1654
    @rtype: boolean
1655
    @return: whether overall this call was successful (and we can expect
1656
         reasonable values in the respose)
1657

1658
    """
1659
    node = ninfo.name
1660
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1661

    
1662
    # main result, nresult should be a non-empty dict
1663
    test = not nresult or not isinstance(nresult, dict)
1664
    _ErrorIf(test, self.ENODERPC, node,
1665
                  "unable to verify node: no data returned")
1666
    if test:
1667
      return False
1668

    
1669
    # compares ganeti version
1670
    local_version = constants.PROTOCOL_VERSION
1671
    remote_version = nresult.get("version", None)
1672
    test = not (remote_version and
1673
                isinstance(remote_version, (list, tuple)) and
1674
                len(remote_version) == 2)
1675
    _ErrorIf(test, self.ENODERPC, node,
1676
             "connection to node returned invalid data")
1677
    if test:
1678
      return False
1679

    
1680
    test = local_version != remote_version[0]
1681
    _ErrorIf(test, self.ENODEVERSION, node,
1682
             "incompatible protocol versions: master %s,"
1683
             " node %s", local_version, remote_version[0])
1684
    if test:
1685
      return False
1686

    
1687
    # node seems compatible, we can actually try to look into its results
1688

    
1689
    # full package version
1690
    self._ErrorIf(constants.RELEASE_VERSION != remote_version[1],
1691
                  self.ENODEVERSION, node,
1692
                  "software version mismatch: master %s, node %s",
1693
                  constants.RELEASE_VERSION, remote_version[1],
1694
                  code=self.ETYPE_WARNING)
1695

    
1696
    hyp_result = nresult.get(constants.NV_HYPERVISOR, None)
1697
    if ninfo.vm_capable and isinstance(hyp_result, dict):
1698
      for hv_name, hv_result in hyp_result.iteritems():
1699
        test = hv_result is not None
1700
        _ErrorIf(test, self.ENODEHV, node,
1701
                 "hypervisor %s verify failure: '%s'", hv_name, hv_result)
1702

    
1703
    hvp_result = nresult.get(constants.NV_HVPARAMS, None)
1704
    if ninfo.vm_capable and isinstance(hvp_result, list):
1705
      for item, hv_name, hv_result in hvp_result:
1706
        _ErrorIf(True, self.ENODEHV, node,
1707
                 "hypervisor %s parameter verify failure (source %s): %s",
1708
                 hv_name, item, hv_result)
1709

    
1710
    test = nresult.get(constants.NV_NODESETUP,
1711
                       ["Missing NODESETUP results"])
1712
    _ErrorIf(test, self.ENODESETUP, node, "node setup error: %s",
1713
             "; ".join(test))
1714

    
1715
    return True
1716

    
1717
  def _VerifyNodeTime(self, ninfo, nresult,
1718
                      nvinfo_starttime, nvinfo_endtime):
1719
    """Check the node time.
1720

1721
    @type ninfo: L{objects.Node}
1722
    @param ninfo: the node to check
1723
    @param nresult: the remote results for the node
1724
    @param nvinfo_starttime: the start time of the RPC call
1725
    @param nvinfo_endtime: the end time of the RPC call
1726

1727
    """
1728
    node = ninfo.name
1729
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1730

    
1731
    ntime = nresult.get(constants.NV_TIME, None)
1732
    try:
1733
      ntime_merged = utils.MergeTime(ntime)
1734
    except (ValueError, TypeError):
1735
      _ErrorIf(True, self.ENODETIME, node, "Node returned invalid time")
1736
      return
1737

    
1738
    if ntime_merged < (nvinfo_starttime - constants.NODE_MAX_CLOCK_SKEW):
1739
      ntime_diff = "%.01fs" % abs(nvinfo_starttime - ntime_merged)
1740
    elif ntime_merged > (nvinfo_endtime + constants.NODE_MAX_CLOCK_SKEW):
1741
      ntime_diff = "%.01fs" % abs(ntime_merged - nvinfo_endtime)
1742
    else:
1743
      ntime_diff = None
1744

    
1745
    _ErrorIf(ntime_diff is not None, self.ENODETIME, node,
1746
             "Node time diverges by at least %s from master node time",
1747
             ntime_diff)
1748

    
1749
  def _VerifyNodeLVM(self, ninfo, nresult, vg_name):
1750
    """Check the node LVM results.
1751

1752
    @type ninfo: L{objects.Node}
1753
    @param ninfo: the node to check
1754
    @param nresult: the remote results for the node
1755
    @param vg_name: the configured VG name
1756

1757
    """
1758
    if vg_name is None:
1759
      return
1760

    
1761
    node = ninfo.name
1762
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1763

    
1764
    # checks vg existence and size > 20G
1765
    vglist = nresult.get(constants.NV_VGLIST, None)
1766
    test = not vglist
1767
    _ErrorIf(test, self.ENODELVM, node, "unable to check volume groups")
1768
    if not test:
1769
      vgstatus = utils.CheckVolumeGroupSize(vglist, vg_name,
1770
                                            constants.MIN_VG_SIZE)
1771
      _ErrorIf(vgstatus, self.ENODELVM, node, vgstatus)
1772

    
1773
    # check pv names
1774
    pvlist = nresult.get(constants.NV_PVLIST, None)
1775
    test = pvlist is None
1776
    _ErrorIf(test, self.ENODELVM, node, "Can't get PV list from node")
1777
    if not test:
1778
      # check that ':' is not present in PV names, since it's a
1779
      # special character for lvcreate (denotes the range of PEs to
1780
      # use on the PV)
1781
      for _, pvname, owner_vg in pvlist:
1782
        test = ":" in pvname
1783
        _ErrorIf(test, self.ENODELVM, node, "Invalid character ':' in PV"
1784
                 " '%s' of VG '%s'", pvname, owner_vg)
1785

    
1786
  def _VerifyNodeBridges(self, ninfo, nresult, bridges):
1787
    """Check the node bridges.
1788

1789
    @type ninfo: L{objects.Node}
1790
    @param ninfo: the node to check
1791
    @param nresult: the remote results for the node
1792
    @param bridges: the expected list of bridges
1793

1794
    """
1795
    if not bridges:
1796
      return
1797

    
1798
    node = ninfo.name
1799
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1800

    
1801
    missing = nresult.get(constants.NV_BRIDGES, None)
1802
    test = not isinstance(missing, list)
1803
    _ErrorIf(test, self.ENODENET, node,
1804
             "did not return valid bridge information")
1805
    if not test:
1806
      _ErrorIf(bool(missing), self.ENODENET, node, "missing bridges: %s" %
1807
               utils.CommaJoin(sorted(missing)))
1808

    
1809
  def _VerifyNodeNetwork(self, ninfo, nresult):
1810
    """Check the node network connectivity results.
1811

1812
    @type ninfo: L{objects.Node}
1813
    @param ninfo: the node to check
1814
    @param nresult: the remote results for the node
1815

1816
    """
1817
    node = ninfo.name
1818
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1819

    
1820
    test = constants.NV_NODELIST not in nresult
1821
    _ErrorIf(test, self.ENODESSH, node,
1822
             "node hasn't returned node ssh connectivity data")
1823
    if not test:
1824
      if nresult[constants.NV_NODELIST]:
1825
        for a_node, a_msg in nresult[constants.NV_NODELIST].items():
1826
          _ErrorIf(True, self.ENODESSH, node,
1827
                   "ssh communication with node '%s': %s", a_node, a_msg)
1828

    
1829
    test = constants.NV_NODENETTEST not in nresult
1830
    _ErrorIf(test, self.ENODENET, node,
1831
             "node hasn't returned node tcp connectivity data")
1832
    if not test:
1833
      if nresult[constants.NV_NODENETTEST]:
1834
        nlist = utils.NiceSort(nresult[constants.NV_NODENETTEST].keys())
1835
        for anode in nlist:
1836
          _ErrorIf(True, self.ENODENET, node,
1837
                   "tcp communication with node '%s': %s",
1838
                   anode, nresult[constants.NV_NODENETTEST][anode])
1839

    
1840
    test = constants.NV_MASTERIP not in nresult
1841
    _ErrorIf(test, self.ENODENET, node,
1842
             "node hasn't returned node master IP reachability data")
1843
    if not test:
1844
      if not nresult[constants.NV_MASTERIP]:
1845
        if node == self.master_node:
1846
          msg = "the master node cannot reach the master IP (not configured?)"
1847
        else:
1848
          msg = "cannot reach the master IP"
1849
        _ErrorIf(True, self.ENODENET, node, msg)
1850

    
1851
  def _VerifyInstance(self, instance, instanceconfig, node_image,
1852
                      diskstatus):
1853
    """Verify an instance.
1854

1855
    This function checks to see if the required block devices are
1856
    available on the instance's node.
1857

1858
    """
1859
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
1860
    node_current = instanceconfig.primary_node
1861

    
1862
    node_vol_should = {}
1863
    instanceconfig.MapLVsByNode(node_vol_should)
1864

    
1865
    for node in node_vol_should:
1866
      n_img = node_image[node]
1867
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1868
        # ignore missing volumes on offline or broken nodes
1869
        continue
1870
      for volume in node_vol_should[node]:
1871
        test = volume not in n_img.volumes
1872
        _ErrorIf(test, self.EINSTANCEMISSINGDISK, instance,
1873
                 "volume %s missing on node %s", volume, node)
1874

    
1875
    if instanceconfig.admin_up:
1876
      pri_img = node_image[node_current]
1877
      test = instance not in pri_img.instances and not pri_img.offline
1878
      _ErrorIf(test, self.EINSTANCEDOWN, instance,
1879
               "instance not running on its primary node %s",
1880
               node_current)
1881

    
1882
    diskdata = [(nname, success, status, idx)
1883
                for (nname, disks) in diskstatus.items()
1884
                for idx, (success, status) in enumerate(disks)]
1885

    
1886
    for nname, success, bdev_status, idx in diskdata:
1887
      # the 'ghost node' construction in Exec() ensures that we have a
1888
      # node here
1889
      snode = node_image[nname]
1890
      bad_snode = snode.ghost or snode.offline
1891
      _ErrorIf(instanceconfig.admin_up and not success and not bad_snode,
1892
               self.EINSTANCEFAULTYDISK, instance,
1893
               "couldn't retrieve status for disk/%s on %s: %s",
1894
               idx, nname, bdev_status)
1895
      _ErrorIf((instanceconfig.admin_up and success and
1896
                bdev_status.ldisk_status == constants.LDS_FAULTY),
1897
               self.EINSTANCEFAULTYDISK, instance,
1898
               "disk/%s on %s is faulty", idx, nname)
1899

    
1900
  def _VerifyOrphanVolumes(self, node_vol_should, node_image, reserved):
1901
    """Verify if there are any unknown volumes in the cluster.
1902

1903
    The .os, .swap and backup volumes are ignored. All other volumes are
1904
    reported as unknown.
1905

1906
    @type reserved: L{ganeti.utils.FieldSet}
1907
    @param reserved: a FieldSet of reserved volume names
1908

1909
    """
1910
    for node, n_img in node_image.items():
1911
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
1912
        # skip non-healthy nodes
1913
        continue
1914
      for volume in n_img.volumes:
1915
        test = ((node not in node_vol_should or
1916
                volume not in node_vol_should[node]) and
1917
                not reserved.Matches(volume))
1918
        self._ErrorIf(test, self.ENODEORPHANLV, node,
1919
                      "volume %s is unknown", volume)
1920

    
1921
  def _VerifyNPlusOneMemory(self, node_image, instance_cfg):
1922
    """Verify N+1 Memory Resilience.
1923

1924
    Check that if one single node dies we can still start all the
1925
    instances it was primary for.
1926

1927
    """
1928
    cluster_info = self.cfg.GetClusterInfo()
1929
    for node, n_img in node_image.items():
1930
      # This code checks that every node which is now listed as
1931
      # secondary has enough memory to host all instances it is
1932
      # supposed to should a single other node in the cluster fail.
1933
      # FIXME: not ready for failover to an arbitrary node
1934
      # FIXME: does not support file-backed instances
1935
      # WARNING: we currently take into account down instances as well
1936
      # as up ones, considering that even if they're down someone
1937
      # might want to start them even in the event of a node failure.
1938
      if n_img.offline:
1939
        # we're skipping offline nodes from the N+1 warning, since
1940
        # most likely we don't have good memory infromation from them;
1941
        # we already list instances living on such nodes, and that's
1942
        # enough warning
1943
        continue
1944
      for prinode, instances in n_img.sbp.items():
1945
        needed_mem = 0
1946
        for instance in instances:
1947
          bep = cluster_info.FillBE(instance_cfg[instance])
1948
          if bep[constants.BE_AUTO_BALANCE]:
1949
            needed_mem += bep[constants.BE_MEMORY]
1950
        test = n_img.mfree < needed_mem
1951
        self._ErrorIf(test, self.ENODEN1, node,
1952
                      "not enough memory to accomodate instance failovers"
1953
                      " should node %s fail (%dMiB needed, %dMiB available)",
1954
                      prinode, needed_mem, n_img.mfree)
1955

    
1956
  @classmethod
1957
  def _VerifyFiles(cls, errorif, nodeinfo, master_node, all_nvinfo,
1958
                   (files_all, files_all_opt, files_mc, files_vm)):
1959
    """Verifies file checksums collected from all nodes.
1960

1961
    @param errorif: Callback for reporting errors
1962
    @param nodeinfo: List of L{objects.Node} objects
1963
    @param master_node: Name of master node
1964
    @param all_nvinfo: RPC results
1965

1966
    """
1967
    node_names = frozenset(node.name for node in nodeinfo)
1968

    
1969
    assert master_node in node_names
1970
    assert (len(files_all | files_all_opt | files_mc | files_vm) ==
1971
            sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
1972
           "Found file listed in more than one file list"
1973

    
1974
    # Define functions determining which nodes to consider for a file
1975
    file2nodefn = dict([(filename, fn)
1976
      for (files, fn) in [(files_all, None),
1977
                          (files_all_opt, None),
1978
                          (files_mc, lambda node: (node.master_candidate or
1979
                                                   node.name == master_node)),
1980
                          (files_vm, lambda node: node.vm_capable)]
1981
      for filename in files])
1982

    
1983
    fileinfo = dict((filename, {}) for filename in file2nodefn.keys())
1984

    
1985
    for node in nodeinfo:
1986
      nresult = all_nvinfo[node.name]
1987

    
1988
      if nresult.fail_msg or not nresult.payload:
1989
        node_files = None
1990
      else:
1991
        node_files = nresult.payload.get(constants.NV_FILELIST, None)
1992

    
1993
      test = not (node_files and isinstance(node_files, dict))
1994
      errorif(test, cls.ENODEFILECHECK, node.name,
1995
              "Node did not return file checksum data")
1996
      if test:
1997
        continue
1998

    
1999
      for (filename, checksum) in node_files.items():
2000
        # Check if the file should be considered for a node
2001
        fn = file2nodefn[filename]
2002
        if fn is None or fn(node):
2003
          fileinfo[filename].setdefault(checksum, set()).add(node.name)
2004

    
2005
    for (filename, checksums) in fileinfo.items():
2006
      assert compat.all(len(i) > 10 for i in checksums), "Invalid checksum"
2007

    
2008
      # Nodes having the file
2009
      with_file = frozenset(node_name
2010
                            for nodes in fileinfo[filename].values()
2011
                            for node_name in nodes)
2012

    
2013
      # Nodes missing file
2014
      missing_file = node_names - with_file
2015

    
2016
      if filename in files_all_opt:
2017
        # All or no nodes
2018
        errorif(missing_file and missing_file != node_names,
2019
                cls.ECLUSTERFILECHECK, None,
2020
                "File %s is optional, but it must exist on all or no nodes (not"
2021
                " found on %s)",
2022
                filename, utils.CommaJoin(utils.NiceSort(missing_file)))
2023
      else:
2024
        errorif(missing_file, cls.ECLUSTERFILECHECK, None,
2025
                "File %s is missing from node(s) %s", filename,
2026
                utils.CommaJoin(utils.NiceSort(missing_file)))
2027

    
2028
      # See if there are multiple versions of the file
2029
      test = len(checksums) > 1
2030
      if test:
2031
        variants = ["variant %s on %s" %
2032
                    (idx + 1, utils.CommaJoin(utils.NiceSort(nodes)))
2033
                    for (idx, (checksum, nodes)) in
2034
                      enumerate(sorted(checksums.items()))]
2035
      else:
2036
        variants = []
2037

    
2038
      errorif(test, cls.ECLUSTERFILECHECK, None,
2039
              "File %s found with %s different checksums (%s)",
2040
              filename, len(checksums), "; ".join(variants))
2041

    
2042
  def _VerifyNodeDrbd(self, ninfo, nresult, instanceinfo, drbd_helper,
2043
                      drbd_map):
2044
    """Verifies and the node DRBD status.
2045

2046
    @type ninfo: L{objects.Node}
2047
    @param ninfo: the node to check
2048
    @param nresult: the remote results for the node
2049
    @param instanceinfo: the dict of instances
2050
    @param drbd_helper: the configured DRBD usermode helper
2051
    @param drbd_map: the DRBD map as returned by
2052
        L{ganeti.config.ConfigWriter.ComputeDRBDMap}
2053

2054
    """
2055
    node = ninfo.name
2056
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2057

    
2058
    if drbd_helper:
2059
      helper_result = nresult.get(constants.NV_DRBDHELPER, None)
2060
      test = (helper_result == None)
2061
      _ErrorIf(test, self.ENODEDRBDHELPER, node,
2062
               "no drbd usermode helper returned")
2063
      if helper_result:
2064
        status, payload = helper_result
2065
        test = not status
2066
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
2067
                 "drbd usermode helper check unsuccessful: %s", payload)
2068
        test = status and (payload != drbd_helper)
2069
        _ErrorIf(test, self.ENODEDRBDHELPER, node,
2070
                 "wrong drbd usermode helper: %s", payload)
2071

    
2072
    # compute the DRBD minors
2073
    node_drbd = {}
2074
    for minor, instance in drbd_map[node].items():
2075
      test = instance not in instanceinfo
2076
      _ErrorIf(test, self.ECLUSTERCFG, None,
2077
               "ghost instance '%s' in temporary DRBD map", instance)
2078
        # ghost instance should not be running, but otherwise we
2079
        # don't give double warnings (both ghost instance and
2080
        # unallocated minor in use)
2081
      if test:
2082
        node_drbd[minor] = (instance, False)
2083
      else:
2084
        instance = instanceinfo[instance]
2085
        node_drbd[minor] = (instance.name, instance.admin_up)
2086

    
2087
    # and now check them
2088
    used_minors = nresult.get(constants.NV_DRBDLIST, [])
2089
    test = not isinstance(used_minors, (tuple, list))
2090
    _ErrorIf(test, self.ENODEDRBD, node,
2091
             "cannot parse drbd status file: %s", str(used_minors))
2092
    if test:
2093
      # we cannot check drbd status
2094
      return
2095

    
2096
    for minor, (iname, must_exist) in node_drbd.items():
2097
      test = minor not in used_minors and must_exist
2098
      _ErrorIf(test, self.ENODEDRBD, node,
2099
               "drbd minor %d of instance %s is not active", minor, iname)
2100
    for minor in used_minors:
2101
      test = minor not in node_drbd
2102
      _ErrorIf(test, self.ENODEDRBD, node,
2103
               "unallocated drbd minor %d is in use", minor)
2104

    
2105
  def _UpdateNodeOS(self, ninfo, nresult, nimg):
2106
    """Builds the node OS structures.
2107

2108
    @type ninfo: L{objects.Node}
2109
    @param ninfo: the node to check
2110
    @param nresult: the remote results for the node
2111
    @param nimg: the node image object
2112

2113
    """
2114
    node = ninfo.name
2115
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2116

    
2117
    remote_os = nresult.get(constants.NV_OSLIST, None)
2118
    test = (not isinstance(remote_os, list) or
2119
            not compat.all(isinstance(v, list) and len(v) == 7
2120
                           for v in remote_os))
2121

    
2122
    _ErrorIf(test, self.ENODEOS, node,
2123
             "node hasn't returned valid OS data")
2124

    
2125
    nimg.os_fail = test
2126

    
2127
    if test:
2128
      return
2129

    
2130
    os_dict = {}
2131

    
2132
    for (name, os_path, status, diagnose,
2133
         variants, parameters, api_ver) in nresult[constants.NV_OSLIST]:
2134

    
2135
      if name not in os_dict:
2136
        os_dict[name] = []
2137

    
2138
      # parameters is a list of lists instead of list of tuples due to
2139
      # JSON lacking a real tuple type, fix it:
2140
      parameters = [tuple(v) for v in parameters]
2141
      os_dict[name].append((os_path, status, diagnose,
2142
                            set(variants), set(parameters), set(api_ver)))
2143

    
2144
    nimg.oslist = os_dict
2145

    
2146
  def _VerifyNodeOS(self, ninfo, nimg, base):
2147
    """Verifies the node OS list.
2148

2149
    @type ninfo: L{objects.Node}
2150
    @param ninfo: the node to check
2151
    @param nimg: the node image object
2152
    @param base: the 'template' node we match against (e.g. from the master)
2153

2154
    """
2155
    node = ninfo.name
2156
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2157

    
2158
    assert not nimg.os_fail, "Entered _VerifyNodeOS with failed OS rpc?"
2159

    
2160
    beautify_params = lambda l: ["%s: %s" % (k, v) for (k, v) in l]
2161
    for os_name, os_data in nimg.oslist.items():
2162
      assert os_data, "Empty OS status for OS %s?!" % os_name
2163
      f_path, f_status, f_diag, f_var, f_param, f_api = os_data[0]
2164
      _ErrorIf(not f_status, self.ENODEOS, node,
2165
               "Invalid OS %s (located at %s): %s", os_name, f_path, f_diag)
2166
      _ErrorIf(len(os_data) > 1, self.ENODEOS, node,
2167
               "OS '%s' has multiple entries (first one shadows the rest): %s",
2168
               os_name, utils.CommaJoin([v[0] for v in os_data]))
2169
      # this will catched in backend too
2170
      _ErrorIf(compat.any(v >= constants.OS_API_V15 for v in f_api)
2171
               and not f_var, self.ENODEOS, node,
2172
               "OS %s with API at least %d does not declare any variant",
2173
               os_name, constants.OS_API_V15)
2174
      # comparisons with the 'base' image
2175
      test = os_name not in base.oslist
2176
      _ErrorIf(test, self.ENODEOS, node,
2177
               "Extra OS %s not present on reference node (%s)",
2178
               os_name, base.name)
2179
      if test:
2180
        continue
2181
      assert base.oslist[os_name], "Base node has empty OS status?"
2182
      _, b_status, _, b_var, b_param, b_api = base.oslist[os_name][0]
2183
      if not b_status:
2184
        # base OS is invalid, skipping
2185
        continue
2186
      for kind, a, b in [("API version", f_api, b_api),
2187
                         ("variants list", f_var, b_var),
2188
                         ("parameters", beautify_params(f_param),
2189
                          beautify_params(b_param))]:
2190
        _ErrorIf(a != b, self.ENODEOS, node,
2191
                 "OS %s for %s differs from reference node %s: [%s] vs. [%s]",
2192
                 kind, os_name, base.name,
2193
                 utils.CommaJoin(sorted(a)), utils.CommaJoin(sorted(b)))
2194

    
2195
    # check any missing OSes
2196
    missing = set(base.oslist.keys()).difference(nimg.oslist.keys())
2197
    _ErrorIf(missing, self.ENODEOS, node,
2198
             "OSes present on reference node %s but missing on this node: %s",
2199
             base.name, utils.CommaJoin(missing))
2200

    
2201
  def _VerifyOob(self, ninfo, nresult):
2202
    """Verifies out of band functionality of a node.
2203

2204
    @type ninfo: L{objects.Node}
2205
    @param ninfo: the node to check
2206
    @param nresult: the remote results for the node
2207

2208
    """
2209
    node = ninfo.name
2210
    # We just have to verify the paths on master and/or master candidates
2211
    # as the oob helper is invoked on the master
2212
    if ((ninfo.master_candidate or ninfo.master_capable) and
2213
        constants.NV_OOB_PATHS in nresult):
2214
      for path_result in nresult[constants.NV_OOB_PATHS]:
2215
        self._ErrorIf(path_result, self.ENODEOOBPATH, node, path_result)
2216

    
2217
  def _UpdateNodeVolumes(self, ninfo, nresult, nimg, vg_name):
2218
    """Verifies and updates the node volume data.
2219

2220
    This function will update a L{NodeImage}'s internal structures
2221
    with data from the remote call.
2222

2223
    @type ninfo: L{objects.Node}
2224
    @param ninfo: the node to check
2225
    @param nresult: the remote results for the node
2226
    @param nimg: the node image object
2227
    @param vg_name: the configured VG name
2228

2229
    """
2230
    node = ninfo.name
2231
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2232

    
2233
    nimg.lvm_fail = True
2234
    lvdata = nresult.get(constants.NV_LVLIST, "Missing LV data")
2235
    if vg_name is None:
2236
      pass
2237
    elif isinstance(lvdata, basestring):
2238
      _ErrorIf(True, self.ENODELVM, node, "LVM problem on node: %s",
2239
               utils.SafeEncode(lvdata))
2240
    elif not isinstance(lvdata, dict):
2241
      _ErrorIf(True, self.ENODELVM, node, "rpc call to node failed (lvlist)")
2242
    else:
2243
      nimg.volumes = lvdata
2244
      nimg.lvm_fail = False
2245

    
2246
  def _UpdateNodeInstances(self, ninfo, nresult, nimg):
2247
    """Verifies and updates the node instance list.
2248

2249
    If the listing was successful, then updates this node's instance
2250
    list. Otherwise, it marks the RPC call as failed for the instance
2251
    list key.
2252

2253
    @type ninfo: L{objects.Node}
2254
    @param ninfo: the node to check
2255
    @param nresult: the remote results for the node
2256
    @param nimg: the node image object
2257

2258
    """
2259
    idata = nresult.get(constants.NV_INSTANCELIST, None)
2260
    test = not isinstance(idata, list)
2261
    self._ErrorIf(test, self.ENODEHV, ninfo.name, "rpc call to node failed"
2262
                  " (instancelist): %s", utils.SafeEncode(str(idata)))
2263
    if test:
2264
      nimg.hyp_fail = True
2265
    else:
2266
      nimg.instances = idata
2267

    
2268
  def _UpdateNodeInfo(self, ninfo, nresult, nimg, vg_name):
2269
    """Verifies and computes a node information map
2270

2271
    @type ninfo: L{objects.Node}
2272
    @param ninfo: the node to check
2273
    @param nresult: the remote results for the node
2274
    @param nimg: the node image object
2275
    @param vg_name: the configured VG name
2276

2277
    """
2278
    node = ninfo.name
2279
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2280

    
2281
    # try to read free memory (from the hypervisor)
2282
    hv_info = nresult.get(constants.NV_HVINFO, None)
2283
    test = not isinstance(hv_info, dict) or "memory_free" not in hv_info
2284
    _ErrorIf(test, self.ENODEHV, node, "rpc call to node failed (hvinfo)")
2285
    if not test:
2286
      try:
2287
        nimg.mfree = int(hv_info["memory_free"])
2288
      except (ValueError, TypeError):
2289
        _ErrorIf(True, self.ENODERPC, node,
2290
                 "node returned invalid nodeinfo, check hypervisor")
2291

    
2292
    # FIXME: devise a free space model for file based instances as well
2293
    if vg_name is not None:
2294
      test = (constants.NV_VGLIST not in nresult or
2295
              vg_name not in nresult[constants.NV_VGLIST])
2296
      _ErrorIf(test, self.ENODELVM, node,
2297
               "node didn't return data for the volume group '%s'"
2298
               " - it is either missing or broken", vg_name)
2299
      if not test:
2300
        try:
2301
          nimg.dfree = int(nresult[constants.NV_VGLIST][vg_name])
2302
        except (ValueError, TypeError):
2303
          _ErrorIf(True, self.ENODERPC, node,
2304
                   "node returned invalid LVM info, check LVM status")
2305

    
2306
  def _CollectDiskInfo(self, nodelist, node_image, instanceinfo):
2307
    """Gets per-disk status information for all instances.
2308

2309
    @type nodelist: list of strings
2310
    @param nodelist: Node names
2311
    @type node_image: dict of (name, L{objects.Node})
2312
    @param node_image: Node objects
2313
    @type instanceinfo: dict of (name, L{objects.Instance})
2314
    @param instanceinfo: Instance objects
2315
    @rtype: {instance: {node: [(succes, payload)]}}
2316
    @return: a dictionary of per-instance dictionaries with nodes as
2317
        keys and disk information as values; the disk information is a
2318
        list of tuples (success, payload)
2319

2320
    """
2321
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2322

    
2323
    node_disks = {}
2324
    node_disks_devonly = {}
2325
    diskless_instances = set()
2326
    diskless = constants.DT_DISKLESS
2327

    
2328
    for nname in nodelist:
2329
      node_instances = list(itertools.chain(node_image[nname].pinst,
2330
                                            node_image[nname].sinst))
2331
      diskless_instances.update(inst for inst in node_instances
2332
                                if instanceinfo[inst].disk_template == diskless)
2333
      disks = [(inst, disk)
2334
               for inst in node_instances
2335
               for disk in instanceinfo[inst].disks]
2336

    
2337
      if not disks:
2338
        # No need to collect data
2339
        continue
2340

    
2341
      node_disks[nname] = disks
2342

    
2343
      # Creating copies as SetDiskID below will modify the objects and that can
2344
      # lead to incorrect data returned from nodes
2345
      devonly = [dev.Copy() for (_, dev) in disks]
2346

    
2347
      for dev in devonly:
2348
        self.cfg.SetDiskID(dev, nname)
2349

    
2350
      node_disks_devonly[nname] = devonly
2351

    
2352
    assert len(node_disks) == len(node_disks_devonly)
2353

    
2354
    # Collect data from all nodes with disks
2355
    result = self.rpc.call_blockdev_getmirrorstatus_multi(node_disks.keys(),
2356
                                                          node_disks_devonly)
2357

    
2358
    assert len(result) == len(node_disks)
2359

    
2360
    instdisk = {}
2361

    
2362
    for (nname, nres) in result.items():
2363
      disks = node_disks[nname]
2364

    
2365
      if nres.offline:
2366
        # No data from this node
2367
        data = len(disks) * [(False, "node offline")]
2368
      else:
2369
        msg = nres.fail_msg
2370
        _ErrorIf(msg, self.ENODERPC, nname,
2371
                 "while getting disk information: %s", msg)
2372
        if msg:
2373
          # No data from this node
2374
          data = len(disks) * [(False, msg)]
2375
        else:
2376
          data = []
2377
          for idx, i in enumerate(nres.payload):
2378
            if isinstance(i, (tuple, list)) and len(i) == 2:
2379
              data.append(i)
2380
            else:
2381
              logging.warning("Invalid result from node %s, entry %d: %s",
2382
                              nname, idx, i)
2383
              data.append((False, "Invalid result from the remote node"))
2384

    
2385
      for ((inst, _), status) in zip(disks, data):
2386
        instdisk.setdefault(inst, {}).setdefault(nname, []).append(status)
2387

    
2388
    # Add empty entries for diskless instances.
2389
    for inst in diskless_instances:
2390
      assert inst not in instdisk
2391
      instdisk[inst] = {}
2392

    
2393
    assert compat.all(len(statuses) == len(instanceinfo[inst].disks) and
2394
                      len(nnames) <= len(instanceinfo[inst].all_nodes) and
2395
                      compat.all(isinstance(s, (tuple, list)) and
2396
                                 len(s) == 2 for s in statuses)
2397
                      for inst, nnames in instdisk.items()
2398
                      for nname, statuses in nnames.items())
2399
    assert set(instdisk) == set(instanceinfo), "instdisk consistency failure"
2400

    
2401
    return instdisk
2402

    
2403
  def BuildHooksEnv(self):
2404
    """Build hooks env.
2405

2406
    Cluster-Verify hooks just ran in the post phase and their failure makes
2407
    the output be logged in the verify output and the verification to fail.
2408

2409
    """
2410
    env = {
2411
      "CLUSTER_TAGS": " ".join(self.cfg.GetClusterInfo().GetTags())
2412
      }
2413

    
2414
    env.update(("NODE_TAGS_%s" % node.name, " ".join(node.GetTags()))
2415
               for node in self.my_node_info.values())
2416

    
2417
    return env
2418

    
2419
  def BuildHooksNodes(self):
2420
    """Build hooks nodes.
2421

2422
    """
2423
    assert self.my_node_names, ("Node list not gathered,"
2424
      " has CheckPrereq been executed?")
2425
    return ([], self.my_node_names)
2426

    
2427
  def Exec(self, feedback_fn):
2428
    """Verify integrity of the node group, performing various test on nodes.
2429

2430
    """
2431
    # This method has too many local variables. pylint: disable-msg=R0914
2432
    self.bad = False
2433
    _ErrorIf = self._ErrorIf # pylint: disable-msg=C0103
2434
    verbose = self.op.verbose
2435
    self._feedback_fn = feedback_fn
2436

    
2437
    vg_name = self.cfg.GetVGName()
2438
    drbd_helper = self.cfg.GetDRBDHelper()
2439
    cluster = self.cfg.GetClusterInfo()
2440
    groupinfo = self.cfg.GetAllNodeGroupsInfo()
2441
    hypervisors = cluster.enabled_hypervisors
2442
    node_data_list = [self.my_node_info[name] for name in self.my_node_names]
2443

    
2444
    i_non_redundant = [] # Non redundant instances
2445
    i_non_a_balanced = [] # Non auto-balanced instances
2446
    n_offline = 0 # Count of offline nodes
2447
    n_drained = 0 # Count of nodes being drained
2448
    node_vol_should = {}
2449

    
2450
    # FIXME: verify OS list
2451

    
2452
    # File verification
2453
    filemap = _ComputeAncillaryFiles(cluster, False)
2454

    
2455
    # do local checksums
2456
    master_node = self.master_node = self.cfg.GetMasterNode()
2457
    master_ip = self.cfg.GetMasterIP()
2458

    
2459
    feedback_fn("* Gathering data (%d nodes)" % len(self.my_node_names))
2460

    
2461
    # We will make nodes contact all nodes in their group, and one node from
2462
    # every other group.
2463
    # TODO: should it be a *random* node, different every time?
2464
    online_nodes = [node.name for node in node_data_list if not node.offline]
2465
    other_group_nodes = {}
2466

    
2467
    for name in sorted(self.all_node_info):
2468
      node = self.all_node_info[name]
2469
      if (node.group not in other_group_nodes
2470
          and node.group != self.group_uuid
2471
          and not node.offline):
2472
        other_group_nodes[node.group] = node.name
2473

    
2474
    node_verify_param = {
2475
      constants.NV_FILELIST:
2476
        utils.UniqueSequence(filename
2477
                             for files in filemap
2478
                             for filename in files),
2479
      constants.NV_NODELIST: online_nodes + other_group_nodes.values(),
2480
      constants.NV_HYPERVISOR: hypervisors,
2481
      constants.NV_HVPARAMS:
2482
        _GetAllHypervisorParameters(cluster, self.all_inst_info.values()),
2483
      constants.NV_NODENETTEST: [(node.name, node.primary_ip, node.secondary_ip)
2484
                                 for node in node_data_list
2485
                                 if not node.offline],
2486
      constants.NV_INSTANCELIST: hypervisors,
2487
      constants.NV_VERSION: None,
2488
      constants.NV_HVINFO: self.cfg.GetHypervisorType(),
2489
      constants.NV_NODESETUP: None,
2490
      constants.NV_TIME: None,
2491
      constants.NV_MASTERIP: (master_node, master_ip),
2492
      constants.NV_OSLIST: None,
2493
      constants.NV_VMNODES: self.cfg.GetNonVmCapableNodeList(),
2494
      }
2495

    
2496
    if vg_name is not None:
2497
      node_verify_param[constants.NV_VGLIST] = None
2498
      node_verify_param[constants.NV_LVLIST] = vg_name
2499
      node_verify_param[constants.NV_PVLIST] = [vg_name]
2500
      node_verify_param[constants.NV_DRBDLIST] = None
2501

    
2502
    if drbd_helper:
2503
      node_verify_param[constants.NV_DRBDHELPER] = drbd_helper
2504

    
2505
    # bridge checks
2506
    # FIXME: this needs to be changed per node-group, not cluster-wide
2507
    bridges = set()
2508
    default_nicpp = cluster.nicparams[constants.PP_DEFAULT]
2509
    if default_nicpp[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2510
      bridges.add(default_nicpp[constants.NIC_LINK])
2511
    for instance in self.my_inst_info.values():
2512
      for nic in instance.nics:
2513
        full_nic = cluster.SimpleFillNIC(nic.nicparams)
2514
        if full_nic[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2515
          bridges.add(full_nic[constants.NIC_LINK])
2516

    
2517
    if bridges:
2518
      node_verify_param[constants.NV_BRIDGES] = list(bridges)
2519

    
2520
    # Build our expected cluster state
2521
    node_image = dict((node.name, self.NodeImage(offline=node.offline,
2522
                                                 name=node.name,
2523
                                                 vm_capable=node.vm_capable))
2524
                      for node in node_data_list)
2525

    
2526
    # Gather OOB paths
2527
    oob_paths = []
2528
    for node in self.all_node_info.values():
2529
      path = _SupportsOob(self.cfg, node)
2530
      if path and path not in oob_paths:
2531
        oob_paths.append(path)
2532

    
2533
    if oob_paths:
2534
      node_verify_param[constants.NV_OOB_PATHS] = oob_paths
2535

    
2536
    for instance in self.my_inst_names:
2537
      inst_config = self.my_inst_info[instance]
2538

    
2539
      for nname in inst_config.all_nodes:
2540
        if nname not in node_image:
2541
          gnode = self.NodeImage(name=nname)
2542
          gnode.ghost = (nname not in self.all_node_info)
2543
          node_image[nname] = gnode
2544

    
2545
      inst_config.MapLVsByNode(node_vol_should)
2546

    
2547
      pnode = inst_config.primary_node
2548
      node_image[pnode].pinst.append(instance)
2549

    
2550
      for snode in inst_config.secondary_nodes:
2551
        nimg = node_image[snode]
2552
        nimg.sinst.append(instance)
2553
        if pnode not in nimg.sbp:
2554
          nimg.sbp[pnode] = []
2555
        nimg.sbp[pnode].append(instance)
2556

    
2557
    # At this point, we have the in-memory data structures complete,
2558
    # except for the runtime information, which we'll gather next
2559

    
2560
    # Due to the way our RPC system works, exact response times cannot be
2561
    # guaranteed (e.g. a broken node could run into a timeout). By keeping the
2562
    # time before and after executing the request, we can at least have a time
2563
    # window.
2564
    nvinfo_starttime = time.time()
2565
    all_nvinfo = self.rpc.call_node_verify(self.my_node_names,
2566
                                           node_verify_param,
2567
                                           self.cfg.GetClusterName())
2568
    if self.extra_lv_nodes and vg_name is not None:
2569
      extra_lv_nvinfo = \
2570
          self.rpc.call_node_verify(self.extra_lv_nodes,
2571
                                    {constants.NV_LVLIST: vg_name},
2572
                                    self.cfg.GetClusterName())
2573
    else:
2574
      extra_lv_nvinfo = {}
2575
    nvinfo_endtime = time.time()
2576

    
2577
    all_drbd_map = self.cfg.ComputeDRBDMap()
2578

    
2579
    feedback_fn("* Gathering disk information (%s nodes)" %
2580
                len(self.my_node_names))
2581
    instdisk = self._CollectDiskInfo(self.my_node_names, node_image,
2582
                                     self.my_inst_info)
2583

    
2584
    feedback_fn("* Verifying configuration file consistency")
2585

    
2586
    # If not all nodes are being checked, we need to make sure the master node
2587
    # and a non-checked vm_capable node are in the list.
2588
    absent_nodes = set(self.all_node_info).difference(self.my_node_info)
2589
    if absent_nodes:
2590
      vf_nvinfo = all_nvinfo.copy()
2591
      vf_node_info = list(self.my_node_info.values())
2592
      additional_nodes = []
2593
      if master_node not in self.my_node_info:
2594
        additional_nodes.append(master_node)
2595
        vf_node_info.append(self.all_node_info[master_node])
2596
      # Add the first vm_capable node we find which is not included
2597
      for node in absent_nodes:
2598
        nodeinfo = self.all_node_info[node]
2599
        if nodeinfo.vm_capable and not nodeinfo.offline:
2600
          additional_nodes.append(node)
2601
          vf_node_info.append(self.all_node_info[node])
2602
          break
2603
      key = constants.NV_FILELIST
2604
      vf_nvinfo.update(self.rpc.call_node_verify(additional_nodes,
2605
                                                 {key: node_verify_param[key]},
2606
                                                 self.cfg.GetClusterName()))
2607
    else:
2608
      vf_nvinfo = all_nvinfo
2609
      vf_node_info = self.my_node_info.values()
2610

    
2611
    self._VerifyFiles(_ErrorIf, vf_node_info, master_node, vf_nvinfo, filemap)
2612

    
2613
    feedback_fn("* Verifying node status")
2614

    
2615
    refos_img = None
2616

    
2617
    for node_i in node_data_list:
2618
      node = node_i.name
2619
      nimg = node_image[node]
2620

    
2621
      if node_i.offline:
2622
        if verbose:
2623
          feedback_fn("* Skipping offline node %s" % (node,))
2624
        n_offline += 1
2625
        continue
2626

    
2627
      if node == master_node:
2628
        ntype = "master"
2629
      elif node_i.master_candidate:
2630
        ntype = "master candidate"
2631
      elif node_i.drained:
2632
        ntype = "drained"
2633
        n_drained += 1
2634
      else:
2635
        ntype = "regular"
2636
      if verbose:
2637
        feedback_fn("* Verifying node %s (%s)" % (node, ntype))
2638

    
2639
      msg = all_nvinfo[node].fail_msg
2640
      _ErrorIf(msg, self.ENODERPC, node, "while contacting node: %s", msg)
2641
      if msg:
2642
        nimg.rpc_fail = True
2643
        continue
2644

    
2645
      nresult = all_nvinfo[node].payload
2646

    
2647
      nimg.call_ok = self._VerifyNode(node_i, nresult)
2648
      self._VerifyNodeTime(node_i, nresult, nvinfo_starttime, nvinfo_endtime)
2649
      self._VerifyNodeNetwork(node_i, nresult)
2650
      self._VerifyOob(node_i, nresult)
2651

    
2652
      if nimg.vm_capable:
2653
        self._VerifyNodeLVM(node_i, nresult, vg_name)
2654
        self._VerifyNodeDrbd(node_i, nresult, self.all_inst_info, drbd_helper,
2655
                             all_drbd_map)
2656

    
2657
        self._UpdateNodeVolumes(node_i, nresult, nimg, vg_name)
2658
        self._UpdateNodeInstances(node_i, nresult, nimg)
2659
        self._UpdateNodeInfo(node_i, nresult, nimg, vg_name)
2660
        self._UpdateNodeOS(node_i, nresult, nimg)
2661

    
2662
        if not nimg.os_fail:
2663
          if refos_img is None:
2664
            refos_img = nimg
2665
          self._VerifyNodeOS(node_i, nimg, refos_img)
2666
        self._VerifyNodeBridges(node_i, nresult, bridges)
2667

    
2668
        # Check whether all running instancies are primary for the node. (This
2669
        # can no longer be done from _VerifyInstance below, since some of the
2670
        # wrong instances could be from other node groups.)
2671
        non_primary_inst = set(nimg.instances).difference(nimg.pinst)
2672

    
2673
        for inst in non_primary_inst:
2674
          test = inst in self.all_inst_info
2675
          _ErrorIf(test, self.EINSTANCEWRONGNODE, inst,
2676
                   "instance should not run on node %s", node_i.name)
2677
          _ErrorIf(not test, self.ENODEORPHANINSTANCE, node_i.name,
2678
                   "node is running unknown instance %s", inst)
2679

    
2680
    for node, result in extra_lv_nvinfo.items():
2681
      self._UpdateNodeVolumes(self.all_node_info[node], result.payload,
2682
                              node_image[node], vg_name)
2683

    
2684
    feedback_fn("* Verifying instance status")
2685
    for instance in self.my_inst_names:
2686
      if verbose:
2687
        feedback_fn("* Verifying instance %s" % instance)
2688
      inst_config = self.my_inst_info[instance]
2689
      self._VerifyInstance(instance, inst_config, node_image,
2690
                           instdisk[instance])
2691
      inst_nodes_offline = []
2692

    
2693
      pnode = inst_config.primary_node
2694
      pnode_img = node_image[pnode]
2695
      _ErrorIf(pnode_img.rpc_fail and not pnode_img.offline,
2696
               self.ENODERPC, pnode, "instance %s, connection to"
2697
               " primary node failed", instance)
2698

    
2699
      _ErrorIf(inst_config.admin_up and pnode_img.offline,
2700
               self.EINSTANCEBADNODE, instance,
2701
               "instance is marked as running and lives on offline node %s",
2702
               inst_config.primary_node)
2703

    
2704
      # If the instance is non-redundant we cannot survive losing its primary
2705
      # node, so we are not N+1 compliant. On the other hand we have no disk
2706
      # templates with more than one secondary so that situation is not well
2707
      # supported either.
2708
      # FIXME: does not support file-backed instances
2709
      if not inst_config.secondary_nodes:
2710
        i_non_redundant.append(instance)
2711

    
2712
      _ErrorIf(len(inst_config.secondary_nodes) > 1, self.EINSTANCELAYOUT,
2713
               instance, "instance has multiple secondary nodes: %s",
2714
               utils.CommaJoin(inst_config.secondary_nodes),
2715
               code=self.ETYPE_WARNING)
2716

    
2717
      if inst_config.disk_template in constants.DTS_INT_MIRROR:
2718
        pnode = inst_config.primary_node
2719
        instance_nodes = utils.NiceSort(inst_config.all_nodes)
2720
        instance_groups = {}
2721

    
2722
        for node in instance_nodes:
2723
          instance_groups.setdefault(self.all_node_info[node].group,
2724
                                     []).append(node)
2725

    
2726
        pretty_list = [
2727
          "%s (group %s)" % (utils.CommaJoin(nodes), groupinfo[group].name)
2728
          # Sort so that we always list the primary node first.
2729
          for group, nodes in sorted(instance_groups.items(),
2730
                                     key=lambda (_, nodes): pnode in nodes,
2731
                                     reverse=True)]
2732

    
2733
        self._ErrorIf(len(instance_groups) > 1, self.EINSTANCESPLITGROUPS,
2734
                      instance, "instance has primary and secondary nodes in"
2735
                      " different groups: %s", utils.CommaJoin(pretty_list),
2736
                      code=self.ETYPE_WARNING)
2737

    
2738
      if not cluster.FillBE(inst_config)[constants.BE_AUTO_BALANCE]:
2739
        i_non_a_balanced.append(instance)
2740

    
2741
      for snode in inst_config.secondary_nodes:
2742
        s_img = node_image[snode]
2743
        _ErrorIf(s_img.rpc_fail and not s_img.offline, self.ENODERPC, snode,
2744
                 "instance %s, connection to secondary node failed", instance)
2745

    
2746
        if s_img.offline:
2747
          inst_nodes_offline.append(snode)
2748

    
2749
      # warn that the instance lives on offline nodes
2750
      _ErrorIf(inst_nodes_offline, self.EINSTANCEBADNODE, instance,
2751
               "instance has offline secondary node(s) %s",
2752
               utils.CommaJoin(inst_nodes_offline))
2753
      # ... or ghost/non-vm_capable nodes
2754
      for node in inst_config.all_nodes:
2755
        _ErrorIf(node_image[node].ghost, self.EINSTANCEBADNODE, instance,
2756
                 "instance lives on ghost node %s", node)
2757
        _ErrorIf(not node_image[node].vm_capable, self.EINSTANCEBADNODE,
2758
                 instance, "instance lives on non-vm_capable node %s", node)
2759

    
2760
    feedback_fn("* Verifying orphan volumes")
2761
    reserved = utils.FieldSet(*cluster.reserved_lvs)
2762

    
2763
    # We will get spurious "unknown volume" warnings if any node of this group
2764
    # is secondary for an instance whose primary is in another group. To avoid
2765
    # them, we find these instances and add their volumes to node_vol_should.
2766
    for inst in self.all_inst_info.values():
2767
      for secondary in inst.secondary_nodes:
2768
        if (secondary in self.my_node_info
2769
            and inst.name not in self.my_inst_info):
2770
          inst.MapLVsByNode(node_vol_should)
2771
          break
2772

    
2773
    self._VerifyOrphanVolumes(node_vol_should, node_image, reserved)
2774

    
2775
    if constants.VERIFY_NPLUSONE_MEM not in self.op.skip_checks:
2776
      feedback_fn("* Verifying N+1 Memory redundancy")
2777
      self._VerifyNPlusOneMemory(node_image, self.my_inst_info)
2778

    
2779
    feedback_fn("* Other Notes")
2780
    if i_non_redundant:
2781
      feedback_fn("  - NOTICE: %d non-redundant instance(s) found."
2782
                  % len(i_non_redundant))
2783

    
2784
    if i_non_a_balanced:
2785
      feedback_fn("  - NOTICE: %d non-auto-balanced instance(s) found."
2786
                  % len(i_non_a_balanced))
2787

    
2788
    if n_offline:
2789
      feedback_fn("  - NOTICE: %d offline node(s) found." % n_offline)
2790

    
2791
    if n_drained:
2792
      feedback_fn("  - NOTICE: %d drained node(s) found." % n_drained)
2793

    
2794
    return not self.bad
2795

    
2796
  def HooksCallBack(self, phase, hooks_results, feedback_fn, lu_result):
2797
    """Analyze the post-hooks' result
2798

2799
    This method analyses the hook result, handles it, and sends some
2800
    nicely-formatted feedback back to the user.
2801

2802
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
2803
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
2804
    @param hooks_results: the results of the multi-node hooks rpc call
2805
    @param feedback_fn: function used send feedback back to the caller
2806
    @param lu_result: previous Exec result
2807
    @return: the new Exec result, based on the previous result
2808
        and hook results
2809

2810
    """
2811
    # We only really run POST phase hooks, and are only interested in
2812
    # their results
2813
    if phase == constants.HOOKS_PHASE_POST:
2814
      # Used to change hooks' output to proper indentation
2815
      feedback_fn("* Hooks Results")
2816
      assert hooks_results, "invalid result from hooks"
2817

    
2818
      for node_name in hooks_results:
2819
        res = hooks_results[node_name]
2820
        msg = res.fail_msg
2821
        test = msg and not res.offline
2822
        self._ErrorIf(test, self.ENODEHOOKS, node_name,
2823
                      "Communication failure in hooks execution: %s", msg)
2824
        if res.offline or msg:
2825
          # No need to investigate payload if node is offline or gave an error.
2826
          # override manually lu_result here as _ErrorIf only
2827
          # overrides self.bad
2828
          lu_result = 1
2829
          continue
2830
        for script, hkr, output in res.payload:
2831
          test = hkr == constants.HKR_FAIL
2832
          self._ErrorIf(test, self.ENODEHOOKS, node_name,
2833
                        "Script %s failed, output:", script)
2834
          if test:
2835
            output = self._HOOKS_INDENT_RE.sub('      ', output)
2836
            feedback_fn("%s" % output)
2837
            lu_result = 0
2838

    
2839
      return lu_result
2840

    
2841

    
2842
class LUClusterVerifyDisks(NoHooksLU):
2843
  """Verifies the cluster disks status.
2844

2845
  """
2846
  REQ_BGL = False
2847

    
2848
  def ExpandNames(self):
2849
    self.needed_locks = {
2850
      locking.LEVEL_NODE: locking.ALL_SET,
2851
      locking.LEVEL_INSTANCE: locking.ALL_SET,
2852
    }
2853
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2854

    
2855
  def Exec(self, feedback_fn):
2856
    """Verify integrity of cluster disks.
2857

2858
    @rtype: tuple of three items
2859
    @return: a tuple of (dict of node-to-node_error, list of instances
2860
        which need activate-disks, dict of instance: (node, volume) for
2861
        missing volumes
2862

2863
    """
2864
    result = res_nodes, res_instances, res_missing = {}, [], {}
2865

    
2866
    nodes = utils.NiceSort(self.cfg.GetVmCapableNodeList())
2867
    instances = self.cfg.GetAllInstancesInfo().values()
2868

    
2869
    nv_dict = {}
2870
    for inst in instances:
2871
      inst_lvs = {}
2872
      if not inst.admin_up:
2873
        continue
2874
      inst.MapLVsByNode(inst_lvs)
2875
      # transform { iname: {node: [vol,],},} to {(node, vol): iname}
2876
      for node, vol_list in inst_lvs.iteritems():
2877
        for vol in vol_list:
2878
          nv_dict[(node, vol)] = inst
2879

    
2880
    if not nv_dict:
2881
      return result
2882

    
2883
    node_lvs = self.rpc.call_lv_list(nodes, [])
2884
    for node, node_res in node_lvs.items():
2885
      if node_res.offline:
2886
        continue
2887
      msg = node_res.fail_msg
2888
      if msg:
2889
        logging.warning("Error enumerating LVs on node %s: %s", node, msg)
2890
        res_nodes[node] = msg
2891
        continue
2892

    
2893
      lvs = node_res.payload
2894
      for lv_name, (_, _, lv_online) in lvs.items():
2895
        inst = nv_dict.pop((node, lv_name), None)
2896
        if (not lv_online and inst is not None
2897
            and inst.name not in res_instances):
2898
          res_instances.append(inst.name)
2899

    
2900
    # any leftover items in nv_dict are missing LVs, let's arrange the
2901
    # data better
2902
    for key, inst in nv_dict.iteritems():
2903
      if inst.name not in res_missing:
2904
        res_missing[inst.name] = []
2905
      res_missing[inst.name].append(key)
2906

    
2907
    return result
2908

    
2909

    
2910
class LUClusterRepairDiskSizes(NoHooksLU):
2911
  """Verifies the cluster disks sizes.
2912

2913
  """
2914
  REQ_BGL = False
2915

    
2916
  def ExpandNames(self):
2917
    if self.op.instances:
2918
      self.wanted_names = _GetWantedInstances(self, self.op.instances)
2919
      self.needed_locks = {
2920
        locking.LEVEL_NODE: [],
2921
        locking.LEVEL_INSTANCE: self.wanted_names,
2922
        }
2923
      self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE
2924
    else:
2925
      self.wanted_names = None
2926
      self.needed_locks = {
2927
        locking.LEVEL_NODE: locking.ALL_SET,
2928
        locking.LEVEL_INSTANCE: locking.ALL_SET,
2929
        }
2930
    self.share_locks = dict.fromkeys(locking.LEVELS, 1)
2931

    
2932
  def DeclareLocks(self, level):
2933
    if level == locking.LEVEL_NODE and self.wanted_names is not None:
2934
      self._LockInstancesNodes(primary_only=True)
2935

    
2936
  def CheckPrereq(self):
2937
    """Check prerequisites.
2938

2939
    This only checks the optional instance list against the existing names.
2940

2941
    """
2942
    if self.wanted_names is None:
2943
      self.wanted_names = self.glm.list_owned(locking.LEVEL_INSTANCE)
2944

    
2945
    self.wanted_instances = [self.cfg.GetInstanceInfo(name) for name
2946
                             in self.wanted_names]
2947

    
2948
  def _EnsureChildSizes(self, disk):
2949
    """Ensure children of the disk have the needed disk size.
2950

2951
    This is valid mainly for DRBD8 and fixes an issue where the
2952
    children have smaller disk size.
2953

2954
    @param disk: an L{ganeti.objects.Disk} object
2955

2956
    """
2957
    if disk.dev_type == constants.LD_DRBD8:
2958
      assert disk.children, "Empty children for DRBD8?"
2959
      fchild = disk.children[0]
2960
      mismatch = fchild.size < disk.size
2961
      if mismatch:
2962
        self.LogInfo("Child disk has size %d, parent %d, fixing",
2963
                     fchild.size, disk.size)
2964
        fchild.size = disk.size
2965

    
2966
      # and we recurse on this child only, not on the metadev
2967
      return self._EnsureChildSizes(fchild) or mismatch
2968
    else:
2969
      return False
2970

    
2971
  def Exec(self, feedback_fn):
2972
    """Verify the size of cluster disks.
2973

2974
    """
2975
    # TODO: check child disks too
2976
    # TODO: check differences in size between primary/secondary nodes
2977
    per_node_disks = {}
2978
    for instance in self.wanted_instances:
2979
      pnode = instance.primary_node
2980
      if pnode not in per_node_disks:
2981
        per_node_disks[pnode] = []
2982
      for idx, disk in enumerate(instance.disks):
2983
        per_node_disks[pnode].append((instance, idx, disk))
2984

    
2985
    changed = []
2986
    for node, dskl in per_node_disks.items():
2987
      newl = [v[2].Copy() for v in dskl]
2988
      for dsk in newl:
2989
        self.cfg.SetDiskID(dsk, node)
2990
      result = self.rpc.call_blockdev_getsize(node, newl)
2991
      if result.fail_msg:
2992
        self.LogWarning("Failure in blockdev_getsize call to node"
2993
                        " %s, ignoring", node)
2994
        continue
2995
      if len(result.payload) != len(dskl):
2996
        logging.warning("Invalid result from node %s: len(dksl)=%d,"
2997
                        " result.payload=%s", node, len(dskl), result.payload)
2998
        self.LogWarning("Invalid result from node %s, ignoring node results",
2999
                        node)
3000
        continue
3001
      for ((instance, idx, disk), size) in zip(dskl, result.payload):
3002
        if size is None:
3003
          self.LogWarning("Disk %d of instance %s did not return size"
3004
                          " information, ignoring", idx, instance.name)
3005
          continue
3006
        if not isinstance(size, (int, long)):
3007
          self.LogWarning("Disk %d of instance %s did not return valid"
3008
                          " size information, ignoring", idx, instance.name)
3009
          continue
3010
        size = size >> 20
3011
        if size != disk.size:
3012
          self.LogInfo("Disk %d of instance %s has mismatched size,"
3013
                       " correcting: recorded %d, actual %d", idx,
3014
                       instance.name, disk.size, size)
3015
          disk.size = size
3016
          self.cfg.Update(instance, feedback_fn)
3017
          changed.append((instance.name, idx, size))
3018
        if self._EnsureChildSizes(disk):
3019
          self.cfg.Update(instance, feedback_fn)
3020
          changed.append((instance.name, idx, disk.size))
3021
    return changed
3022

    
3023

    
3024
class LUClusterRename(LogicalUnit):
3025
  """Rename the cluster.
3026

3027
  """
3028
  HPATH = "cluster-rename"
3029
  HTYPE = constants.HTYPE_CLUSTER
3030

    
3031
  def BuildHooksEnv(self):
3032
    """Build hooks env.
3033

3034
    """
3035
    return {
3036
      "OP_TARGET": self.cfg.GetClusterName(),
3037
      "NEW_NAME": self.op.name,
3038
      }
3039

    
3040
  def BuildHooksNodes(self):
3041
    """Build hooks nodes.
3042

3043
    """
3044
    return ([self.cfg.GetMasterNode()], self.cfg.GetNodeList())
3045

    
3046
  def CheckPrereq(self):
3047
    """Verify that the passed name is a valid one.
3048

3049
    """
3050
    hostname = netutils.GetHostname(name=self.op.name,
3051
                                    family=self.cfg.GetPrimaryIPFamily())
3052

    
3053
    new_name = hostname.name
3054
    self.ip = new_ip = hostname.ip
3055
    old_name = self.cfg.GetClusterName()
3056
    old_ip = self.cfg.GetMasterIP()
3057
    if new_name == old_name and new_ip == old_ip:
3058
      raise errors.OpPrereqError("Neither the name nor the IP address of the"
3059
                                 " cluster has changed",
3060
                                 errors.ECODE_INVAL)
3061
    if new_ip != old_ip:
3062
      if netutils.TcpPing(new_ip, constants.DEFAULT_NODED_PORT):
3063
        raise errors.OpPrereqError("The given cluster IP address (%s) is"
3064
                                   " reachable on the network" %
3065
                                   new_ip, errors.ECODE_NOTUNIQUE)
3066

    
3067
    self.op.name = new_name
3068

    
3069
  def Exec(self, feedback_fn):
3070
    """Rename the cluster.
3071

3072
    """
3073
    clustername = self.op.name
3074
    ip = self.ip
3075

    
3076
    # shutdown the master IP
3077
    master = self.cfg.GetMasterNode()
3078
    result = self.rpc.call_node_stop_master(master, False)
3079
    result.Raise("Could not disable the master role")
3080

    
3081
    try:
3082
      cluster = self.cfg.GetClusterInfo()
3083
      cluster.cluster_name = clustername
3084
      cluster.master_ip = ip
3085
      self.cfg.Update(cluster, feedback_fn)
3086

    
3087
      # update the known hosts file
3088
      ssh.WriteKnownHostsFile(self.cfg, constants.SSH_KNOWN_HOSTS_FILE)
3089
      node_list = self.cfg.GetOnlineNodeList()
3090
      try:
3091
        node_list.remove(master)
3092
      except ValueError:
3093
        pass
3094
      _UploadHelper(self, node_list, constants.SSH_KNOWN_HOSTS_FILE)
3095
    finally:
3096
      result = self.rpc.call_node_start_master(master, False, False)
3097
      msg = result.fail_msg
3098
      if msg:
3099
        self.LogWarning("Could not re-enable the master role on"
3100
                        " the master, please restart manually: %s", msg)
3101

    
3102
    return clustername
3103

    
3104

    
3105
class LUClusterSetParams(LogicalUnit):
3106
  """Change the parameters of the cluster.
3107

3108
  """
3109
  HPATH = "cluster-modify"
3110
  HTYPE = constants.HTYPE_CLUSTER
3111
  REQ_BGL = False
3112

    
3113
  def CheckArguments(self):
3114
    """Check parameters
3115

3116
    """
3117
    if self.op.uid_pool:
3118
      uidpool.CheckUidPool(self.op.uid_pool)
3119

    
3120
    if self.op.add_uids:
3121
      uidpool.CheckUidPool(self.op.add_uids)
3122

    
3123
    if self.op.remove_uids:
3124
      uidpool.CheckUidPool(self.op.remove_uids)
3125

    
3126
  def ExpandNames(self):
3127
    # FIXME: in the future maybe other cluster params won't require checking on
3128
    # all nodes to be modified.
3129
    self.needed_locks = {
3130
      locking.LEVEL_NODE: locking.ALL_SET,
3131
    }
3132
    self.share_locks[locking.LEVEL_NODE] = 1
3133

    
3134
  def BuildHooksEnv(self):
3135
    """Build hooks env.
3136

3137
    """
3138
    return {
3139
      "OP_TARGET": self.cfg.GetClusterName(),
3140
      "NEW_VG_NAME": self.op.vg_name,
3141
      }
3142

    
3143
  def BuildHooksNodes(self):
3144
    """Build hooks nodes.
3145

3146
    """
3147
    mn = self.cfg.GetMasterNode()
3148
    return ([mn], [mn])
3149

    
3150
  def CheckPrereq(self):
3151
    """Check prerequisites.
3152

3153
    This checks whether the given params don't conflict and
3154
    if the given volume group is valid.
3155

3156
    """
3157
    if self.op.vg_name is not None and not self.op.vg_name:
3158
      if self.cfg.HasAnyDiskOfType(constants.LD_LV):
3159
        raise errors.OpPrereqError("Cannot disable lvm storage while lvm-based"
3160
                                   " instances exist", errors.ECODE_INVAL)
3161

    
3162
    if self.op.drbd_helper is not None and not self.op.drbd_helper:
3163
      if self.cfg.HasAnyDiskOfType(constants.LD_DRBD8):
3164
        raise errors.OpPrereqError("Cannot disable drbd helper while"
3165
                                   " drbd-based instances exist",
3166
                                   errors.ECODE_INVAL)
3167

    
3168
    node_list = self.glm.list_owned(locking.LEVEL_NODE)
3169

    
3170
    # if vg_name not None, checks given volume group on all nodes
3171
    if self.op.vg_name:
3172
      vglist = self.rpc.call_vg_list(node_list)
3173
      for node in node_list:
3174
        msg = vglist[node].fail_msg
3175
        if msg:
3176
          # ignoring down node
3177
          self.LogWarning("Error while gathering data on node %s"
3178
                          " (ignoring node): %s", node, msg)
3179
          continue
3180
        vgstatus = utils.CheckVolumeGroupSize(vglist[node].payload,
3181
                                              self.op.vg_name,
3182
                                              constants.MIN_VG_SIZE)
3183
        if vgstatus:
3184
          raise errors.OpPrereqError("Error on node '%s': %s" %
3185
                                     (node, vgstatus), errors.ECODE_ENVIRON)
3186

    
3187
    if self.op.drbd_helper:
3188
      # checks given drbd helper on all nodes
3189
      helpers = self.rpc.call_drbd_helper(node_list)
3190
      for node in node_list:
3191
        ninfo = self.cfg.GetNodeInfo(node)
3192
        if ninfo.offline:
3193
          self.LogInfo("Not checking drbd helper on offline node %s", node)
3194
          continue
3195
        msg = helpers[node].fail_msg
3196
        if msg:
3197
          raise errors.OpPrereqError("Error checking drbd helper on node"
3198
                                     " '%s': %s" % (node, msg),
3199
                                     errors.ECODE_ENVIRON)
3200
        node_helper = helpers[node].payload
3201
        if node_helper != self.op.drbd_helper:
3202
          raise errors.OpPrereqError("Error on node '%s': drbd helper is %s" %
3203
                                     (node, node_helper), errors.ECODE_ENVIRON)
3204

    
3205
    self.cluster = cluster = self.cfg.GetClusterInfo()
3206
    # validate params changes
3207
    if self.op.beparams:
3208
      utils.ForceDictType(self.op.beparams, constants.BES_PARAMETER_TYPES)
3209
      self.new_beparams = cluster.SimpleFillBE(self.op.beparams)
3210

    
3211
    if self.op.ndparams:
3212
      utils.ForceDictType(self.op.ndparams, constants.NDS_PARAMETER_TYPES)
3213
      self.new_ndparams = cluster.SimpleFillND(self.op.ndparams)
3214

    
3215
      # TODO: we need a more general way to handle resetting
3216
      # cluster-level parameters to default values
3217
      if self.new_ndparams["oob_program"] == "":
3218
        self.new_ndparams["oob_program"] = \
3219
            constants.NDC_DEFAULTS[constants.ND_OOB_PROGRAM]
3220

    
3221
    if self.op.nicparams:
3222
      utils.ForceDictType(self.op.nicparams, constants.NICS_PARAMETER_TYPES)
3223
      self.new_nicparams = cluster.SimpleFillNIC(self.op.nicparams)
3224
      objects.NIC.CheckParameterSyntax(self.new_nicparams)
3225
      nic_errors = []
3226

    
3227
      # check all instances for consistency
3228
      for instance in self.cfg.GetAllInstancesInfo().values():
3229
        for nic_idx, nic in enumerate(instance.nics):
3230
          params_copy = copy.deepcopy(nic.nicparams)
3231
          params_filled = objects.FillDict(self.new_nicparams, params_copy)
3232

    
3233
          # check parameter syntax
3234
          try:
3235
            objects.NIC.CheckParameterSyntax(params_filled)
3236
          except errors.ConfigurationError, err:
3237
            nic_errors.append("Instance %s, nic/%d: %s" %
3238
                              (instance.name, nic_idx, err))
3239

    
3240
          # if we're moving instances to routed, check that they have an ip
3241
          target_mode = params_filled[constants.NIC_MODE]
3242
          if target_mode == constants.NIC_MODE_ROUTED and not nic.ip:
3243
            nic_errors.append("Instance %s, nic/%d: routed NIC with no ip"
3244
                              " address" % (instance.name, nic_idx))
3245
      if nic_errors:
3246
        raise errors.OpPrereqError("Cannot apply the change, errors:\n%s" %
3247
                                   "\n".join(nic_errors))
3248

    
3249
    # hypervisor list/parameters
3250
    self.new_hvparams = new_hvp = objects.FillDict(cluster.hvparams, {})
3251
    if self.op.hvparams:
3252
      for hv_name, hv_dict in self.op.hvparams.items():
3253
        if hv_name not in self.new_hvparams:
3254
          self.new_hvparams[hv_name] = hv_dict
3255
        else:
3256
          self.new_hvparams[hv_name].update(hv_dict)
3257

    
3258
    # os hypervisor parameters
3259
    self.new_os_hvp = objects.FillDict(cluster.os_hvp, {})
3260
    if self.op.os_hvp:
3261
      for os_name, hvs in self.op.os_hvp.items():
3262
        if os_name not in self.new_os_hvp:
3263
          self.new_os_hvp[os_name] = hvs
3264
        else:
3265
          for hv_name, hv_dict in hvs.items():
3266
            if hv_name not in self.new_os_hvp[os_name]:
3267
              self.new_os_hvp[os_name][hv_name] = hv_dict
3268
            else:
3269
              self.new_os_hvp[os_name][hv_name].update(hv_dict)
3270

    
3271
    # os parameters
3272
    self.new_osp = objects.FillDict(cluster.osparams, {})
3273
    if self.op.osparams:
3274
      for os_name, osp in self.op.osparams.items():
3275
        if os_name not in self.new_osp:
3276
          self.new_osp[os_name] = {}
3277

    
3278
        self.new_osp[os_name] = _GetUpdatedParams(self.new_osp[os_name], osp,
3279
                                                  use_none=True)
3280

    
3281
        if not self.new_osp[os_name]:
3282
          # we removed all parameters
3283
          del self.new_osp[os_name]
3284
        else:
3285
          # check the parameter validity (remote check)
3286
          _CheckOSParams(self, False, [self.cfg.GetMasterNode()],
3287
                         os_name, self.new_osp[os_name])
3288

    
3289
    # changes to the hypervisor list
3290
    if self.op.enabled_hypervisors is not None:
3291
      self.hv_list = self.op.enabled_hypervisors
3292
      for hv in self.hv_list:
3293
        # if the hypervisor doesn't already exist in the cluster
3294
        # hvparams, we initialize it to empty, and then (in both
3295
        # cases) we make sure to fill the defaults, as we might not
3296
        # have a complete defaults list if the hypervisor wasn't
3297
        # enabled before
3298
        if hv not in new_hvp:
3299
          new_hvp[hv] = {}
3300
        new_hvp[hv] = objects.FillDict(constants.HVC_DEFAULTS[hv], new_hvp[hv])
3301
        utils.ForceDictType(new_hvp[hv], constants.HVS_PARAMETER_TYPES)
3302
    else:
3303
      self.hv_list = cluster.enabled_hypervisors
3304

    
3305
    if self.op.hvparams or self.op.enabled_hypervisors is not None:
3306
      # either the enabled list has changed, or the parameters have, validate
3307
      for hv_name, hv_params in self.new_hvparams.items():
3308
        if ((self.op.hvparams and hv_name in self.op.hvparams) or
3309
            (self.op.enabled_hypervisors and
3310
             hv_name in self.op.enabled_hypervisors)):
3311
          # either this is a new hypervisor, or its parameters have changed
3312
          hv_class = hypervisor.GetHypervisor(hv_name)
3313
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3314
          hv_class.CheckParameterSyntax(hv_params)
3315
          _CheckHVParams(self, node_list, hv_name, hv_params)
3316

    
3317
    if self.op.os_hvp:
3318
      # no need to check any newly-enabled hypervisors, since the
3319
      # defaults have already been checked in the above code-block
3320
      for os_name, os_hvp in self.new_os_hvp.items():
3321
        for hv_name, hv_params in os_hvp.items():
3322
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3323
          # we need to fill in the new os_hvp on top of the actual hv_p
3324
          cluster_defaults = self.new_hvparams.get(hv_name, {})
3325
          new_osp = objects.FillDict(cluster_defaults, hv_params)
3326
          hv_class = hypervisor.GetHypervisor(hv_name)
3327
          hv_class.CheckParameterSyntax(new_osp)
3328
          _CheckHVParams(self, node_list, hv_name, new_osp)
3329

    
3330
    if self.op.default_iallocator:
3331
      alloc_script = utils.FindFile(self.op.default_iallocator,
3332
                                    constants.IALLOCATOR_SEARCH_PATH,
3333
                                    os.path.isfile)
3334
      if alloc_script is None:
3335
        raise errors.OpPrereqError("Invalid default iallocator script '%s'"
3336
                                   " specified" % self.op.default_iallocator,
3337
                                   errors.ECODE_INVAL)
3338

    
3339
  def Exec(self, feedback_fn):
3340
    """Change the parameters of the cluster.
3341

3342
    """
3343
    if self.op.vg_name is not None:
3344
      new_volume = self.op.vg_name
3345
      if not new_volume:
3346
        new_volume = None
3347
      if new_volume != self.cfg.GetVGName():
3348
        self.cfg.SetVGName(new_volume)
3349
      else:
3350
        feedback_fn("Cluster LVM configuration already in desired"
3351
                    " state, not changing")
3352
    if self.op.drbd_helper is not None:
3353
      new_helper = self.op.drbd_helper
3354
      if not new_helper:
3355
        new_helper = None
3356
      if new_helper != self.cfg.GetDRBDHelper():
3357
        self.cfg.SetDRBDHelper(new_helper)
3358
      else:
3359
        feedback_fn("Cluster DRBD helper already in desired state,"
3360
                    " not changing")
3361
    if self.op.hvparams:
3362
      self.cluster.hvparams = self.new_hvparams
3363
    if self.op.os_hvp:
3364
      self.cluster.os_hvp = self.new_os_hvp
3365
    if self.op.enabled_hypervisors is not None:
3366
      self.cluster.hvparams = self.new_hvparams
3367
      self.cluster.enabled_hypervisors = self.op.enabled_hypervisors
3368
    if self.op.beparams:
3369
      self.cluster.beparams[constants.PP_DEFAULT] = self.new_beparams
3370
    if self.op.nicparams:
3371
      self.cluster.nicparams[constants.PP_DEFAULT] = self.new_nicparams
3372
    if self.op.osparams:
3373
      self.cluster.osparams = self.new_osp
3374
    if self.op.ndparams:
3375
      self.cluster.ndparams = self.new_ndparams
3376

    
3377
    if self.op.candidate_pool_size is not None:
3378
      self.cluster.candidate_pool_size = self.op.candidate_pool_size
3379
      # we need to update the pool size here, otherwise the save will fail
3380
      _AdjustCandidatePool(self, [])
3381

    
3382
    if self.op.maintain_node_health is not None:
3383
      self.cluster.maintain_node_health = self.op.maintain_node_health
3384

    
3385
    if self.op.prealloc_wipe_disks is not None:
3386
      self.cluster.prealloc_wipe_disks = self.op.prealloc_wipe_disks
3387

    
3388
    if self.op.add_uids is not None:
3389
      uidpool.AddToUidPool(self.cluster.uid_pool, self.op.add_uids)
3390

    
3391
    if self.op.remove_uids is not None:
3392
      uidpool.RemoveFromUidPool(self.cluster.uid_pool, self.op.remove_uids)
3393

    
3394
    if self.op.uid_pool is not None:
3395
      self.cluster.uid_pool = self.op.uid_pool
3396

    
3397
    if self.op.default_iallocator is not None:
3398
      self.cluster.default_iallocator = self.op.default_iallocator
3399

    
3400
    if self.op.reserved_lvs is not None:
3401
      self.cluster.reserved_lvs = self.op.reserved_lvs
3402

    
3403
    def helper_os(aname, mods, desc):
3404
      desc += " OS list"
3405
      lst = getattr(self.cluster, aname)
3406
      for key, val in mods:
3407
        if key == constants.DDM_ADD:
3408
          if val in lst:
3409
            feedback_fn("OS %s already in %s, ignoring" % (val, desc))
3410
          else:
3411
            lst.append(val)
3412
        elif key == constants.DDM_REMOVE:
3413
          if val in lst:
3414
            lst.remove(val)
3415
          else:
3416
            feedback_fn("OS %s not found in %s, ignoring" % (val, desc))
3417
        else:
3418
          raise errors.ProgrammerError("Invalid modification '%s'" % key)
3419

    
3420
    if self.op.hidden_os:
3421
      helper_os("hidden_os", self.op.hidden_os, "hidden")
3422

    
3423
    if self.op.blacklisted_os:
3424
      helper_os("blacklisted_os", self.op.blacklisted_os, "blacklisted")
3425

    
3426
    if self.op.master_netdev:
3427
      master = self.cfg.GetMasterNode()
3428
      feedback_fn("Shutting down master ip on the current netdev (%s)" %
3429
                  self.cluster.master_netdev)
3430
      result = self.rpc.call_node_stop_master(master, False)
3431
      result.Raise("Could not disable the master ip")
3432
      feedback_fn("Changing master_netdev from %s to %s" %
3433
                  (self.cluster.master_netdev, self.op.master_netdev))
3434
      self.cluster.master_netdev = self.op.master_netdev
3435

    
3436
    self.cfg.Update(self.cluster, feedback_fn)
3437

    
3438
    if self.op.master_netdev:
3439
      feedback_fn("Starting the master ip on the new master netdev (%s)" %
3440
                  self.op.master_netdev)
3441
      result = self.rpc.call_node_start_master(master, False, False)
3442
      if result.fail_msg:
3443
        self.LogWarning("Could not re-enable the master ip on"
3444
                        " the master, please restart manually: %s",
3445
                        result.fail_msg)
3446

    
3447

    
3448
def _UploadHelper(lu, nodes, fname):
3449
  """Helper for uploading a file and showing warnings.
3450

3451
  """
3452
  if os.path.exists(fname):
3453
    result = lu.rpc.call_upload_file(nodes, fname)
3454
    for to_node, to_result in result.items():
3455
      msg = to_result.fail_msg
3456
      if msg:
3457
        msg = ("Copy of file %s to node %s failed: %s" %
3458
               (fname, to_node, msg))
3459
        lu.proc.LogWarning(msg)
3460

    
3461

    
3462
def _ComputeAncillaryFiles(cluster, redist):
3463
  """Compute files external to Ganeti which need to be consistent.
3464

3465
  @type redist: boolean
3466
  @param redist: Whether to include files which need to be redistributed
3467

3468
  """
3469
  # Compute files for all nodes
3470
  files_all = set([
3471
    constants.SSH_KNOWN_HOSTS_FILE,
3472
    constants.CONFD_HMAC_KEY,
3473
    constants.CLUSTER_DOMAIN_SECRET_FILE,
3474
    ])
3475

    
3476
  if not redist:
3477
    files_all.update(constants.ALL_CERT_FILES)
3478
    files_all.update(ssconf.SimpleStore().GetFileList())
3479

    
3480
  if cluster.modify_etc_hosts:
3481
    files_all.add(constants.ETC_HOSTS)
3482

    
3483
  # Files which must either exist on all nodes or on none
3484
  files_all_opt = set([
3485
    constants.RAPI_USERS_FILE,
3486
    ])
3487

    
3488
  # Files which should only be on master candidates
3489
  files_mc = set()
3490
  if not redist:
3491
    files_mc.add(constants.CLUSTER_CONF_FILE)
3492

    
3493
  # Files which should only be on VM-capable nodes
3494
  files_vm = set(filename
3495
    for hv_name in cluster.enabled_hypervisors
3496
    for filename in hypervisor.GetHypervisor(hv_name).GetAncillaryFiles())
3497

    
3498
  # Filenames must be unique
3499
  assert (len(files_all | files_all_opt | files_mc | files_vm) ==
3500
          sum(map(len, [files_all, files_all_opt, files_mc, files_vm]))), \
3501
         "Found file listed in more than one file list"
3502

    
3503
  return (files_all, files_all_opt, files_mc, files_vm)
3504

    
3505

    
3506
def _RedistributeAncillaryFiles(lu, additional_nodes=None, additional_vm=True):
3507
  """Distribute additional files which are part of the cluster configuration.
3508

3509
  ConfigWriter takes care of distributing the config and ssconf files, but
3510
  there are more files which should be distributed to all nodes. This function
3511
  makes sure those are copied.
3512

3513
  @param lu: calling logical unit
3514
  @param additional_nodes: list of nodes not in the config to distribute to
3515
  @type additional_vm: boolean
3516
  @param additional_vm: whether the additional nodes are vm-capable or not
3517

3518
  """
3519
  # Gather target nodes
3520
  cluster = lu.cfg.GetClusterInfo()
3521
  master_info = lu.cfg.GetNodeInfo(lu.cfg.GetMasterNode())
3522

    
3523
  online_nodes = lu.cfg.GetOnlineNodeList()
3524
  vm_nodes = lu.cfg.GetVmCapableNodeList()
3525

    
3526
  if additional_nodes is not None:
3527
    online_nodes.extend(additional_nodes)
3528
    if additional_vm:
3529
      vm_nodes.extend(additional_nodes)
3530

    
3531
  # Never distribute to master node
3532
  for nodelist in [online_nodes, vm_nodes]:
3533
    if master_info.name in nodelist:
3534
      nodelist.remove(master_info.name)
3535

    
3536
  # Gather file lists
3537
  (files_all, files_all_opt, files_mc, files_vm) = \
3538
    _ComputeAncillaryFiles(cluster, True)
3539

    
3540
  # Never re-distribute configuration file from here
3541
  assert not (constants.CLUSTER_CONF_FILE in files_all or
3542
              constants.CLUSTER_CONF_FILE in files_vm)
3543
  assert not files_mc, "Master candidates not handled in this function"
3544

    
3545
  filemap = [
3546
    (online_nodes, files_all),
3547
    (online_nodes, files_all_opt),
3548
    (vm_nodes, files_vm),
3549
    ]
3550

    
3551
  # Upload the files
3552
  for (node_list, files) in filemap:
3553
    for fname in files:
3554
      _UploadHelper(lu, node_list, fname)
3555

    
3556

    
3557
class LUClusterRedistConf(NoHooksLU):
3558
  """Force the redistribution of cluster configuration.
3559

3560
  This is a very simple LU.
3561

3562
  """
3563
  REQ_BGL = False
3564

    
3565
  def ExpandNames(self):
3566
    self.needed_locks = {
3567
      locking.LEVEL_NODE: locking.ALL_SET,
3568
    }
3569
    self.share_locks[locking.LEVEL_NODE] = 1
3570

    
3571
  def Exec(self, feedback_fn):
3572
    """Redistribute the configuration.
3573

3574
    """
3575
    self.cfg.Update(self.cfg.GetClusterInfo(), feedback_fn)
3576
    _RedistributeAncillaryFiles(self)
3577

    
3578

    
3579
def _WaitForSync(lu, instance, disks=None, oneshot=False):
3580
  """Sleep and poll for an instance's disk to sync.
3581

3582
  """
3583
  if not instance.disks or disks is not None and not disks:
3584
    return True
3585

    
3586
  disks = _ExpandCheckDisks(instance, disks)
3587

    
3588
  if not oneshot:
3589
    lu.proc.LogInfo("Waiting for instance %s to sync disks." % instance.name)
3590

    
3591
  node = instance.primary_node
3592

    
3593
  for dev in disks:
3594
    lu.cfg.SetDiskID(dev, node)
3595

    
3596
  # TODO: Convert to utils.Retry
3597

    
3598
  retries = 0
3599
  degr_retries = 10 # in seconds, as we sleep 1 second each time
3600
  while True:
3601
    max_time = 0
3602
    done = True
3603
    cumul_degraded = False
3604
    rstats = lu.rpc.call_blockdev_getmirrorstatus(node, disks)
3605
    msg = rstats.fail_msg
3606
    if msg:
3607
      lu.LogWarning("Can't get any data from node %s: %s", node, msg)
3608
      retries += 1
3609
      if retries >= 10:
3610
        raise errors.RemoteError("Can't contact node %s for mirror data,"
3611
                                 " aborting." % node)
3612
      time.sleep(6)
3613
      continue
3614
    rstats = rstats.payload
3615
    retries = 0
3616
    for i, mstat in enumerate(rstats):
3617
      if mstat is None:
3618
        lu.LogWarning("Can't compute data for node %s/%s",
3619
                           node, disks[i].iv_name)
3620
        continue
3621

    
3622
      cumul_degraded = (cumul_degraded or
3623
                        (mstat.is_degraded and mstat.sync_percent is None))
3624
      if mstat.sync_percent is not None:
3625
        done = False
3626
        if mstat.estimated_time is not None:
3627
          rem_time = ("%s remaining (estimated)" %
3628
                      utils.FormatSeconds(mstat.estimated_time))
3629
          max_time = mstat.estimated_time
3630
        else:
3631
          rem_time = "no time estimate"
3632
        lu.proc.LogInfo("- device %s: %5.2f%% done, %s" %
3633
                        (disks[i].iv_name, mstat.sync_percent, rem_time))
3634

    
3635
    # if we're done but degraded, let's do a few small retries, to
3636
    # make sure we see a stable and not transient situation; therefore
3637
    # we force restart of the loop
3638
    if (done or oneshot) and cumul_degraded and degr_retries > 0:
3639
      logging.info("Degraded disks found, %d retries left", degr_retries)
3640
      degr_retries -= 1
3641
      time.sleep(1)
3642
      continue
3643

    
3644
    if done or oneshot:
3645
      break
3646

    
3647
    time.sleep(min(60, max_time))
3648

    
3649
  if done:
3650
    lu.proc.LogInfo("Instance %s's disks are in sync." % instance.name)
3651
  return not cumul_degraded
3652

    
3653

    
3654
def _CheckDiskConsistency(lu, dev, node, on_primary, ldisk=False):
3655
  """Check that mirrors are not degraded.
3656

3657
  The ldisk parameter, if True, will change the test from the
3658
  is_degraded attribute (which represents overall non-ok status for
3659
  the device(s)) to the ldisk (representing the local storage status).
3660

3661
  """
3662
  lu.cfg.SetDiskID(dev, node)
3663

    
3664
  result = True
3665

    
3666
  if on_primary or dev.AssembleOnSecondary():
3667
    rstats = lu.rpc.call_blockdev_find(node, dev)
3668
    msg = rstats.fail_msg
3669
    if msg:
3670
      lu.LogWarning("Can't find disk on node %s: %s", node, msg)
3671
      result = False
3672
    elif not rstats.payload:
3673
      lu.LogWarning("Can't find disk on node %s", node)
3674
      result = False
3675
    else:
3676
      if ldisk:
3677
        result = result and rstats.payload.ldisk_status == constants.LDS_OKAY
3678
      else:
3679
        result = result and not rstats.payload.is_degraded
3680

    
3681
  if dev.children:
3682
    for child in dev.children:
3683
      result = result and _CheckDiskConsistency(lu, child, node, on_primary)
3684

    
3685
  return result
3686

    
3687

    
3688
class LUOobCommand(NoHooksLU):
3689
  """Logical unit for OOB handling.
3690

3691
  """
3692
  REG_BGL = False
3693
  _SKIP_MASTER = (constants.OOB_POWER_OFF, constants.OOB_POWER_CYCLE)
3694

    
3695
  def ExpandNames(self):
3696
    """Gather locks we need.
3697

3698
    """
3699
    if self.op.node_names:
3700
      self.op.node_names = _GetWantedNodes(self, self.op.node_names)
3701
      lock_names = self.op.node_names
3702
    else:
3703
      lock_names = locking.ALL_SET
3704

    
3705
    self.needed_locks = {
3706
      locking.LEVEL_NODE: lock_names,
3707
      }
3708

    
3709
  def CheckPrereq(self):
3710
    """Check prerequisites.
3711

3712
    This checks:
3713
     - the node exists in the configuration
3714
     - OOB is supported
3715

3716
    Any errors are signaled by raising errors.OpPrereqError.
3717

3718
    """
3719
    self.nodes = []
3720
    self.master_node = self.cfg.GetMasterNode()
3721

    
3722
    assert self.op.power_delay >= 0.0
3723

    
3724
    if self.op.node_names:
3725
      if (self.op.command in self._SKIP_MASTER and
3726
          self.master_node in self.op.node_names):
3727
        master_node_obj = self.cfg.GetNodeInfo(self.master_node)
3728
        master_oob_handler = _SupportsOob(self.cfg, master_node_obj)
3729

    
3730
        if master_oob_handler:
3731
          additional_text = ("run '%s %s %s' if you want to operate on the"
3732
                             " master regardless") % (master_oob_handler,
3733
                                                      self.op.command,
3734
                                                      self.master_node)
3735
        else:
3736
          additional_text = "it does not support out-of-band operations"
3737

    
3738
        raise errors.OpPrereqError(("Operating on the master node %s is not"
3739
                                    " allowed for %s; %s") %
3740
                                   (self.master_node, self.op.command,
3741
                                    additional_text), errors.ECODE_INVAL)
3742
    else:
3743
      self.op.node_names = self.cfg.GetNodeList()
3744
      if self.op.command in self._SKIP_MASTER:
3745
        self.op.node_names.remove(self.master_node)
3746

    
3747
    if self.op.command in self._SKIP_MASTER:
3748
      assert self.master_node not in self.op.node_names
3749

    
3750
    for node_name in self.op.node_names:
3751
      node = self.cfg.GetNodeInfo(node_name)
3752

    
3753
      if node is None:
3754
        raise errors.OpPrereqError("Node %s not found" % node_name,
3755
                                   errors.ECODE_NOENT)
3756
      else:
3757
        self.nodes.append(node)
3758

    
3759
      if (not self.op.ignore_status and
3760
          (self.op.command == constants.OOB_POWER_OFF and not node.offline)):
3761
        raise errors.OpPrereqError(("Cannot power off node %s because it is"
3762
                                    " not marked offline") % node_name,
3763
                                   errors.ECODE_STATE)
3764

    
3765
  def Exec(self, feedback_fn):
3766
    """Execute OOB and return result if we expect any.
3767

3768
    """
3769
    master_node = self.master_node
3770
    ret = []
3771

    
3772
    for idx, node in enumerate(utils.NiceSort(self.nodes,
3773
                                              key=lambda node: node.name)):
3774
      node_entry = [(constants.RS_NORMAL, node.name)]
3775
      ret.append(node_entry)
3776

    
3777
      oob_program = _SupportsOob(self.cfg, node)
3778

    
3779
      if not oob_program:
3780
        node_entry.append((constants.RS_UNAVAIL, None))
3781
        continue
3782

    
3783
      logging.info("Executing out-of-band command '%s' using '%s' on %s",
3784
                   self.op.command, oob_program, node.name)
3785
      result = self.rpc.call_run_oob(master_node, oob_program,
3786
                                     self.op.command, node.name,
3787
                                     self.op.timeout)
3788

    
3789
      if result.fail_msg:
3790
        self.LogWarning("Out-of-band RPC failed on node '%s': %s",
3791
                        node.name, result.fail_msg)
3792
        node_entry.append((constants.RS_NODATA, None))
3793
      else:
3794
        try:
3795
          self._CheckPayload(result)
3796
        except errors.OpExecError, err:
3797
          self.LogWarning("Payload returned by node '%s' is not valid: %s",
3798
                          node.name, err)
3799
          node_entry.append((constants.RS_NODATA, None))
3800
        else:
3801
          if self.op.command == constants.OOB_HEALTH:
3802
            # For health we should log important events
3803
            for item, status in result.payload:
3804
              if status in [constants.OOB_STATUS_WARNING,
3805
                            constants.OOB_STATUS_CRITICAL]:
3806
                self.LogWarning("Item '%s' on node '%s' has status '%s'",
3807
                                item, node.name, status)
3808

    
3809
          if self.op.command == constants.OOB_POWER_ON:
3810
            node.powered = True
3811
          elif self.op.command == constants.OOB_POWER_OFF:
3812
            node.powered = False
3813
          elif self.op.command == constants.OOB_POWER_STATUS:
3814
            powered = result.payload[constants.OOB_POWER_STATUS_POWERED]
3815
            if powered != node.powered:
3816
              logging.warning(("Recorded power state (%s) of node '%s' does not"
3817
                               " match actual power state (%s)"), node.powered,
3818
                              node.name, powered)
3819

    
3820
          # For configuration changing commands we should update the node
3821
          if self.op.command in (constants.OOB_POWER_ON,
3822
                                 constants.OOB_POWER_OFF):
3823
            self.cfg.Update(node, feedback_fn)
3824

    
3825
          node_entry.append((constants.RS_NORMAL, result.payload))
3826

    
3827
          if (self.op.command == constants.OOB_POWER_ON and
3828
              idx < len(self.nodes) - 1):
3829
            time.sleep(self.op.power_delay)
3830

    
3831
    return ret
3832

    
3833
  def _CheckPayload(self, result):
3834
    """Checks if the payload is valid.
3835

3836
    @param result: RPC result
3837
    @raises errors.OpExecError: If payload is not valid
3838

3839
    """
3840
    errs = []
3841
    if self.op.command == constants.OOB_HEALTH:
3842
      if not isinstance(result.payload, list):
3843
        errs.append("command 'health' is expected to return a list but got %s" %
3844
                    type(result.payload))
3845
      else:
3846
        for item, status in result.payload:
3847
          if status not in constants.OOB_STATUSES:
3848
            errs.append("health item '%s' has invalid status '%s'" %
3849
                        (item, status))
3850

    
3851
    if self.op.command == constants.OOB_POWER_STATUS:
3852
      if not isinstance(result.payload, dict):
3853
        errs.append("power-status is expected to return a dict but got %s" %
3854
                    type(result.payload))
3855

    
3856
    if self.op.command in [
3857
        constants.OOB_POWER_ON,
3858
        constants.OOB_POWER_OFF,
3859
        constants.OOB_POWER_CYCLE,
3860
        ]:
3861
      if result.payload is not None:
3862
        errs.append("%s is expected to not return payload but got '%s'" %
3863
                    (self.op.command, result.payload))
3864

    
3865
    if errs:
3866
      raise errors.OpExecError("Check of out-of-band payload failed due to %s" %
3867
                               utils.CommaJoin(errs))
3868

    
3869
class _OsQuery(_QueryBase):
3870
  FIELDS = query.OS_FIELDS
3871

    
3872
  def ExpandNames(self, lu):
3873
    # Lock all nodes in shared mode
3874
    # Temporary removal of locks, should be reverted later
3875
    # TODO: reintroduce locks when they are lighter-weight
3876
    lu.needed_locks = {}
3877
    #self.share_locks[locking.LEVEL_NODE] = 1
3878
    #self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET
3879

    
3880
    # The following variables interact with _QueryBase._GetNames
3881
    if self.names:
3882
      self.wanted = self.names
3883
    else:
3884
      self.wanted = locking.ALL_SET
3885

    
3886
    self.do_locking = self.use_locking
3887

    
3888
  def DeclareLocks(self, lu, level):
3889
    pass
3890

    
3891
  @staticmethod
3892
  def _DiagnoseByOS(rlist):
3893
    """Remaps a per-node return list into an a per-os per-node dictionary
3894

3895
    @param rlist: a map with node names as keys and OS objects as values
3896

3897
    @rtype: dict
3898
    @return: a dictionary with osnames as keys and as value another
3899
        map, with nodes as keys and tuples of (path, status, diagnose,
3900
        variants, parameters, api_versions) as values, eg::
3901

3902
          {"debian-etch": {"node1": [(/usr/lib/..., True, "", [], []),
3903
                                     (/srv/..., False, "invalid api")],
3904
                           "node2": [(/srv/..., True, "", [], [])]}
3905
          }
3906

3907
    """
3908
    all_os = {}
3909
    # we build here the list of nodes that didn't fail the RPC (at RPC
3910
    # level), so that nodes with a non-responding node daemon don't
3911
    # make all OSes invalid
3912
    good_nodes = [node_name for node_name in rlist
3913
                  if not rlist[node_name].fail_msg]
3914
    for node_name, nr in rlist.items():
3915
      if nr.fail_msg or not nr.payload:
3916
        continue
3917
      for (name, path, status, diagnose, variants,
3918
           params, api_versions) in nr.payload:
3919
        if name not in all_os:
3920
          # build a list of nodes for this os containing empty lists
3921
          # for each node in node_list
3922
          all_os[name] = {}
3923
          for nname in good_nodes:
3924
            all_os[name][nname] = []
3925
        # convert params from [name, help] to (name, help)
3926
        params = [tuple(v) for v in params]
3927
        all_os[name][node_name].append((path, status, diagnose,
3928
                                        variants, params, api_versions))
3929
    return all_os
3930

    
3931
  def _GetQueryData(self, lu):
3932
    """Computes the list of nodes and their attributes.
3933

3934
    """
3935
    # Locking is not used
3936
    assert not (compat.any(lu.glm.is_owned(level)
3937
                           for level in locking.LEVELS
3938
                           if level != locking.LEVEL_CLUSTER) or
3939
                self.do_locking or self.use_locking)
3940

    
3941
    valid_nodes = [node.name
3942
                   for node in lu.cfg.GetAllNodesInfo().values()
3943
                   if not node.offline and node.vm_capable]
3944
    pol = self._DiagnoseByOS(lu.rpc.call_os_diagnose(valid_nodes))
3945
    cluster = lu.cfg.GetClusterInfo()
3946

    
3947
    data = {}
3948

    
3949
    for (os_name, os_data) in pol.items():
3950
      info = query.OsInfo(name=os_name, valid=True, node_status=os_data,
3951
                          hidden=(os_name in cluster.hidden_os),
3952
                          blacklisted=(os_name in cluster.blacklisted_os))
3953

    
3954
      variants = set()
3955
      parameters = set()
3956
      api_versions = set()
3957

    
3958
      for idx, osl in enumerate(os_data.values()):
3959
        info.valid = bool(info.valid and osl and osl[0][1])
3960
        if not info.valid:
3961
          break
3962

    
3963
        (node_variants, node_params, node_api) = osl[0][3:6]
3964
        if idx == 0:
3965
          # First entry
3966
          variants.update(node_variants)
3967
          parameters.update(node_params)
3968
          api_versions.update(node_api)
3969
        else:
3970
          # Filter out inconsistent values
3971
          variants.intersection_update(node_variants)
3972
          parameters.intersection_update(node_params)
3973
          api_versions.intersection_update(node_api)
3974

    
3975
      info.variants = list(variants)
3976
      info.parameters = list(parameters)
3977
      info.api_versions = list(api_versions)
3978

    
3979
      data[os_name] = info
3980

    
3981
    # Prepare data in requested order
3982
    return [data[name] for name in self._GetNames(lu, pol.keys(), None)
3983
            if name in data]
3984

    
3985

    
3986
class LUOsDiagnose(NoHooksLU):
3987
  """Logical unit for OS diagnose/query.
3988

3989
  """
3990
  REQ_BGL = False
3991

    
3992
  @staticmethod
3993
  def _BuildFilter(fields, names):
3994
    """Builds a filter for querying OSes.
3995

3996
    """
3997
    name_filter = qlang.MakeSimpleFilter("name", names)
3998

    
3999
    # Legacy behaviour: Hide hidden, blacklisted or invalid OSes if the
4000
    # respective field is not requested
4001
    status_filter = [[qlang.OP_NOT, [qlang.OP_TRUE, fname]]
4002
                     for fname in ["hidden", "blacklisted"]
4003
                     if fname not in fields]
4004
    if "valid" not in fields:
4005
      status_filter.append([qlang.OP_TRUE, "valid"])
4006

    
4007
    if status_filter:
4008
      status_filter.insert(0, qlang.OP_AND)
4009
    else:
4010
      status_filter = None
4011

    
4012
    if name_filter and status_filter:
4013
      return [qlang.OP_AND, name_filter, status_filter]
4014
    elif name_filter:
4015
      return name_filter
4016
    else:
4017
      return status_filter
4018

    
4019
  def CheckArguments(self):
4020
    self.oq = _OsQuery(self._BuildFilter(self.op.output_fields, self.op.names),
4021
                       self.op.output_fields, False)
4022

    
4023
  def ExpandNames(self):
4024
    self.oq.ExpandNames(self)
4025

    
4026
  def Exec(self, feedback_fn):
4027
    return self.oq.OldStyleQuery(self)
4028

    
4029

    
4030
class LUNodeRemove(LogicalUnit):
4031
  """Logical unit for removing a node.
4032

4033
  """
4034
  HPATH = "node-remove"
4035
  HTYPE = constants.HTYPE_NODE
4036

    
4037
  def BuildHooksEnv(self):
4038
    """Build hooks env.
4039

4040
    This doesn't run on the target node in the pre phase as a failed
4041
    node would then be impossible to remove.
4042

4043
    """
4044
    return {
4045
      "OP_TARGET": self.op.node_name,
4046
      "NODE_NAME": self.op.node_name,
4047
      }
4048

    
4049
  def BuildHooksNodes(self):
4050
    """Build hooks nodes.
4051

4052
    """
4053
    all_nodes = self.cfg.GetNodeList()
4054
    try:
4055
      all_nodes.remove(self.op.node_name)
4056
    except ValueError:
4057
      logging.warning("Node '%s', which is about to be removed, was not found"
4058
                      " in the list of all nodes", self.op.node_name)
4059
    return (all_nodes, all_nodes)
4060

    
4061
  def CheckPrereq(self):
4062
    """Check prerequisites.
4063

4064
    This checks:
4065
     - the node exists in the configuration
4066
     - it does not have primary or secondary instances
4067
     - it's not the master
4068

4069
    Any errors are signaled by raising errors.OpPrereqError.