Statistics
| Branch: | Tag: | Revision:

root / lib / cmdlib.py @ a8282327

History | View | Annotate | Download (508.8 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Module implementing the master-side code."""
23

    
24
# pylint: disable=W0201,C0302
25

    
26
# W0201 since most LU attributes are defined in CheckPrereq or similar
27
# functions
28

    
29
# C0302: since we have waaaay too many lines in this module
30

    
31
import os
32
import os.path
33
import time
34
import re
35
import platform
36
import logging
37
import copy
38
import OpenSSL
39
import socket
40
import tempfile
41
import shutil
42
import itertools
43
import operator
44

    
45
from ganeti import ssh
46
from ganeti import utils
47
from ganeti import errors
48
from ganeti import hypervisor
49
from ganeti import locking
50
from ganeti import constants
51
from ganeti import objects
52
from ganeti import serializer
53
from ganeti import ssconf
54
from ganeti import uidpool
55
from ganeti import compat
56
from ganeti import masterd
57
from ganeti import netutils
58
from ganeti import query
59
from ganeti import qlang
60
from ganeti import opcodes
61
from ganeti import ht
62
from ganeti import rpc
63

    
64
import ganeti.masterd.instance # pylint: disable=W0611
65

    
66

    
67
#: Size of DRBD meta block device
68
DRBD_META_SIZE = 128
69

    
70
# States of instance
71
INSTANCE_UP = [constants.ADMINST_UP]
72
INSTANCE_DOWN = [constants.ADMINST_DOWN]
73
INSTANCE_OFFLINE = [constants.ADMINST_OFFLINE]
74
INSTANCE_ONLINE = [constants.ADMINST_DOWN, constants.ADMINST_UP]
75
INSTANCE_NOT_RUNNING = [constants.ADMINST_DOWN, constants.ADMINST_OFFLINE]
76

    
77

    
78
class ResultWithJobs:
79
  """Data container for LU results with jobs.
80

81
  Instances of this class returned from L{LogicalUnit.Exec} will be recognized
82
  by L{mcpu.Processor._ProcessResult}. The latter will then submit the jobs
83
  contained in the C{jobs} attribute and include the job IDs in the opcode
84
  result.
85

86
  """
87
  def __init__(self, jobs, **kwargs):
88
    """Initializes this class.
89

90
    Additional return values can be specified as keyword arguments.
91

92
    @type jobs: list of lists of L{opcode.OpCode}
93
    @param jobs: A list of lists of opcode objects
94

95
    """
96
    self.jobs = jobs
97
    self.other = kwargs
98

    
99

    
100
class LogicalUnit(object):
101
  """Logical Unit base class.
102

103
  Subclasses must follow these rules:
104
    - implement ExpandNames
105
    - implement CheckPrereq (except when tasklets are used)
106
    - implement Exec (except when tasklets are used)
107
    - implement BuildHooksEnv
108
    - implement BuildHooksNodes
109
    - redefine HPATH and HTYPE
110
    - optionally redefine their run requirements:
111
        REQ_BGL: the LU needs to hold the Big Ganeti Lock exclusively
112

113
  Note that all commands require root permissions.
114

115
  @ivar dry_run_result: the value (if any) that will be returned to the caller
116
      in dry-run mode (signalled by opcode dry_run parameter)
117

118
  """
119
  HPATH = None
120
  HTYPE = None
121
  REQ_BGL = True
122

    
123
  def __init__(self, processor, op, context, rpc_runner):
124
    """Constructor for LogicalUnit.
125

126
    This needs to be overridden in derived classes in order to check op
127
    validity.
128

129
    """
130
    self.proc = processor
131
    self.op = op
132
    self.cfg = context.cfg
133
    self.glm = context.glm
134
    # readability alias
135
    self.owned_locks = context.glm.list_owned
136
    self.context = context
137
    self.rpc = rpc_runner
138
    # Dicts used to declare locking needs to mcpu
139
    self.needed_locks = None
140
    self.share_locks = dict.fromkeys(locking.LEVELS, 0)
141
    self.add_locks = {}
142
    self.remove_locks = {}
143
    # Used to force good behavior when calling helper functions
144
    self.recalculate_locks = {}
145
    # logging
146
    self.Log = processor.Log # pylint: disable=C0103
147
    self.LogWarning = processor.LogWarning # pylint: disable=C0103
148
    self.LogInfo = processor.LogInfo # pylint: disable=C0103
149
    self.LogStep = processor.LogStep # pylint: disable=C0103
150
    # support for dry-run
151
    self.dry_run_result = None
152
    # support for generic debug attribute
153
    if (not hasattr(self.op, "debug_level") or
154
        not isinstance(self.op.debug_level, int)):
155
      self.op.debug_level = 0
156

    
157
    # Tasklets
158
    self.tasklets = None
159

    
160
    # Validate opcode parameters and set defaults
161
    self.op.Validate(True)
162

    
163
    self.CheckArguments()
164

    
165
  def CheckArguments(self):
166
    """Check syntactic validity for the opcode arguments.
167

168
    This method is for doing a simple syntactic check and ensure
169
    validity of opcode parameters, without any cluster-related
170
    checks. While the same can be accomplished in ExpandNames and/or
171
    CheckPrereq, doing these separate is better because:
172

173
      - ExpandNames is left as as purely a lock-related function
174
      - CheckPrereq is run after we have acquired locks (and possible
175
        waited for them)
176

177
    The function is allowed to change the self.op attribute so that
178
    later methods can no longer worry about missing parameters.
179

180
    """
181
    pass
182

    
183
  def ExpandNames(self):
184
    """Expand names for this LU.
185

186
    This method is called before starting to execute the opcode, and it should
187
    update all the parameters of the opcode to their canonical form (e.g. a
188
    short node name must be fully expanded after this method has successfully
189
    completed). This way locking, hooks, logging, etc. can work correctly.
190

191
    LUs which implement this method must also populate the self.needed_locks
192
    member, as a dict with lock levels as keys, and a list of needed lock names
193
    as values. Rules:
194

195
      - use an empty dict if you don't need any lock
196
      - if you don't need any lock at a particular level omit that level
197
      - don't put anything for the BGL level
198
      - if you want all locks at a level use locking.ALL_SET as a value
199

200
    If you need to share locks (rather than acquire them exclusively) at one
201
    level you can modify self.share_locks, setting a true value (usually 1) for
202
    that level. By default locks are not shared.
203

204
    This function can also define a list of tasklets, which then will be
205
    executed in order instead of the usual LU-level CheckPrereq and Exec
206
    functions, if those are not defined by the LU.
207

208
    Examples::
209

210
      # Acquire all nodes and one instance
211
      self.needed_locks = {
212
        locking.LEVEL_NODE: locking.ALL_SET,
213
        locking.LEVEL_INSTANCE: ['instance1.example.com'],
214
      }
215
      # Acquire just two nodes
216
      self.needed_locks = {
217
        locking.LEVEL_NODE: ['node1.example.com', 'node2.example.com'],
218
      }
219
      # Acquire no locks
220
      self.needed_locks = {} # No, you can't leave it to the default value None
221

222
    """
223
    # The implementation of this method is mandatory only if the new LU is
224
    # concurrent, so that old LUs don't need to be changed all at the same
225
    # time.
226
    if self.REQ_BGL:
227
      self.needed_locks = {} # Exclusive LUs don't need locks.
228
    else:
229
      raise NotImplementedError
230

    
231
  def DeclareLocks(self, level):
232
    """Declare LU locking needs for a level
233

234
    While most LUs can just declare their locking needs at ExpandNames time,
235
    sometimes there's the need to calculate some locks after having acquired
236
    the ones before. This function is called just before acquiring locks at a
237
    particular level, but after acquiring the ones at lower levels, and permits
238
    such calculations. It can be used to modify self.needed_locks, and by
239
    default it does nothing.
240

241
    This function is only called if you have something already set in
242
    self.needed_locks for the level.
243

244
    @param level: Locking level which is going to be locked
245
    @type level: member of ganeti.locking.LEVELS
246

247
    """
248

    
249
  def CheckPrereq(self):
250
    """Check prerequisites for this LU.
251

252
    This method should check that the prerequisites for the execution
253
    of this LU are fulfilled. It can do internode communication, but
254
    it should be idempotent - no cluster or system changes are
255
    allowed.
256

257
    The method should raise errors.OpPrereqError in case something is
258
    not fulfilled. Its return value is ignored.
259

260
    This method should also update all the parameters of the opcode to
261
    their canonical form if it hasn't been done by ExpandNames before.
262

263
    """
264
    if self.tasklets is not None:
265
      for (idx, tl) in enumerate(self.tasklets):
266
        logging.debug("Checking prerequisites for tasklet %s/%s",
267
                      idx + 1, len(self.tasklets))
268
        tl.CheckPrereq()
269
    else:
270
      pass
271

    
272
  def Exec(self, feedback_fn):
273
    """Execute the LU.
274

275
    This method should implement the actual work. It should raise
276
    errors.OpExecError for failures that are somewhat dealt with in
277
    code, or expected.
278

279
    """
280
    if self.tasklets is not None:
281
      for (idx, tl) in enumerate(self.tasklets):
282
        logging.debug("Executing tasklet %s/%s", idx + 1, len(self.tasklets))
283
        tl.Exec(feedback_fn)
284
    else:
285
      raise NotImplementedError
286

    
287
  def BuildHooksEnv(self):
288
    """Build hooks environment for this LU.
289

290
    @rtype: dict
291
    @return: Dictionary containing the environment that will be used for
292
      running the hooks for this LU. The keys of the dict must not be prefixed
293
      with "GANETI_"--that'll be added by the hooks runner. The hooks runner
294
      will extend the environment with additional variables. If no environment
295
      should be defined, an empty dictionary should be returned (not C{None}).
296
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
297
      will not be called.
298

299
    """
300
    raise NotImplementedError
301

    
302
  def BuildHooksNodes(self):
303
    """Build list of nodes to run LU's hooks.
304

305
    @rtype: tuple; (list, list)
306
    @return: Tuple containing a list of node names on which the hook
307
      should run before the execution and a list of node names on which the
308
      hook should run after the execution. No nodes should be returned as an
309
      empty list (and not None).
310
    @note: If the C{HPATH} attribute of the LU class is C{None}, this function
311
      will not be called.
312

313
    """
314
    raise NotImplementedError
315

    
316
  def HooksCallBack(self, phase, hook_results, feedback_fn, lu_result):
317
    """Notify the LU about the results of its hooks.
318

319
    This method is called every time a hooks phase is executed, and notifies
320
    the Logical Unit about the hooks' result. The LU can then use it to alter
321
    its result based on the hooks.  By default the method does nothing and the
322
    previous result is passed back unchanged but any LU can define it if it
323
    wants to use the local cluster hook-scripts somehow.
324

325
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
326
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
327
    @param hook_results: the results of the multi-node hooks rpc call
328
    @param feedback_fn: function used send feedback back to the caller
329
    @param lu_result: the previous Exec result this LU had, or None
330
        in the PRE phase
331
    @return: the new Exec result, based on the previous result
332
        and hook results
333

334
    """
335
    # API must be kept, thus we ignore the unused argument and could
336
    # be a function warnings
337
    # pylint: disable=W0613,R0201
338
    return lu_result
339

    
340
  def _ExpandAndLockInstance(self):
341
    """Helper function to expand and lock an instance.
342

343
    Many LUs that work on an instance take its name in self.op.instance_name
344
    and need to expand it and then declare the expanded name for locking. This
345
    function does it, and then updates self.op.instance_name to the expanded
346
    name. It also initializes needed_locks as a dict, if this hasn't been done
347
    before.
348

349
    """
350
    if self.needed_locks is None:
351
      self.needed_locks = {}
352
    else:
353
      assert locking.LEVEL_INSTANCE not in self.needed_locks, \
354
        "_ExpandAndLockInstance called with instance-level locks set"
355
    self.op.instance_name = _ExpandInstanceName(self.cfg,
356
                                                self.op.instance_name)
357
    self.needed_locks[locking.LEVEL_INSTANCE] = self.op.instance_name
358

    
359
  def _LockInstancesNodes(self, primary_only=False,
360
                          level=locking.LEVEL_NODE):
361
    """Helper function to declare instances' nodes for locking.
362

363
    This function should be called after locking one or more instances to lock
364
    their nodes. Its effect is populating self.needed_locks[locking.LEVEL_NODE]
365
    with all primary or secondary nodes for instances already locked and
366
    present in self.needed_locks[locking.LEVEL_INSTANCE].
367

368
    It should be called from DeclareLocks, and for safety only works if
369
    self.recalculate_locks[locking.LEVEL_NODE] is set.
370

371
    In the future it may grow parameters to just lock some instance's nodes, or
372
    to just lock primaries or secondary nodes, if needed.
373

374
    If should be called in DeclareLocks in a way similar to::
375

376
      if level == locking.LEVEL_NODE:
377
        self._LockInstancesNodes()
378

379
    @type primary_only: boolean
380
    @param primary_only: only lock primary nodes of locked instances
381
    @param level: Which lock level to use for locking nodes
382

383
    """
384
    assert level in self.recalculate_locks, \
385
      "_LockInstancesNodes helper function called with no nodes to recalculate"
386

    
387
    # TODO: check if we're really been called with the instance locks held
388

    
389
    # For now we'll replace self.needed_locks[locking.LEVEL_NODE], but in the
390
    # future we might want to have different behaviors depending on the value
391
    # of self.recalculate_locks[locking.LEVEL_NODE]
392
    wanted_nodes = []
393
    locked_i = self.owned_locks(locking.LEVEL_INSTANCE)
394
    for _, instance in self.cfg.GetMultiInstanceInfo(locked_i):
395
      wanted_nodes.append(instance.primary_node)
396
      if not primary_only:
397
        wanted_nodes.extend(instance.secondary_nodes)
398

    
399
    if self.recalculate_locks[level] == constants.LOCKS_REPLACE:
400
      self.needed_locks[level] = wanted_nodes
401
    elif self.recalculate_locks[level] == constants.LOCKS_APPEND:
402
      self.needed_locks[level].extend(wanted_nodes)
403
    else:
404
      raise errors.ProgrammerError("Unknown recalculation mode")
405

    
406
    del self.recalculate_locks[level]
407

    
408

    
409
class NoHooksLU(LogicalUnit): # pylint: disable=W0223
410
  """Simple LU which runs no hooks.
411

412
  This LU is intended as a parent for other LogicalUnits which will
413
  run no hooks, in order to reduce duplicate code.
414

415
  """
416
  HPATH = None
417
  HTYPE = None
418

    
419
  def BuildHooksEnv(self):
420
    """Empty BuildHooksEnv for NoHooksLu.
421

422
    This just raises an error.
423

424
    """
425
    raise AssertionError("BuildHooksEnv called for NoHooksLUs")
426

    
427
  def BuildHooksNodes(self):
428
    """Empty BuildHooksNodes for NoHooksLU.
429

430
    """
431
    raise AssertionError("BuildHooksNodes called for NoHooksLU")
432

    
433

    
434
class Tasklet:
435
  """Tasklet base class.
436

437
  Tasklets are subcomponents for LUs. LUs can consist entirely of tasklets or
438
  they can mix legacy code with tasklets. Locking needs to be done in the LU,
439
  tasklets know nothing about locks.
440

441
  Subclasses must follow these rules:
442
    - Implement CheckPrereq
443
    - Implement Exec
444

445
  """
446
  def __init__(self, lu):
447
    self.lu = lu
448

    
449
    # Shortcuts
450
    self.cfg = lu.cfg
451
    self.rpc = lu.rpc
452

    
453
  def CheckPrereq(self):
454
    """Check prerequisites for this tasklets.
455

456
    This method should check whether the prerequisites for the execution of
457
    this tasklet are fulfilled. It can do internode communication, but it
458
    should be idempotent - no cluster or system changes are allowed.
459

460
    The method should raise errors.OpPrereqError in case something is not
461
    fulfilled. Its return value is ignored.
462

463
    This method should also update all parameters to their canonical form if it
464
    hasn't been done before.
465

466
    """
467
    pass
468

    
469
  def Exec(self, feedback_fn):
470
    """Execute the tasklet.
471

472
    This method should implement the actual work. It should raise
473
    errors.OpExecError for failures that are somewhat dealt with in code, or
474
    expected.
475

476
    """
477
    raise NotImplementedError
478

    
479

    
480
class _QueryBase:
481
  """Base for query utility classes.
482

483
  """
484
  #: Attribute holding field definitions
485
  FIELDS = None
486

    
487
  def __init__(self, qfilter, fields, use_locking):
488
    """Initializes this class.
489

490
    """
491
    self.use_locking = use_locking
492

    
493
    self.query = query.Query(self.FIELDS, fields, qfilter=qfilter,
494
                             namefield="name")
495
    self.requested_data = self.query.RequestedData()
496
    self.names = self.query.RequestedNames()
497

    
498
    # Sort only if no names were requested
499
    self.sort_by_name = not self.names
500

    
501
    self.do_locking = None
502
    self.wanted = None
503

    
504
  def _GetNames(self, lu, all_names, lock_level):
505
    """Helper function to determine names asked for in the query.
506

507
    """
508
    if self.do_locking:
509
      names = lu.owned_locks(lock_level)
510
    else:
511
      names = all_names
512

    
513
    if self.wanted == locking.ALL_SET:
514
      assert not self.names
515
      # caller didn't specify names, so ordering is not important
516
      return utils.NiceSort(names)
517

    
518
    # caller specified names and we must keep the same order
519
    assert self.names
520
    assert not self.do_locking or lu.glm.is_owned(lock_level)
521

    
522
    missing = set(self.wanted).difference(names)
523
    if missing:
524
      raise errors.OpExecError("Some items were removed before retrieving"
525
                               " their data: %s" % missing)
526

    
527
    # Return expanded names
528
    return self.wanted
529

    
530
  def ExpandNames(self, lu):
531
    """Expand names for this query.
532

533
    See L{LogicalUnit.ExpandNames}.
534

535
    """
536
    raise NotImplementedError()
537

    
538
  def DeclareLocks(self, lu, level):
539
    """Declare locks for this query.
540

541
    See L{LogicalUnit.DeclareLocks}.
542

543
    """
544
    raise NotImplementedError()
545

    
546
  def _GetQueryData(self, lu):
547
    """Collects all data for this query.
548

549
    @return: Query data object
550

551
    """
552
    raise NotImplementedError()
553

    
554
  def NewStyleQuery(self, lu):
555
    """Collect data and execute query.
556

557
    """
558
    return query.GetQueryResponse(self.query, self._GetQueryData(lu),
559
                                  sort_by_name=self.sort_by_name)
560

    
561
  def OldStyleQuery(self, lu):
562
    """Collect data and execute query.
563

564
    """
565
    return self.query.OldStyleQuery(self._GetQueryData(lu),
566
                                    sort_by_name=self.sort_by_name)
567

    
568

    
569
def _ShareAll():
570
  """Returns a dict declaring all lock levels shared.
571

572
  """
573
  return dict.fromkeys(locking.LEVELS, 1)
574

    
575

    
576
def _MakeLegacyNodeInfo(data):
577
  """Formats the data returned by L{rpc.RpcRunner.call_node_info}.
578

579
  Converts the data into a single dictionary. This is fine for most use cases,
580
  but some require information from more than one volume group or hypervisor.
581

582
  """
583
  (bootid, (vg_info, ), (hv_info, )) = data
584

    
585
  return utils.JoinDisjointDicts(utils.JoinDisjointDicts(vg_info, hv_info), {
586
    "bootid": bootid,
587
    })
588

    
589

    
590
def _CheckInstanceNodeGroups(cfg, instance_name, owned_groups):
591
  """Checks if the owned node groups are still correct for an instance.
592

593
  @type cfg: L{config.ConfigWriter}
594
  @param cfg: The cluster configuration
595
  @type instance_name: string
596
  @param instance_name: Instance name
597
  @type owned_groups: set or frozenset
598
  @param owned_groups: List of currently owned node groups
599

600
  """
601
  inst_groups = cfg.GetInstanceNodeGroups(instance_name)
602

    
603
  if not owned_groups.issuperset(inst_groups):
604
    raise errors.OpPrereqError("Instance %s's node groups changed since"
605
                               " locks were acquired, current groups are"
606
                               " are '%s', owning groups '%s'; retry the"
607
                               " operation" %
608
                               (instance_name,
609
                                utils.CommaJoin(inst_groups),
610
                                utils.CommaJoin(owned_groups)),
611
                               errors.ECODE_STATE)
612

    
613
  return inst_groups
614

    
615

    
616
def _CheckNodeGroupInstances(cfg, group_uuid, owned_instances):
617
  """Checks if the instances in a node group are still correct.
618

619
  @type cfg: L{config.ConfigWriter}
620
  @param cfg: The cluster configuration
621
  @type group_uuid: string
622
  @param group_uuid: Node group UUID
623
  @type owned_instances: set or frozenset
624
  @param owned_instances: List of currently owned instances
625

626
  """
627
  wanted_instances = cfg.GetNodeGroupInstances(group_uuid)
628
  if owned_instances != wanted_instances:
629
    raise errors.OpPrereqError("Instances in node group '%s' changed since"
630
                               " locks were acquired, wanted '%s', have '%s';"
631
                               " retry the operation" %
632
                               (group_uuid,
633
                                utils.CommaJoin(wanted_instances),
634
                                utils.CommaJoin(owned_instances)),
635
                               errors.ECODE_STATE)
636

    
637
  return wanted_instances
638

    
639

    
640
def _SupportsOob(cfg, node):
641
  """Tells if node supports OOB.
642

643
  @type cfg: L{config.ConfigWriter}
644
  @param cfg: The cluster configuration
645
  @type node: L{objects.Node}
646
  @param node: The node
647
  @return: The OOB script if supported or an empty string otherwise
648

649
  """
650
  return cfg.GetNdParams(node)[constants.ND_OOB_PROGRAM]
651

    
652

    
653
def _GetWantedNodes(lu, nodes):
654
  """Returns list of checked and expanded node names.
655

656
  @type lu: L{LogicalUnit}
657
  @param lu: the logical unit on whose behalf we execute
658
  @type nodes: list
659
  @param nodes: list of node names or None for all nodes
660
  @rtype: list
661
  @return: the list of nodes, sorted
662
  @raise errors.ProgrammerError: if the nodes parameter is wrong type
663

664
  """
665
  if nodes:
666
    return [_ExpandNodeName(lu.cfg, name) for name in nodes]
667

    
668
  return utils.NiceSort(lu.cfg.GetNodeList())
669

    
670

    
671
def _GetWantedInstances(lu, instances):
672
  """Returns list of checked and expanded instance names.
673

674
  @type lu: L{LogicalUnit}
675
  @param lu: the logical unit on whose behalf we execute
676
  @type instances: list
677
  @param instances: list of instance names or None for all instances
678
  @rtype: list
679
  @return: the list of instances, sorted
680
  @raise errors.OpPrereqError: if the instances parameter is wrong type
681
  @raise errors.OpPrereqError: if any of the passed instances is not found
682

683
  """
684
  if instances:
685
    wanted = [_ExpandInstanceName(lu.cfg, name) for name in instances]
686
  else:
687
    wanted = utils.NiceSort(lu.cfg.GetInstanceList())
688
  return wanted
689

    
690

    
691
def _GetUpdatedParams(old_params, update_dict,
692
                      use_default=True, use_none=False):
693
  """Return the new version of a parameter dictionary.
694

695
  @type old_params: dict
696
  @param old_params: old parameters
697
  @type update_dict: dict
698
  @param update_dict: dict containing new parameter values, or
699
      constants.VALUE_DEFAULT to reset the parameter to its default
700
      value
701
  @param use_default: boolean
702
  @type use_default: whether to recognise L{constants.VALUE_DEFAULT}
703
      values as 'to be deleted' values
704
  @param use_none: boolean
705
  @type use_none: whether to recognise C{None} values as 'to be
706
      deleted' values
707
  @rtype: dict
708
  @return: the new parameter dictionary
709

710
  """
711
  params_copy = copy.deepcopy(old_params)
712
  for key, val in update_dict.iteritems():
713
    if ((use_default and val == constants.VALUE_DEFAULT) or
714
        (use_none and val is None)):
715
      try:
716
        del params_copy[key]
717
      except KeyError:
718
        pass
719
    else:
720
      params_copy[key] = val
721
  return params_copy
722

    
723

    
724
def _UpdateAndVerifySubDict(base, updates, type_check):
725
  """Updates and verifies a dict with sub dicts of the same type.
726

727
  @param base: The dict with the old data
728
  @param updates: The dict with the new data
729
  @param type_check: Dict suitable to ForceDictType to verify correct types
730
  @returns: A new dict with updated and verified values
731

732
  """
733
  def fn(old, value):
734
    new = _GetUpdatedParams(old, value)
735
    utils.ForceDictType(new, type_check)
736
    return new
737

    
738
  ret = copy.deepcopy(base)
739
  ret.update(dict((key, fn(base.get(key, {}), value))
740
                  for key, value in updates.items()))
741
  return ret
742

    
743

    
744
def _MergeAndVerifyHvState(op_input, obj_input):
745
  """Combines the hv state from an opcode with the one of the object
746

747
  @param op_input: The input dict from the opcode
748
  @param obj_input: The input dict from the objects
749
  @return: The verified and updated dict
750

751
  """
752
  if op_input:
753
    invalid_hvs = set(op_input) - constants.HYPER_TYPES
754
    if invalid_hvs:
755
      raise errors.OpPrereqError("Invalid hypervisor(s) in hypervisor state:"
756
                                 " %s" % utils.CommaJoin(invalid_hvs),
757
                                 errors.ECODE_INVAL)
758
    if obj_input is None:
759
      obj_input = {}
760
    type_check = constants.HVSTS_PARAMETER_TYPES
761
    return _UpdateAndVerifySubDict(obj_input, op_input, type_check)
762

    
763
  return None
764

    
765

    
766
def _MergeAndVerifyDiskState(op_input, obj_input):
767
  """Combines the disk state from an opcode with the one of the object
768

769
  @param op_input: The input dict from the opcode
770
  @param obj_input: The input dict from the objects
771
  @return: The verified and updated dict
772
  """
773
  if op_input:
774
    invalid_dst = set(op_input) - constants.DS_VALID_TYPES
775
    if invalid_dst:
776
      raise errors.OpPrereqError("Invalid storage type(s) in disk state: %s" %
777
                                 utils.CommaJoin(invalid_dst),
778
                                 errors.ECODE_INVAL)
779
    type_check = constants.DSS_PARAMETER_TYPES
780
    if obj_input is None:
781
      obj_input = {}
782
    return dict((key, _UpdateAndVerifySubDict(obj_input.get(key, {}), value,
783
                                              type_check))
784
                for key, value in op_input.items())
785

    
786
  return None
787

    
788

    
789
def _ReleaseLocks(lu, level, names=None, keep=None):
790
  """Releases locks owned by an LU.
791

792
  @type lu: L{LogicalUnit}
793
  @param level: Lock level
794
  @type names: list or None
795
  @param names: Names of locks to release
796
  @type keep: list or None
797
  @param keep: Names of locks to retain
798

799
  """
800
  assert not (keep is not None and names is not None), \
801
         "Only one of the 'names' and the 'keep' parameters can be given"
802

    
803
  if names is not None:
804
    should_release = names.__contains__
805
  elif keep:
806
    should_release = lambda name: name not in keep
807
  else:
808
    should_release = None
809

    
810
  owned = lu.owned_locks(level)
811
  if not owned:
812
    # Not owning any lock at this level, do nothing
813
    pass
814

    
815
  elif should_release:
816
    retain = []
817
    release = []
818

    
819
    # Determine which locks to release
820
    for name in owned:
821
      if should_release(name):
822
        release.append(name)
823
      else:
824
        retain.append(name)
825

    
826
    assert len(lu.owned_locks(level)) == (len(retain) + len(release))
827

    
828
    # Release just some locks
829
    lu.glm.release(level, names=release)
830

    
831
    assert frozenset(lu.owned_locks(level)) == frozenset(retain)
832
  else:
833
    # Release everything
834
    lu.glm.release(level)
835

    
836
    assert not lu.glm.is_owned(level), "No locks should be owned"
837

    
838

    
839
def _MapInstanceDisksToNodes(instances):
840
  """Creates a map from (node, volume) to instance name.
841

842
  @type instances: list of L{objects.Instance}
843
  @rtype: dict; tuple of (node name, volume name) as key, instance name as value
844

845
  """
846
  return dict(((node, vol), inst.name)
847
              for inst in instances
848
              for (node, vols) in inst.MapLVsByNode().items()
849
              for vol in vols)
850

    
851

    
852
def _RunPostHook(lu, node_name):
853
  """Runs the post-hook for an opcode on a single node.
854

855
  """
856
  hm = lu.proc.BuildHooksManager(lu)
857
  try:
858
    hm.RunPhase(constants.HOOKS_PHASE_POST, nodes=[node_name])
859
  except:
860
    # pylint: disable=W0702
861
    lu.LogWarning("Errors occurred running hooks on %s" % node_name)
862

    
863

    
864
def _CheckOutputFields(static, dynamic, selected):
865
  """Checks whether all selected fields are valid.
866

867
  @type static: L{utils.FieldSet}
868
  @param static: static fields set
869
  @type dynamic: L{utils.FieldSet}
870
  @param dynamic: dynamic fields set
871

872
  """
873
  f = utils.FieldSet()
874
  f.Extend(static)
875
  f.Extend(dynamic)
876

    
877
  delta = f.NonMatching(selected)
878
  if delta:
879
    raise errors.OpPrereqError("Unknown output fields selected: %s"
880
                               % ",".join(delta), errors.ECODE_INVAL)
881

    
882

    
883
def _CheckGlobalHvParams(params):
884
  """Validates that given hypervisor params are not global ones.
885

886
  This will ensure that instances don't get customised versions of
887
  global params.
888

889
  """
890
  used_globals = constants.HVC_GLOBALS.intersection(params)
891
  if used_globals:
892
    msg = ("The following hypervisor parameters are global and cannot"
893
           " be customized at instance level, please modify them at"
894
           " cluster level: %s" % utils.CommaJoin(used_globals))
895
    raise errors.OpPrereqError(msg, errors.ECODE_INVAL)
896

    
897

    
898
def _CheckNodeOnline(lu, node, msg=None):
899
  """Ensure that a given node is online.
900

901
  @param lu: the LU on behalf of which we make the check
902
  @param node: the node to check
903
  @param msg: if passed, should be a message to replace the default one
904
  @raise errors.OpPrereqError: if the node is offline
905

906
  """
907
  if msg is None:
908
    msg = "Can't use offline node"
909
  if lu.cfg.GetNodeInfo(node).offline:
910
    raise errors.OpPrereqError("%s: %s" % (msg, node), errors.ECODE_STATE)
911

    
912

    
913
def _CheckNodeNotDrained(lu, node):
914
  """Ensure that a given node is not drained.
915

916
  @param lu: the LU on behalf of which we make the check
917
  @param node: the node to check
918
  @raise errors.OpPrereqError: if the node is drained
919

920
  """
921
  if lu.cfg.GetNodeInfo(node).drained:
922
    raise errors.OpPrereqError("Can't use drained node %s" % node,
923
                               errors.ECODE_STATE)
924

    
925

    
926
def _CheckNodeVmCapable(lu, node):
927
  """Ensure that a given node is vm capable.
928

929
  @param lu: the LU on behalf of which we make the check
930
  @param node: the node to check
931
  @raise errors.OpPrereqError: if the node is not vm capable
932

933
  """
934
  if not lu.cfg.GetNodeInfo(node).vm_capable:
935
    raise errors.OpPrereqError("Can't use non-vm_capable node %s" % node,
936
                               errors.ECODE_STATE)
937

    
938

    
939
def _CheckNodeHasOS(lu, node, os_name, force_variant):
940
  """Ensure that a node supports a given OS.
941

942
  @param lu: the LU on behalf of which we make the check
943
  @param node: the node to check
944
  @param os_name: the OS to query about
945
  @param force_variant: whether to ignore variant errors
946
  @raise errors.OpPrereqError: if the node is not supporting the OS
947

948
  """
949
  result = lu.rpc.call_os_get(node, os_name)
950
  result.Raise("OS '%s' not in supported OS list for node %s" %
951
               (os_name, node),
952
               prereq=True, ecode=errors.ECODE_INVAL)
953
  if not force_variant:
954
    _CheckOSVariant(result.payload, os_name)
955

    
956

    
957
def _CheckNodeHasSecondaryIP(lu, node, secondary_ip, prereq):
958
  """Ensure that a node has the given secondary ip.
959

960
  @type lu: L{LogicalUnit}
961
  @param lu: the LU on behalf of which we make the check
962
  @type node: string
963
  @param node: the node to check
964
  @type secondary_ip: string
965
  @param secondary_ip: the ip to check
966
  @type prereq: boolean
967
  @param prereq: whether to throw a prerequisite or an execute error
968
  @raise errors.OpPrereqError: if the node doesn't have the ip, and prereq=True
969
  @raise errors.OpExecError: if the node doesn't have the ip, and prereq=False
970

971
  """
972
  result = lu.rpc.call_node_has_ip_address(node, secondary_ip)
973
  result.Raise("Failure checking secondary ip on node %s" % node,
974
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
975
  if not result.payload:
976
    msg = ("Node claims it doesn't have the secondary ip you gave (%s),"
977
           " please fix and re-run this command" % secondary_ip)
978
    if prereq:
979
      raise errors.OpPrereqError(msg, errors.ECODE_ENVIRON)
980
    else:
981
      raise errors.OpExecError(msg)
982

    
983

    
984
def _GetClusterDomainSecret():
985
  """Reads the cluster domain secret.
986

987
  """
988
  return utils.ReadOneLineFile(constants.CLUSTER_DOMAIN_SECRET_FILE,
989
                               strict=True)
990

    
991

    
992
def _CheckInstanceState(lu, instance, req_states, msg=None):
993
  """Ensure that an instance is in one of the required states.
994

995
  @param lu: the LU on behalf of which we make the check
996
  @param instance: the instance to check
997
  @param msg: if passed, should be a message to replace the default one
998
  @raise errors.OpPrereqError: if the instance is not in the required state
999

1000
  """
1001
  if msg is None:
1002
    msg = "can't use instance from outside %s states" % ", ".join(req_states)
1003
  if instance.admin_state not in req_states:
1004
    raise errors.OpPrereqError("Instance %s is marked to be %s, %s" %
1005
                               (instance, instance.admin_state, msg),
1006
                               errors.ECODE_STATE)
1007

    
1008
  if constants.ADMINST_UP not in req_states:
1009
    pnode = instance.primary_node
1010
    ins_l = lu.rpc.call_instance_list([pnode], [instance.hypervisor])[pnode]
1011
    ins_l.Raise("Can't contact node %s for instance information" % pnode,
1012
                prereq=True, ecode=errors.ECODE_ENVIRON)
1013

    
1014
    if instance.name in ins_l.payload:
1015
      raise errors.OpPrereqError("Instance %s is running, %s" %
1016
                                 (instance.name, msg), errors.ECODE_STATE)
1017

    
1018

    
1019
def _ExpandItemName(fn, name, kind):
1020
  """Expand an item name.
1021

1022
  @param fn: the function to use for expansion
1023
  @param name: requested item name
1024
  @param kind: text description ('Node' or 'Instance')
1025
  @return: the resolved (full) name
1026
  @raise errors.OpPrereqError: if the item is not found
1027

1028
  """
1029
  full_name = fn(name)
1030
  if full_name is None:
1031
    raise errors.OpPrereqError("%s '%s' not known" % (kind, name),
1032
                               errors.ECODE_NOENT)
1033
  return full_name
1034

    
1035

    
1036
def _ExpandNodeName(cfg, name):
1037
  """Wrapper over L{_ExpandItemName} for nodes."""
1038
  return _ExpandItemName(cfg.ExpandNodeName, name, "Node")
1039

    
1040

    
1041
def _ExpandInstanceName(cfg, name):
1042
  """Wrapper over L{_ExpandItemName} for instance."""
1043
  return _ExpandItemName(cfg.ExpandInstanceName, name, "Instance")
1044

    
1045

    
1046
def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status,
1047
                          minmem, maxmem, vcpus, nics, disk_template, disks,
1048
                          bep, hvp, hypervisor_name, tags):
1049
  """Builds instance related env variables for hooks
1050

1051
  This builds the hook environment from individual variables.
1052

1053
  @type name: string
1054
  @param name: the name of the instance
1055
  @type primary_node: string
1056
  @param primary_node: the name of the instance's primary node
1057
  @type secondary_nodes: list
1058
  @param secondary_nodes: list of secondary nodes as strings
1059
  @type os_type: string
1060
  @param os_type: the name of the instance's OS
1061
  @type status: string
1062
  @param status: the desired status of the instance
1063
  @type minmem: string
1064
  @param minmem: the minimum memory size of the instance
1065
  @type maxmem: string
1066
  @param maxmem: the maximum memory size of the instance
1067
  @type vcpus: string
1068
  @param vcpus: the count of VCPUs the instance has
1069
  @type nics: list
1070
  @param nics: list of tuples (ip, mac, mode, link) representing
1071
      the NICs the instance has
1072
  @type disk_template: string
1073
  @param disk_template: the disk template of the instance
1074
  @type disks: list
1075
  @param disks: the list of (size, mode) pairs
1076
  @type bep: dict
1077
  @param bep: the backend parameters for the instance
1078
  @type hvp: dict
1079
  @param hvp: the hypervisor parameters for the instance
1080
  @type hypervisor_name: string
1081
  @param hypervisor_name: the hypervisor for the instance
1082
  @type tags: list
1083
  @param tags: list of instance tags as strings
1084
  @rtype: dict
1085
  @return: the hook environment for this instance
1086

1087
  """
1088
  env = {
1089
    "OP_TARGET": name,
1090
    "INSTANCE_NAME": name,
1091
    "INSTANCE_PRIMARY": primary_node,
1092
    "INSTANCE_SECONDARIES": " ".join(secondary_nodes),
1093
    "INSTANCE_OS_TYPE": os_type,
1094
    "INSTANCE_STATUS": status,
1095
    "INSTANCE_MINMEM": minmem,
1096
    "INSTANCE_MAXMEM": maxmem,
1097
    # TODO(2.7) remove deprecated "memory" value
1098
    "INSTANCE_MEMORY": maxmem,
1099
    "INSTANCE_VCPUS": vcpus,
1100
    "INSTANCE_DISK_TEMPLATE": disk_template,
1101
    "INSTANCE_HYPERVISOR": hypervisor_name,
1102
  }
1103
  if nics:
1104
    nic_count = len(nics)
1105
    for idx, (ip, mac, mode, link) in enumerate(nics):
1106
      if ip is None:
1107
        ip = ""
1108
      env["INSTANCE_NIC%d_IP" % idx] = ip
1109
      env["INSTANCE_NIC%d_MAC" % idx] = mac
1110
      env["INSTANCE_NIC%d_MODE" % idx] = mode
1111
      env["INSTANCE_NIC%d_LINK" % idx] = link
1112
      if mode == constants.NIC_MODE_BRIDGED:
1113
        env["INSTANCE_NIC%d_BRIDGE" % idx] = link
1114
  else:
1115
    nic_count = 0
1116

    
1117
  env["INSTANCE_NIC_COUNT"] = nic_count
1118

    
1119
  if disks:
1120
    disk_count = len(disks)
1121
    for idx, (size, mode) in enumerate(disks):
1122
      env["INSTANCE_DISK%d_SIZE" % idx] = size
1123
      env["INSTANCE_DISK%d_MODE" % idx] = mode
1124
  else:
1125
    disk_count = 0
1126

    
1127
  env["INSTANCE_DISK_COUNT"] = disk_count
1128

    
1129
  if not tags:
1130
    tags = []
1131

    
1132
  env["INSTANCE_TAGS"] = " ".join(tags)
1133

    
1134
  for source, kind in [(bep, "BE"), (hvp, "HV")]:
1135
    for key, value in source.items():
1136
      env["INSTANCE_%s_%s" % (kind, key)] = value
1137

    
1138
  return env
1139

    
1140

    
1141
def _NICListToTuple(lu, nics):
1142
  """Build a list of nic information tuples.
1143

1144
  This list is suitable to be passed to _BuildInstanceHookEnv or as a return
1145
  value in LUInstanceQueryData.
1146

1147
  @type lu:  L{LogicalUnit}
1148
  @param lu: the logical unit on whose behalf we execute
1149
  @type nics: list of L{objects.NIC}
1150
  @param nics: list of nics to convert to hooks tuples
1151

1152
  """
1153
  hooks_nics = []
1154
  cluster = lu.cfg.GetClusterInfo()
1155
  for nic in nics:
1156
    ip = nic.ip
1157
    mac = nic.mac
1158
    filled_params = cluster.SimpleFillNIC(nic.nicparams)
1159
    mode = filled_params[constants.NIC_MODE]
1160
    link = filled_params[constants.NIC_LINK]
1161
    hooks_nics.append((ip, mac, mode, link))
1162
  return hooks_nics
1163

    
1164

    
1165
def _BuildInstanceHookEnvByObject(lu, instance, override=None):
1166
  """Builds instance related env variables for hooks from an object.
1167

1168
  @type lu: L{LogicalUnit}
1169
  @param lu: the logical unit on whose behalf we execute
1170
  @type instance: L{objects.Instance}
1171
  @param instance: the instance for which we should build the
1172
      environment
1173
  @type override: dict
1174
  @param override: dictionary with key/values that will override
1175
      our values
1176
  @rtype: dict
1177
  @return: the hook environment dictionary
1178

1179
  """
1180
  cluster = lu.cfg.GetClusterInfo()
1181
  bep = cluster.FillBE(instance)
1182
  hvp = cluster.FillHV(instance)
1183
  args = {
1184
    "name": instance.name,
1185
    "primary_node": instance.primary_node,
1186
    "secondary_nodes": instance.secondary_nodes,
1187
    "os_type": instance.os,
1188
    "status": instance.admin_state,
1189
    "maxmem": bep[constants.BE_MAXMEM],
1190
    "minmem": bep[constants.BE_MINMEM],
1191
    "vcpus": bep[constants.BE_VCPUS],
1192
    "nics": _NICListToTuple(lu, instance.nics),
1193
    "disk_template": instance.disk_template,
1194
    "disks": [(disk.size, disk.mode) for disk in instance.disks],
1195
    "bep": bep,
1196
    "hvp": hvp,
1197
    "hypervisor_name": instance.hypervisor,
1198
    "tags": instance.tags,
1199
  }
1200
  if override:
1201
    args.update(override)
1202
  return _BuildInstanceHookEnv(**args) # pylint: disable=W0142
1203

    
1204

    
1205
def _AdjustCandidatePool(lu, exceptions):
1206
  """Adjust the candidate pool after node operations.
1207

1208
  """
1209
  mod_list = lu.cfg.MaintainCandidatePool(exceptions)
1210
  if mod_list:
1211
    lu.LogInfo("Promoted nodes to master candidate role: %s",
1212
               utils.CommaJoin(node.name for node in mod_list))
1213
    for name in mod_list:
1214
      lu.context.ReaddNode(name)
1215
  mc_now, mc_max, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1216
  if mc_now > mc_max:
1217
    lu.LogInfo("Note: more nodes are candidates (%d) than desired (%d)" %
1218
               (mc_now, mc_max))
1219

    
1220

    
1221
def _DecideSelfPromotion(lu, exceptions=None):
1222
  """Decide whether I should promote myself as a master candidate.
1223

1224
  """
1225
  cp_size = lu.cfg.GetClusterInfo().candidate_pool_size
1226
  mc_now, mc_should, _ = lu.cfg.GetMasterCandidateStats(exceptions)
1227
  # the new node will increase mc_max with one, so:
1228
  mc_should = min(mc_should + 1, cp_size)
1229
  return mc_now < mc_should
1230

    
1231

    
1232
def _CheckNicsBridgesExist(lu, target_nics, target_node):
1233
  """Check that the brigdes needed by a list of nics exist.
1234

1235
  """
1236
  cluster = lu.cfg.GetClusterInfo()
1237
  paramslist = [cluster.SimpleFillNIC(nic.nicparams) for nic in target_nics]
1238
  brlist = [params[constants.NIC_LINK] for params in paramslist
1239
            if params[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED]
1240
  if brlist:
1241
    result = lu.rpc.call_bridges_exist(target_node, brlist)
1242
    result.Raise("Error checking bridges on destination node '%s'" %
1243
                 target_node, prereq=True, ecode=errors.ECODE_ENVIRON)
1244

    
1245

    
1246
def _CheckInstanceBridgesExist(lu, instance, node=None):
1247
  """Check that the brigdes needed by an instance exist.
1248

1249
  """
1250
  if node is None:
1251
    node = instance.primary_node
1252
  _CheckNicsBridgesExist(lu, instance.nics, node)
1253

    
1254

    
1255
def _CheckOSVariant(os_obj, name):
1256
  """Check whether an OS name conforms to the os variants specification.
1257

1258
  @type os_obj: L{objects.OS}
1259
  @param os_obj: OS object to check
1260
  @type name: string
1261
  @param name: OS name passed by the user, to check for validity
1262

1263
  """
1264
  variant = objects.OS.GetVariant(name)
1265
  if not os_obj.supported_variants:
1266
    if variant:
1267
      raise errors.OpPrereqError("OS '%s' doesn't support variants ('%s'"
1268
                                 " passed)" % (os_obj.name, variant),
1269
                                 errors.ECODE_INVAL)
1270
    return
1271
  if not variant:
1272
    raise errors.OpPrereqError("OS name must include a variant",
1273
                               errors.ECODE_INVAL)
1274

    
1275
  if variant not in os_obj.supported_variants:
1276
    raise errors.OpPrereqError("Unsupported OS variant", errors.ECODE_INVAL)
1277

    
1278

    
1279
def _GetNodeInstancesInner(cfg, fn):
1280
  return [i for i in cfg.GetAllInstancesInfo().values() if fn(i)]
1281

    
1282

    
1283
def _GetNodeInstances(cfg, node_name):
1284
  """Returns a list of all primary and secondary instances on a node.
1285

1286
  """
1287

    
1288
  return _GetNodeInstancesInner(cfg, lambda inst: node_name in inst.all_nodes)
1289

    
1290

    
1291
def _GetNodePrimaryInstances(cfg, node_name):
1292
  """Returns primary instances on a node.
1293

1294
  """
1295
  return _GetNodeInstancesInner(cfg,
1296
                                lambda inst: node_name == inst.primary_node)
1297

    
1298

    
1299
def _GetNodeSecondaryInstances(cfg, node_name):
1300
  """Returns secondary instances on a node.
1301

1302
  """
1303
  return _GetNodeInstancesInner(cfg,
1304
                                lambda inst: node_name in inst.secondary_nodes)
1305

    
1306

    
1307
def _GetStorageTypeArgs(cfg, storage_type):
1308
  """Returns the arguments for a storage type.
1309

1310
  """
1311
  # Special case for file storage
1312
  if storage_type == constants.ST_FILE:
1313
    # storage.FileStorage wants a list of storage directories
1314
    return [[cfg.GetFileStorageDir(), cfg.GetSharedFileStorageDir()]]
1315

    
1316
  return []
1317

    
1318

    
1319
def _FindFaultyInstanceDisks(cfg, rpc_runner, instance, node_name, prereq):
1320
  faulty = []
1321

    
1322
  for dev in instance.disks:
1323
    cfg.SetDiskID(dev, node_name)
1324

    
1325
  result = rpc_runner.call_blockdev_getmirrorstatus(node_name, instance.disks)
1326
  result.Raise("Failed to get disk status from node %s" % node_name,
1327
               prereq=prereq, ecode=errors.ECODE_ENVIRON)
1328

    
1329
  for idx, bdev_status in enumerate(result.payload):
1330
    if bdev_status and bdev_status.ldisk_status == constants.LDS_FAULTY:
1331
      faulty.append(idx)
1332

    
1333
  return faulty
1334

    
1335

    
1336
def _CheckIAllocatorOrNode(lu, iallocator_slot, node_slot):
1337
  """Check the sanity of iallocator and node arguments and use the
1338
  cluster-wide iallocator if appropriate.
1339

1340
  Check that at most one of (iallocator, node) is specified. If none is
1341
  specified, then the LU's opcode's iallocator slot is filled with the
1342
  cluster-wide default iallocator.
1343

1344
  @type iallocator_slot: string
1345
  @param iallocator_slot: the name of the opcode iallocator slot
1346
  @type node_slot: string
1347
  @param node_slot: the name of the opcode target node slot
1348

1349
  """
1350
  node = getattr(lu.op, node_slot, None)
1351
  iallocator = getattr(lu.op, iallocator_slot, None)
1352

    
1353
  if node is not None and iallocator is not None:
1354
    raise errors.OpPrereqError("Do not specify both, iallocator and node",
1355
                               errors.ECODE_INVAL)
1356
  elif node is None and iallocator is None:
1357
    default_iallocator = lu.cfg.GetDefaultIAllocator()
1358
    if default_iallocator:
1359
      setattr(lu.op, iallocator_slot, default_iallocator)
1360
    else:
1361
      raise errors.OpPrereqError("No iallocator or node given and no"
1362
                                 " cluster-wide default iallocator found;"
1363
                                 " please specify either an iallocator or a"
1364
                                 " node, or set a cluster-wide default"
1365
                                 " iallocator")
1366

    
1367

    
1368
def _GetDefaultIAllocator(cfg, iallocator):
1369
  """Decides on which iallocator to use.
1370

1371
  @type cfg: L{config.ConfigWriter}
1372
  @param cfg: Cluster configuration object
1373
  @type iallocator: string or None
1374
  @param iallocator: Iallocator specified in opcode
1375
  @rtype: string
1376
  @return: Iallocator name
1377

1378
  """
1379
  if not iallocator:
1380
    # Use default iallocator
1381
    iallocator = cfg.GetDefaultIAllocator()
1382

    
1383
  if not iallocator:
1384
    raise errors.OpPrereqError("No iallocator was specified, neither in the"
1385
                               " opcode nor as a cluster-wide default",
1386
                               errors.ECODE_INVAL)
1387

    
1388
  return iallocator
1389

    
1390

    
1391
class LUClusterPostInit(LogicalUnit):
1392
  """Logical unit for running hooks after cluster initialization.
1393

1394
  """
1395
  HPATH = "cluster-init"
1396
  HTYPE = constants.HTYPE_CLUSTER
1397

    
1398
  def BuildHooksEnv(self):
1399
    """Build hooks env.
1400

1401
    """
1402
    return {
1403
      "OP_TARGET": self.cfg.GetClusterName(),
1404
      }
1405

    
1406
  def BuildHooksNodes(self):
1407
    """Build hooks nodes.
1408

1409
    """
1410
    return ([], [self.cfg.GetMasterNode()])
1411

    
1412
  def Exec(self, feedback_fn):
1413
    """Nothing to do.
1414

1415
    """
1416
    return True
1417

    
1418

    
1419
class LUClusterDestroy(LogicalUnit):
1420
  """Logical unit for destroying the cluster.
1421

1422
  """
1423
  HPATH = "cluster-destroy"
1424
  HTYPE = constants.HTYPE_CLUSTER
1425

    
1426
  def BuildHooksEnv(self):
1427
    """Build hooks env.
1428

1429
    """
1430
    return {
1431
      "OP_TARGET": self.cfg.GetClusterName(),
1432
      }
1433

    
1434
  def BuildHooksNodes(self):
1435
    """Build hooks nodes.
1436

1437
    """
1438
    return ([], [])
1439

    
1440
  def CheckPrereq(self):
1441
    """Check prerequisites.
1442

1443
    This checks whether the cluster is empty.
1444

1445
    Any errors are signaled by raising errors.OpPrereqError.
1446

1447
    """
1448
    master = self.cfg.GetMasterNode()
1449

    
1450
    nodelist = self.cfg.GetNodeList()
1451
    if len(nodelist) != 1 or nodelist[0] != master:
1452
      raise errors.OpPrereqError("There are still %d node(s) in"
1453
                                 " this cluster." % (len(nodelist) - 1),
1454
                                 errors.ECODE_INVAL)
1455
    instancelist = self.cfg.GetInstanceList()
1456
    if instancelist:
1457
      raise errors.OpPrereqError("There are still %d instance(s) in"
1458
                                 " this cluster." % len(instancelist),
1459
                                 errors.ECODE_INVAL)
1460

    
1461
  def Exec(self, feedback_fn):
1462
    """Destroys the cluster.
1463

1464
    """
1465
    master_params = self.cfg.GetMasterNetworkParameters()
1466

    
1467
    # Run post hooks on master node before it's removed
1468
    _RunPostHook(self, master_params.name)
1469

    
1470
    ems = self.cfg.GetUseExternalMipScript()
1471
    result = self.rpc.call_node_deactivate_master_ip(master_params.name,
1472
                                                     master_params, ems)
1473
    result.Raise("Could not disable the master role")
1474

    
1475
    return master_params.name
1476

    
1477

    
1478
def _VerifyCertificate(filename):
1479
  """Verifies a certificate for L{LUClusterVerifyConfig}.
1480

1481
  @type filename: string
1482
  @param filename: Path to PEM file
1483

1484
  """
1485
  try:
1486
    cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
1487
                                           utils.ReadFile(filename))
1488
  except Exception, err: # pylint: disable=W0703
1489
    return (LUClusterVerifyConfig.ETYPE_ERROR,
1490
            "Failed to load X509 certificate %s: %s" % (filename, err))
1491

    
1492
  (errcode, msg) = \
1493
    utils.VerifyX509Certificate(cert, constants.SSL_CERT_EXPIRATION_WARN,
1494
                                constants.SSL_CERT_EXPIRATION_ERROR)
1495

    
1496
  if msg:
1497
    fnamemsg = "While verifying %s: %s" % (filename, msg)
1498
  else:
1499
    fnamemsg = None
1500

    
1501
  if errcode is None:
1502
    return (None, fnamemsg)
1503
  elif errcode == utils.CERT_WARNING:
1504
    return (LUClusterVerifyConfig.ETYPE_WARNING, fnamemsg)
1505
  elif errcode == utils.CERT_ERROR:
1506
    return (LUClusterVerifyConfig.ETYPE_ERROR, fnamemsg)
1507

    
1508
  raise errors.ProgrammerError("Unhandled certificate error code %r" % errcode)
1509

    
1510

    
1511
def _GetAllHypervisorParameters(cluster, instances):
1512
  """Compute the set of all hypervisor parameters.
1513

1514
  @type cluster: L{objects.Cluster}
1515
  @param cluster: the cluster object
1516
  @param instances: list of L{objects.Instance}
1517
  @param instances: additional instances from which to obtain parameters
1518
  @rtype: list of (origin, hypervisor, parameters)
1519
  @return: a list with all parameters found, indicating the hypervisor they
1520
       apply to, and the origin (can be "cluster", "os X", or "instance Y")
1521

1522
  """
1523
  hvp_data = []
1524

    
1525
  for hv_name in cluster.enabled_hypervisors:
1526
    hvp_data.append(("cluster", hv_name, cluster.GetHVDefaults(hv_name)))
1527

    
1528
  for os_name, os_hvp in cluster.os_hvp.items():
1529
    for hv_name, hv_params in os_hvp.items():
1530
      if hv_params:
1531
        full_params = cluster.GetHVDefaults(hv_name, os_name=os_name)
1532
        hvp_data.append(("os %s" % os_name, hv_name, full_params))
1533

    
1534
  # TODO: collapse identical parameter values in a single one
1535
  for instance in instances:
1536
    if instance.hvparams:
1537
      hvp_data.append(("instance %s" % instance.name, instance.hypervisor,
1538
                       cluster.FillHV(instance)))
1539

    
1540
  return hvp_data
1541

    
1542

    
1543
class _VerifyErrors(object):
1544
  """Mix-in for cluster/group verify LUs.
1545

1546
  It provides _Error and _ErrorIf, and updates the self.bad boolean. (Expects
1547
  self.op and self._feedback_fn to be available.)
1548

1549
  """
1550

    
1551
  ETYPE_FIELD = "code"
1552
  ETYPE_ERROR = "ERROR"
1553
  ETYPE_WARNING = "WARNING"
1554

    
1555
  def _Error(self, ecode, item, msg, *args, **kwargs):
1556
    """Format an error message.
1557

1558
    Based on the opcode's error_codes parameter, either format a
1559
    parseable error code, or a simpler error string.
1560

1561
    This must be called only from Exec and functions called from Exec.
1562

1563
    """
1564
    ltype = kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR)
1565
    itype, etxt, _ = ecode
1566
    # first complete the msg
1567
    if args:
1568
      msg = msg % args
1569
    # then format the whole message
1570
    if self.op.error_codes: # This is a mix-in. pylint: disable=E1101
1571
      msg = "%s:%s:%s:%s:%s" % (ltype, etxt, itype, item, msg)
1572
    else:
1573
      if item:
1574
        item = " " + item
1575
      else:
1576
        item = ""
1577
      msg = "%s: %s%s: %s" % (ltype, itype, item, msg)
1578
    # and finally report it via the feedback_fn
1579
    self._feedback_fn("  - %s" % msg) # Mix-in. pylint: disable=E1101
1580

    
1581
  def _ErrorIf(self, cond, ecode, *args, **kwargs):
1582
    """Log an error message if the passed condition is True.
1583

1584
    """
1585
    cond = (bool(cond)
1586
            or self.op.debug_simulate_errors) # pylint: disable=E1101
1587

    
1588
    # If the error code is in the list of ignored errors, demote the error to a
1589
    # warning
1590
    (_, etxt, _) = ecode
1591
    if etxt in self.op.ignore_errors:     # pylint: disable=E1101
1592
      kwargs[self.ETYPE_FIELD] = self.ETYPE_WARNING
1593

    
1594
    if cond:
1595
      self._Error(ecode, *args, **kwargs)
1596

    
1597
    # do not mark the operation as failed for WARN cases only
1598
    if kwargs.get(self.ETYPE_FIELD, self.ETYPE_ERROR) == self.ETYPE_ERROR:
1599
      self.bad = self.bad or cond
1600

    
1601

    
1602
class LUClusterVerify(NoHooksLU):
1603
  """Submits all jobs necessary to verify the cluster.
1604

1605
  """
1606
  REQ_BGL = False
1607

    
1608
  def ExpandNames(self):
1609
    self.needed_locks = {}
1610

    
1611
  def Exec(self, feedback_fn):
1612
    jobs = []
1613

    
1614
    if self.op.group_name:
1615
      groups = [self.op.group_name]
1616
      depends_fn = lambda: None
1617
    else:
1618
      groups = self.cfg.GetNodeGroupList()
1619

    
1620
      # Verify global configuration
1621
      jobs.append([
1622
        opcodes.OpClusterVerifyConfig(ignore_errors=self.op.ignore_errors)
1623
        ])
1624

    
1625
      # Always depend on global verification
1626
      depends_fn = lambda: [(-len(jobs), [])]
1627

    
1628
    jobs.extend([opcodes.OpClusterVerifyGroup(group_name=group,
1629
                                            ignore_errors=self.op.ignore_errors,
1630
                                            depends=depends_fn())]
1631
                for group in groups)
1632

    
1633
    # Fix up all parameters
1634
    for op in itertools.chain(*jobs): # pylint: disable=W0142
1635
      op.debug_simulate_errors = self.op.debug_simulate_errors
1636
      op.verbose = self.op.verbose
1637
      op.error_codes = self.op.error_codes
1638
      try:
1639
        op.skip_checks = self.op.skip_checks
1640
      except AttributeError:
1641
        assert not isinstance(op, opcodes.OpClusterVerifyGroup)
1642

    
1643
    return ResultWithJobs(jobs)
1644

    
1645

    
1646
class LUClusterVerifyConfig(NoHooksLU, _VerifyErrors):
1647
  """Verifies the cluster config.
1648

1649
  """
1650
  REQ_BGL = True
1651

    
1652
  def _VerifyHVP(self, hvp_data):
1653
    """Verifies locally the syntax of the hypervisor parameters.
1654

1655
    """
1656
    for item, hv_name, hv_params in hvp_data:
1657
      msg = ("hypervisor %s parameters syntax check (source %s): %%s" %
1658
             (item, hv_name))
1659
      try:
1660
        hv_class = hypervisor.GetHypervisor(hv_name)
1661
        utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
1662
        hv_class.CheckParameterSyntax(hv_params)
1663
      except errors.GenericError, err:
1664
        self._ErrorIf(True, constants.CV_ECLUSTERCFG, None, msg % str(err))
1665

    
1666
  def ExpandNames(self):
1667
    # Information can be safely retrieved as the BGL is acquired in exclusive
1668
    # mode
1669
    assert locking.BGL in self.owned_locks(locking.LEVEL_CLUSTER)
1670
    self.all_group_info = self.cfg.GetAllNodeGroupsInfo()
1671
    self.all_node_info = self.cfg.GetAllNodesInfo()
1672
    self.all_inst_info = self.cfg.GetAllInstancesInfo()
1673
    self.needed_locks = {}
1674

    
1675
  def Exec(self, feedback_fn):
1676
    """Verify integrity of cluster, performing various test on nodes.
1677

1678
    """
1679
    self.bad = False
1680
    self._feedback_fn = feedback_fn
1681

    
1682
    feedback_fn("* Verifying cluster config")
1683

    
1684
    for msg in self.cfg.VerifyConfig():
1685
      self._ErrorIf(True, constants.CV_ECLUSTERCFG, None, msg)
1686

    
1687
    feedback_fn("* Verifying cluster certificate files")
1688

    
1689
    for cert_filename in constants.ALL_CERT_FILES:
1690
      (errcode, msg) = _VerifyCertificate(cert_filename)
1691
      self._ErrorIf(errcode, constants.CV_ECLUSTERCERT, None, msg, code=errcode)
1692

    
1693
    feedback_fn("* Verifying hypervisor parameters")
1694

    
1695
    self._VerifyHVP(_GetAllHypervisorParameters(self.cfg.GetClusterInfo(),
1696
                                                self.all_inst_info.values()))
1697

    
1698
    feedback_fn("* Verifying all nodes belong to an existing group")
1699

    
1700
    # We do this verification here because, should this bogus circumstance
1701
    # occur, it would never be caught by VerifyGroup, which only acts on
1702
    # nodes/instances reachable from existing node groups.
1703

    
1704
    dangling_nodes = set(node.name for node in self.all_node_info.values()
1705
                         if node.group not in self.all_group_info)
1706

    
1707
    dangling_instances = {}
1708
    no_node_instances = []
1709

    
1710
    for inst in self.all_inst_info.values():
1711
      if inst.primary_node in dangling_nodes:
1712
        dangling_instances.setdefault(inst.primary_node, []).append(inst.name)
1713
      elif inst.primary_node not in self.all_node_info:
1714
        no_node_instances.append(inst.name)
1715

    
1716
    pretty_dangling = [
1717
        "%s (%s)" %
1718
        (node.name,
1719
         utils.CommaJoin(dangling_instances.get(node.name,
1720
                                                ["no instances"])))
1721
        for node in dangling_nodes]
1722

    
1723
    self._ErrorIf(bool(dangling_nodes), constants.CV_ECLUSTERDANGLINGNODES,
1724
                  None,
1725
                  "the following nodes (and their instances) belong to a non"
1726
                  " existing group: %s", utils.CommaJoin(pretty_dangling))
1727

    
1728
    self._ErrorIf(bool(no_node_instances), constants.CV_ECLUSTERDANGLINGINST,
1729
                  None,
1730
                  "the following instances have a non-existing primary-node:"
1731
                  " %s", utils.CommaJoin(no_node_instances))
1732

    
1733
    return not self.bad
1734

    
1735

    
1736
class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors):
1737
  """Verifies the status of a node group.
1738

1739
  """
1740
  HPATH = "cluster-verify"
1741
  HTYPE = constants.HTYPE_CLUSTER
1742
  REQ_BGL = False
1743

    
1744
  _HOOKS_INDENT_RE = re.compile("^", re.M)
1745

    
1746
  class NodeImage(object):
1747
    """A class representing the logical and physical status of a node.
1748

1749
    @type name: string
1750
    @ivar name: the node name to which this object refers
1751
    @ivar volumes: a structure as returned from
1752
        L{ganeti.backend.GetVolumeList} (runtime)
1753
    @ivar instances: a list of running instances (runtime)
1754
    @ivar pinst: list of configured primary instances (config)
1755
    @ivar sinst: list of configured secondary instances (config)
1756
    @ivar sbp: dictionary of {primary-node: list of instances} for all
1757
        instances for which this node is secondary (config)
1758
    @ivar mfree: free memory, as reported by hypervisor (runtime)
1759
    @ivar dfree: free disk, as reported by the node (runtime)
1760
    @ivar offline: the offline status (config)
1761
    @type rpc_fail: boolean
1762
    @ivar rpc_fail: whether the RPC verify call was successfull (overall,
1763
        not whether the individual keys were correct) (runtime)
1764
    @type lvm_fail: boolean
1765
    @ivar lvm_fail: whether the RPC call didn't return valid LVM data
1766
    @type hyp_fail: boolean
1767
    @ivar hyp_fail: whether the RPC call didn't return the instance list
1768
    @type ghost: boolean
1769
    @ivar ghost: whether this is a known node or not (config)
1770
    @type os_fail: boolean
1771
    @ivar os_fail: whether the RPC call didn't return valid OS data
1772
    @type oslist: list
1773
    @ivar oslist: list of OSes as diagnosed by DiagnoseOS
1774
    @type vm_capable: boolean
1775
    @ivar vm_capable: whether the node can host instances
1776

1777
    """
1778
    def __init__(self, offline=False, name=None, vm_capable=True):
1779
      self.name = name
1780
      self.volumes = {}
1781
      self.instances = []
1782
      self.pinst = []
1783
      self.sinst = []
1784
      self.sbp = {}
1785
      self.mfree = 0
1786
      self.dfree = 0
1787
      self.offline = offline
1788
      self.vm_capable = vm_capable
1789
      self.rpc_fail = False
1790
      self.lvm_fail = False
1791
      self.hyp_fail = False
1792
      self.ghost = False
1793
      self.os_fail = False
1794
      self.oslist = {}
1795

    
1796
  def ExpandNames(self):
1797
    # This raises errors.OpPrereqError on its own:
1798
    self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name)
1799

    
1800
    # Get instances in node group; this is unsafe and needs verification later
1801
    inst_names = self.cfg.GetNodeGroupInstances(self.group_uuid)
1802

    
1803
    self.needed_locks = {
1804
      locking.LEVEL_INSTANCE: inst_names,
1805
      locking.LEVEL_NODEGROUP: [self.group_uuid],
1806
      locking.LEVEL_NODE: [],
1807
      }
1808

    
1809
    self.share_locks = _ShareAll()
1810

    
1811
  def DeclareLocks(self, level):
1812
    if level == locking.LEVEL_NODE:
1813
      # Get members of node group; this is unsafe and needs verification later
1814
      nodes = set(self.cfg.GetNodeGroup(self.group_uuid).members)
1815

    
1816
      all_inst_info = self.cfg.GetAllInstancesInfo()
1817

    
1818
      # In Exec(), we warn about mirrored instances that have primary and
1819
      # secondary living in separate node groups. To fully verify that
1820
      # volumes for these instances are healthy, we will need to do an
1821
      # extra call to their secondaries. We ensure here those nodes will
1822
      # be locked.
1823
      for inst in self.owned_locks(locking.LEVEL_INSTANCE):
1824
        # Important: access only the instances whose lock is owned
1825
        if all_inst_info[inst].disk_template in constants.DTS_INT_MIRROR:
1826
          nodes.update(all_inst_info[inst].secondary_nodes)
1827

    
1828
      self.needed_locks[locking.LEVEL_NODE] = nodes
1829

    
1830
  def CheckPrereq(self):
1831
    assert self.group_uuid in self.owned_locks(locking.LEVEL_NODEGROUP)
1832
    self.group_info = self.cfg.GetNodeGroup(self.group_uuid)
1833

    
1834
    group_nodes = set(self.group_info.members)
1835
    group_instances = self.cfg.GetNodeGroupInstances(self.group_uuid)
1836

    
1837
    unlocked_nodes = \
1838
        group_nodes.difference(self.owned_locks(locking.LEVEL_NODE))
1839

    
1840
    unlocked_instances = \
1841
        group_instances.difference(self.owned_locks(locking.LEVEL_INSTANCE))
1842

    
1843
    if unlocked_nodes:
1844
      raise errors.OpPrereqError("Missing lock for nodes: %s" %
1845
                                 utils.CommaJoin(unlocked_nodes))
1846

    
1847
    if unlocked_instances:
1848
      raise errors.OpPrereqError("Missing lock for instances: %s" %
1849
                                 utils.CommaJoin(unlocked_instances))
1850

    
1851
    self.all_node_info = self.cfg.GetAllNodesInfo()
1852
    self.all_inst_info = self.cfg.GetAllInstancesInfo()
1853

    
1854
    self.my_node_names = utils.NiceSort(group_nodes)
1855
    self.my_inst_names = utils.NiceSort(group_instances)
1856

    
1857
    self.my_node_info = dict((name, self.all_node_info[name])
1858
                             for name in self.my_node_names)
1859

    
1860
    self.my_inst_info = dict((name, self.all_inst_info[name])
1861
                             for name in self.my_inst_names)
1862

    
1863
    # We detect here the nodes that will need the extra RPC calls for verifying
1864
    # split LV volumes; they should be locked.
1865
    extra_lv_nodes = set()
1866

    
1867
    for inst in self.my_inst_info.values():
1868
      if inst.disk_template in constants.DTS_INT_MIRROR:
1869
        group = self.my_node_info[inst.primary_node].group
1870
        for nname in inst.secondary_nodes:
1871
          if self.all_node_info[nname].group != group:
1872
            extra_lv_nodes.add(nname)
1873

    
1874
    unlocked_lv_nodes = \
1875
        extra_lv_nodes.difference(self.owned_locks(locking.LEVEL_NODE))
1876

    
1877
    if unlocked_lv_nodes:
1878
      raise errors.OpPrereqError("these nodes could be locked: %s" %
1879
                                 utils.CommaJoin(unlocked_lv_nodes))
1880
    self.extra_lv_nodes = list(extra_lv_nodes)
1881

    
1882
  def _VerifyNode(self, ninfo, nresult):
1883
    """Perform some basic validation on data returned from a node.
1884

1885
      - check the result data structure is well formed and has all the
1886
        mandatory fields
1887
      - check ganeti version
1888

1889
    @type ninfo: L{objects.Node}
1890
    @param ninfo: the node to check
1891
    @param nresult: the results from the node
1892
    @rtype: boolean
1893
    @return: whether overall this call was successful (and we can expect
1894
         reasonable values in the respose)
1895

1896
    """
1897
    node = ninfo.name
1898
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
1899

    
1900
    # main result, nresult should be a non-empty dict
1901
    test = not nresult or not isinstance(nresult, dict)
1902
    _ErrorIf(test, constants.CV_ENODERPC, node,
1903
                  "unable to verify node: no data returned")
1904
    if test:
1905
      return False
1906

    
1907
    # compares ganeti version
1908
    local_version = constants.PROTOCOL_VERSION
1909
    remote_version = nresult.get("version", None)
1910
    test = not (remote_version and
1911
                isinstance(remote_version, (list, tuple)) and
1912
                len(remote_version) == 2)
1913
    _ErrorIf(test, constants.CV_ENODERPC, node,
1914
             "connection to node returned invalid data")
1915
    if test:
1916
      return False
1917

    
1918
    test = local_version != remote_version[0]
1919
    _ErrorIf(test, constants.CV_ENODEVERSION, node,
1920
             "incompatible protocol versions: master %s,"
1921
             " node %s", local_version, remote_version[0])
1922
    if test:
1923
      return False
1924

    
1925
    # node seems compatible, we can actually try to look into its results
1926

    
1927
    # full package version
1928
    self._ErrorIf(constants.RELEASE_VERSION != remote_version[1],
1929
                  constants.CV_ENODEVERSION, node,
1930
                  "software version mismatch: master %s, node %s",
1931
                  constants.RELEASE_VERSION, remote_version[1],
1932
                  code=self.ETYPE_WARNING)
1933

    
1934
    hyp_result = nresult.get(constants.NV_HYPERVISOR, None)
1935
    if ninfo.vm_capable and isinstance(hyp_result, dict):
1936
      for hv_name, hv_result in hyp_result.iteritems():
1937
        test = hv_result is not None
1938
        _ErrorIf(test, constants.CV_ENODEHV, node,
1939
                 "hypervisor %s verify failure: '%s'", hv_name, hv_result)
1940

    
1941
    hvp_result = nresult.get(constants.NV_HVPARAMS, None)
1942
    if ninfo.vm_capable and isinstance(hvp_result, list):
1943
      for item, hv_name, hv_result in hvp_result:
1944
        _ErrorIf(True, constants.CV_ENODEHV, node,
1945
                 "hypervisor %s parameter verify failure (source %s): %s",
1946
                 hv_name, item, hv_result)
1947

    
1948
    test = nresult.get(constants.NV_NODESETUP,
1949
                       ["Missing NODESETUP results"])
1950
    _ErrorIf(test, constants.CV_ENODESETUP, node, "node setup error: %s",
1951
             "; ".join(test))
1952

    
1953
    return True
1954

    
1955
  def _VerifyNodeTime(self, ninfo, nresult,
1956
                      nvinfo_starttime, nvinfo_endtime):
1957
    """Check the node time.
1958

1959
    @type ninfo: L{objects.Node}
1960
    @param ninfo: the node to check
1961
    @param nresult: the remote results for the node
1962
    @param nvinfo_starttime: the start time of the RPC call
1963
    @param nvinfo_endtime: the end time of the RPC call
1964

1965
    """
1966
    node = ninfo.name
1967
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
1968

    
1969
    ntime = nresult.get(constants.NV_TIME, None)
1970
    try:
1971
      ntime_merged = utils.MergeTime(ntime)
1972
    except (ValueError, TypeError):
1973
      _ErrorIf(True, constants.CV_ENODETIME, node, "Node returned invalid time")
1974
      return
1975

    
1976
    if ntime_merged < (nvinfo_starttime - constants.NODE_MAX_CLOCK_SKEW):
1977
      ntime_diff = "%.01fs" % abs(nvinfo_starttime - ntime_merged)
1978
    elif ntime_merged > (nvinfo_endtime + constants.NODE_MAX_CLOCK_SKEW):
1979
      ntime_diff = "%.01fs" % abs(ntime_merged - nvinfo_endtime)
1980
    else:
1981
      ntime_diff = None
1982

    
1983
    _ErrorIf(ntime_diff is not None, constants.CV_ENODETIME, node,
1984
             "Node time diverges by at least %s from master node time",
1985
             ntime_diff)
1986

    
1987
  def _VerifyNodeLVM(self, ninfo, nresult, vg_name):
1988
    """Check the node LVM results.
1989

1990
    @type ninfo: L{objects.Node}
1991
    @param ninfo: the node to check
1992
    @param nresult: the remote results for the node
1993
    @param vg_name: the configured VG name
1994

1995
    """
1996
    if vg_name is None:
1997
      return
1998

    
1999
    node = ninfo.name
2000
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
2001

    
2002
    # checks vg existence and size > 20G
2003
    vglist = nresult.get(constants.NV_VGLIST, None)
2004
    test = not vglist
2005
    _ErrorIf(test, constants.CV_ENODELVM, node, "unable to check volume groups")
2006
    if not test:
2007
      vgstatus = utils.CheckVolumeGroupSize(vglist, vg_name,
2008
                                            constants.MIN_VG_SIZE)
2009
      _ErrorIf(vgstatus, constants.CV_ENODELVM, node, vgstatus)
2010

    
2011
    # check pv names
2012
    pvlist = nresult.get(constants.NV_PVLIST, None)
2013
    test = pvlist is None
2014
    _ErrorIf(test, constants.CV_ENODELVM, node, "Can't get PV list from node")
2015
    if not test:
2016
      # check that ':' is not present in PV names, since it's a
2017
      # special character for lvcreate (denotes the range of PEs to
2018
      # use on the PV)
2019
      for _, pvname, owner_vg in pvlist:
2020
        test = ":" in pvname
2021
        _ErrorIf(test, constants.CV_ENODELVM, node,
2022
                 "Invalid character ':' in PV '%s' of VG '%s'",
2023
                 pvname, owner_vg)
2024

    
2025
  def _VerifyNodeBridges(self, ninfo, nresult, bridges):
2026
    """Check the node bridges.
2027

2028
    @type ninfo: L{objects.Node}
2029
    @param ninfo: the node to check
2030
    @param nresult: the remote results for the node
2031
    @param bridges: the expected list of bridges
2032

2033
    """
2034
    if not bridges:
2035
      return
2036

    
2037
    node = ninfo.name
2038
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
2039

    
2040
    missing = nresult.get(constants.NV_BRIDGES, None)
2041
    test = not isinstance(missing, list)
2042
    _ErrorIf(test, constants.CV_ENODENET, node,
2043
             "did not return valid bridge information")
2044
    if not test:
2045
      _ErrorIf(bool(missing), constants.CV_ENODENET, node,
2046
               "missing bridges: %s" % utils.CommaJoin(sorted(missing)))
2047

    
2048
  def _VerifyNodeUserScripts(self, ninfo, nresult):
2049
    """Check the results of user scripts presence and executability on the node
2050

2051
    @type ninfo: L{objects.Node}
2052
    @param ninfo: the node to check
2053
    @param nresult: the remote results for the node
2054

2055
    """
2056
    node = ninfo.name
2057

    
2058
    test = not constants.NV_USERSCRIPTS in nresult
2059
    self._ErrorIf(test, constants.CV_ENODEUSERSCRIPTS, node,
2060
                  "did not return user scripts information")
2061

    
2062
    broken_scripts = nresult.get(constants.NV_USERSCRIPTS, None)
2063
    if not test:
2064
      self._ErrorIf(broken_scripts, constants.CV_ENODEUSERSCRIPTS, node,
2065
                    "user scripts not present or not executable: %s" %
2066
                    utils.CommaJoin(sorted(broken_scripts)))
2067

    
2068
  def _VerifyNodeNetwork(self, ninfo, nresult):
2069
    """Check the node network connectivity results.
2070

2071
    @type ninfo: L{objects.Node}
2072
    @param ninfo: the node to check
2073
    @param nresult: the remote results for the node
2074

2075
    """
2076
    node = ninfo.name
2077
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
2078

    
2079
    test = constants.NV_NODELIST not in nresult
2080
    _ErrorIf(test, constants.CV_ENODESSH, node,
2081
             "node hasn't returned node ssh connectivity data")
2082
    if not test:
2083
      if nresult[constants.NV_NODELIST]:
2084
        for a_node, a_msg in nresult[constants.NV_NODELIST].items():
2085
          _ErrorIf(True, constants.CV_ENODESSH, node,
2086
                   "ssh communication with node '%s': %s", a_node, a_msg)
2087

    
2088
    test = constants.NV_NODENETTEST not in nresult
2089
    _ErrorIf(test, constants.CV_ENODENET, node,
2090
             "node hasn't returned node tcp connectivity data")
2091
    if not test:
2092
      if nresult[constants.NV_NODENETTEST]:
2093
        nlist = utils.NiceSort(nresult[constants.NV_NODENETTEST].keys())
2094
        for anode in nlist:
2095
          _ErrorIf(True, constants.CV_ENODENET, node,
2096
                   "tcp communication with node '%s': %s",
2097
                   anode, nresult[constants.NV_NODENETTEST][anode])
2098

    
2099
    test = constants.NV_MASTERIP not in nresult
2100
    _ErrorIf(test, constants.CV_ENODENET, node,
2101
             "node hasn't returned node master IP reachability data")
2102
    if not test:
2103
      if not nresult[constants.NV_MASTERIP]:
2104
        if node == self.master_node:
2105
          msg = "the master node cannot reach the master IP (not configured?)"
2106
        else:
2107
          msg = "cannot reach the master IP"
2108
        _ErrorIf(True, constants.CV_ENODENET, node, msg)
2109

    
2110
  def _VerifyInstance(self, instance, instanceconfig, node_image,
2111
                      diskstatus):
2112
    """Verify an instance.
2113

2114
    This function checks to see if the required block devices are
2115
    available on the instance's node.
2116

2117
    """
2118
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
2119
    node_current = instanceconfig.primary_node
2120

    
2121
    node_vol_should = {}
2122
    instanceconfig.MapLVsByNode(node_vol_should)
2123

    
2124
    for node in node_vol_should:
2125
      n_img = node_image[node]
2126
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
2127
        # ignore missing volumes on offline or broken nodes
2128
        continue
2129
      for volume in node_vol_should[node]:
2130
        test = volume not in n_img.volumes
2131
        _ErrorIf(test, constants.CV_EINSTANCEMISSINGDISK, instance,
2132
                 "volume %s missing on node %s", volume, node)
2133

    
2134
    if instanceconfig.admin_state == constants.ADMINST_UP:
2135
      pri_img = node_image[node_current]
2136
      test = instance not in pri_img.instances and not pri_img.offline
2137
      _ErrorIf(test, constants.CV_EINSTANCEDOWN, instance,
2138
               "instance not running on its primary node %s",
2139
               node_current)
2140

    
2141
    diskdata = [(nname, success, status, idx)
2142
                for (nname, disks) in diskstatus.items()
2143
                for idx, (success, status) in enumerate(disks)]
2144

    
2145
    for nname, success, bdev_status, idx in diskdata:
2146
      # the 'ghost node' construction in Exec() ensures that we have a
2147
      # node here
2148
      snode = node_image[nname]
2149
      bad_snode = snode.ghost or snode.offline
2150
      _ErrorIf(instanceconfig.admin_state == constants.ADMINST_UP and
2151
               not success and not bad_snode,
2152
               constants.CV_EINSTANCEFAULTYDISK, instance,
2153
               "couldn't retrieve status for disk/%s on %s: %s",
2154
               idx, nname, bdev_status)
2155
      _ErrorIf((instanceconfig.admin_state == constants.ADMINST_UP and
2156
                success and bdev_status.ldisk_status == constants.LDS_FAULTY),
2157
               constants.CV_EINSTANCEFAULTYDISK, instance,
2158
               "disk/%s on %s is faulty", idx, nname)
2159

    
2160
  def _VerifyOrphanVolumes(self, node_vol_should, node_image, reserved):
2161
    """Verify if there are any unknown volumes in the cluster.
2162

2163
    The .os, .swap and backup volumes are ignored. All other volumes are
2164
    reported as unknown.
2165

2166
    @type reserved: L{ganeti.utils.FieldSet}
2167
    @param reserved: a FieldSet of reserved volume names
2168

2169
    """
2170
    for node, n_img in node_image.items():
2171
      if n_img.offline or n_img.rpc_fail or n_img.lvm_fail:
2172
        # skip non-healthy nodes
2173
        continue
2174
      for volume in n_img.volumes:
2175
        test = ((node not in node_vol_should or
2176
                volume not in node_vol_should[node]) and
2177
                not reserved.Matches(volume))
2178
        self._ErrorIf(test, constants.CV_ENODEORPHANLV, node,
2179
                      "volume %s is unknown", volume)
2180

    
2181
  def _VerifyNPlusOneMemory(self, node_image, instance_cfg):
2182
    """Verify N+1 Memory Resilience.
2183

2184
    Check that if one single node dies we can still start all the
2185
    instances it was primary for.
2186

2187
    """
2188
    cluster_info = self.cfg.GetClusterInfo()
2189
    for node, n_img in node_image.items():
2190
      # This code checks that every node which is now listed as
2191
      # secondary has enough memory to host all instances it is
2192
      # supposed to should a single other node in the cluster fail.
2193
      # FIXME: not ready for failover to an arbitrary node
2194
      # FIXME: does not support file-backed instances
2195
      # WARNING: we currently take into account down instances as well
2196
      # as up ones, considering that even if they're down someone
2197
      # might want to start them even in the event of a node failure.
2198
      if n_img.offline:
2199
        # we're skipping offline nodes from the N+1 warning, since
2200
        # most likely we don't have good memory infromation from them;
2201
        # we already list instances living on such nodes, and that's
2202
        # enough warning
2203
        continue
2204
      #TODO(dynmem): use MINMEM for checking
2205
      #TODO(dynmem): also consider ballooning out other instances
2206
      for prinode, instances in n_img.sbp.items():
2207
        needed_mem = 0
2208
        for instance in instances:
2209
          bep = cluster_info.FillBE(instance_cfg[instance])
2210
          if bep[constants.BE_AUTO_BALANCE]:
2211
            needed_mem += bep[constants.BE_MAXMEM]
2212
        test = n_img.mfree < needed_mem
2213
        self._ErrorIf(test, constants.CV_ENODEN1, node,
2214
                      "not enough memory to accomodate instance failovers"
2215
                      " should node %s fail (%dMiB needed, %dMiB available)",
2216
                      prinode, needed_mem, n_img.mfree)
2217

    
2218
  @classmethod
2219
  def _VerifyFiles(cls, errorif, nodeinfo, master_node, all_nvinfo,
2220
                   (files_all, files_opt, files_mc, files_vm)):
2221
    """Verifies file checksums collected from all nodes.
2222

2223
    @param errorif: Callback for reporting errors
2224
    @param nodeinfo: List of L{objects.Node} objects
2225
    @param master_node: Name of master node
2226
    @param all_nvinfo: RPC results
2227

2228
    """
2229
    # Define functions determining which nodes to consider for a file
2230
    files2nodefn = [
2231
      (files_all, None),
2232
      (files_mc, lambda node: (node.master_candidate or
2233
                               node.name == master_node)),
2234
      (files_vm, lambda node: node.vm_capable),
2235
      ]
2236

    
2237
    # Build mapping from filename to list of nodes which should have the file
2238
    nodefiles = {}
2239
    for (files, fn) in files2nodefn:
2240
      if fn is None:
2241
        filenodes = nodeinfo
2242
      else:
2243
        filenodes = filter(fn, nodeinfo)
2244
      nodefiles.update((filename,
2245
                        frozenset(map(operator.attrgetter("name"), filenodes)))
2246
                       for filename in files)
2247

    
2248
    assert set(nodefiles) == (files_all | files_mc | files_vm)
2249

    
2250
    fileinfo = dict((filename, {}) for filename in nodefiles)
2251
    ignore_nodes = set()
2252

    
2253
    for node in nodeinfo:
2254
      if node.offline:
2255
        ignore_nodes.add(node.name)
2256
        continue
2257

    
2258
      nresult = all_nvinfo[node.name]
2259

    
2260
      if nresult.fail_msg or not nresult.payload:
2261
        node_files = None
2262
      else:
2263
        node_files = nresult.payload.get(constants.NV_FILELIST, None)
2264

    
2265
      test = not (node_files and isinstance(node_files, dict))
2266
      errorif(test, constants.CV_ENODEFILECHECK, node.name,
2267
              "Node did not return file checksum data")
2268
      if test:
2269
        ignore_nodes.add(node.name)
2270
        continue
2271

    
2272
      # Build per-checksum mapping from filename to nodes having it
2273
      for (filename, checksum) in node_files.items():
2274
        assert filename in nodefiles
2275
        fileinfo[filename].setdefault(checksum, set()).add(node.name)
2276

    
2277
    for (filename, checksums) in fileinfo.items():
2278
      assert compat.all(len(i) > 10 for i in checksums), "Invalid checksum"
2279

    
2280
      # Nodes having the file
2281
      with_file = frozenset(node_name
2282
                            for nodes in fileinfo[filename].values()
2283
                            for node_name in nodes) - ignore_nodes
2284

    
2285
      expected_nodes = nodefiles[filename] - ignore_nodes
2286

    
2287
      # Nodes missing file
2288
      missing_file = expected_nodes - with_file
2289

    
2290
      if filename in files_opt:
2291
        # All or no nodes
2292
        errorif(missing_file and missing_file != expected_nodes,
2293
                constants.CV_ECLUSTERFILECHECK, None,
2294
                "File %s is optional, but it must exist on all or no"
2295
                " nodes (not found on %s)",
2296
                filename, utils.CommaJoin(utils.NiceSort(missing_file)))
2297
      else:
2298
        errorif(missing_file, constants.CV_ECLUSTERFILECHECK, None,
2299
                "File %s is missing from node(s) %s", filename,
2300
                utils.CommaJoin(utils.NiceSort(missing_file)))
2301

    
2302
        # Warn if a node has a file it shouldn't
2303
        unexpected = with_file - expected_nodes
2304
        errorif(unexpected,
2305
                constants.CV_ECLUSTERFILECHECK, None,
2306
                "File %s should not exist on node(s) %s",
2307
                filename, utils.CommaJoin(utils.NiceSort(unexpected)))
2308

    
2309
      # See if there are multiple versions of the file
2310
      test = len(checksums) > 1
2311
      if test:
2312
        variants = ["variant %s on %s" %
2313
                    (idx + 1, utils.CommaJoin(utils.NiceSort(nodes)))
2314
                    for (idx, (checksum, nodes)) in
2315
                      enumerate(sorted(checksums.items()))]
2316
      else:
2317
        variants = []
2318

    
2319
      errorif(test, constants.CV_ECLUSTERFILECHECK, None,
2320
              "File %s found with %s different checksums (%s)",
2321
              filename, len(checksums), "; ".join(variants))
2322

    
2323
  def _VerifyNodeDrbd(self, ninfo, nresult, instanceinfo, drbd_helper,
2324
                      drbd_map):
2325
    """Verifies and the node DRBD status.
2326

2327
    @type ninfo: L{objects.Node}
2328
    @param ninfo: the node to check
2329
    @param nresult: the remote results for the node
2330
    @param instanceinfo: the dict of instances
2331
    @param drbd_helper: the configured DRBD usermode helper
2332
    @param drbd_map: the DRBD map as returned by
2333
        L{ganeti.config.ConfigWriter.ComputeDRBDMap}
2334

2335
    """
2336
    node = ninfo.name
2337
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
2338

    
2339
    if drbd_helper:
2340
      helper_result = nresult.get(constants.NV_DRBDHELPER, None)
2341
      test = (helper_result == None)
2342
      _ErrorIf(test, constants.CV_ENODEDRBDHELPER, node,
2343
               "no drbd usermode helper returned")
2344
      if helper_result:
2345
        status, payload = helper_result
2346
        test = not status
2347
        _ErrorIf(test, constants.CV_ENODEDRBDHELPER, node,
2348
                 "drbd usermode helper check unsuccessful: %s", payload)
2349
        test = status and (payload != drbd_helper)
2350
        _ErrorIf(test, constants.CV_ENODEDRBDHELPER, node,
2351
                 "wrong drbd usermode helper: %s", payload)
2352

    
2353
    # compute the DRBD minors
2354
    node_drbd = {}
2355
    for minor, instance in drbd_map[node].items():
2356
      test = instance not in instanceinfo
2357
      _ErrorIf(test, constants.CV_ECLUSTERCFG, None,
2358
               "ghost instance '%s' in temporary DRBD map", instance)
2359
        # ghost instance should not be running, but otherwise we
2360
        # don't give double warnings (both ghost instance and
2361
        # unallocated minor in use)
2362
      if test:
2363
        node_drbd[minor] = (instance, False)
2364
      else:
2365
        instance = instanceinfo[instance]
2366
        node_drbd[minor] = (instance.name,
2367
                            instance.admin_state == constants.ADMINST_UP)
2368

    
2369
    # and now check them
2370
    used_minors = nresult.get(constants.NV_DRBDLIST, [])
2371
    test = not isinstance(used_minors, (tuple, list))
2372
    _ErrorIf(test, constants.CV_ENODEDRBD, node,
2373
             "cannot parse drbd status file: %s", str(used_minors))
2374
    if test:
2375
      # we cannot check drbd status
2376
      return
2377

    
2378
    for minor, (iname, must_exist) in node_drbd.items():
2379
      test = minor not in used_minors and must_exist
2380
      _ErrorIf(test, constants.CV_ENODEDRBD, node,
2381
               "drbd minor %d of instance %s is not active", minor, iname)
2382
    for minor in used_minors:
2383
      test = minor not in node_drbd
2384
      _ErrorIf(test, constants.CV_ENODEDRBD, node,
2385
               "unallocated drbd minor %d is in use", minor)
2386

    
2387
  def _UpdateNodeOS(self, ninfo, nresult, nimg):
2388
    """Builds the node OS structures.
2389

2390
    @type ninfo: L{objects.Node}
2391
    @param ninfo: the node to check
2392
    @param nresult: the remote results for the node
2393
    @param nimg: the node image object
2394

2395
    """
2396
    node = ninfo.name
2397
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
2398

    
2399
    remote_os = nresult.get(constants.NV_OSLIST, None)
2400
    test = (not isinstance(remote_os, list) or
2401
            not compat.all(isinstance(v, list) and len(v) == 7
2402
                           for v in remote_os))
2403

    
2404
    _ErrorIf(test, constants.CV_ENODEOS, node,
2405
             "node hasn't returned valid OS data")
2406

    
2407
    nimg.os_fail = test
2408

    
2409
    if test:
2410
      return
2411

    
2412
    os_dict = {}
2413

    
2414
    for (name, os_path, status, diagnose,
2415
         variants, parameters, api_ver) in nresult[constants.NV_OSLIST]:
2416

    
2417
      if name not in os_dict:
2418
        os_dict[name] = []
2419

    
2420
      # parameters is a list of lists instead of list of tuples due to
2421
      # JSON lacking a real tuple type, fix it:
2422
      parameters = [tuple(v) for v in parameters]
2423
      os_dict[name].append((os_path, status, diagnose,
2424
                            set(variants), set(parameters), set(api_ver)))
2425

    
2426
    nimg.oslist = os_dict
2427

    
2428
  def _VerifyNodeOS(self, ninfo, nimg, base):
2429
    """Verifies the node OS list.
2430

2431
    @type ninfo: L{objects.Node}
2432
    @param ninfo: the node to check
2433
    @param nimg: the node image object
2434
    @param base: the 'template' node we match against (e.g. from the master)
2435

2436
    """
2437
    node = ninfo.name
2438
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
2439

    
2440
    assert not nimg.os_fail, "Entered _VerifyNodeOS with failed OS rpc?"
2441

    
2442
    beautify_params = lambda l: ["%s: %s" % (k, v) for (k, v) in l]
2443
    for os_name, os_data in nimg.oslist.items():
2444
      assert os_data, "Empty OS status for OS %s?!" % os_name
2445
      f_path, f_status, f_diag, f_var, f_param, f_api = os_data[0]
2446
      _ErrorIf(not f_status, constants.CV_ENODEOS, node,
2447
               "Invalid OS %s (located at %s): %s", os_name, f_path, f_diag)
2448
      _ErrorIf(len(os_data) > 1, constants.CV_ENODEOS, node,
2449
               "OS '%s' has multiple entries (first one shadows the rest): %s",
2450
               os_name, utils.CommaJoin([v[0] for v in os_data]))
2451
      # comparisons with the 'base' image
2452
      test = os_name not in base.oslist
2453
      _ErrorIf(test, constants.CV_ENODEOS, node,
2454
               "Extra OS %s not present on reference node (%s)",
2455
               os_name, base.name)
2456
      if test:
2457
        continue
2458
      assert base.oslist[os_name], "Base node has empty OS status?"
2459
      _, b_status, _, b_var, b_param, b_api = base.oslist[os_name][0]
2460
      if not b_status:
2461
        # base OS is invalid, skipping
2462
        continue
2463
      for kind, a, b in [("API version", f_api, b_api),
2464
                         ("variants list", f_var, b_var),
2465
                         ("parameters", beautify_params(f_param),
2466
                          beautify_params(b_param))]:
2467
        _ErrorIf(a != b, constants.CV_ENODEOS, node,
2468
                 "OS %s for %s differs from reference node %s: [%s] vs. [%s]",
2469
                 kind, os_name, base.name,
2470
                 utils.CommaJoin(sorted(a)), utils.CommaJoin(sorted(b)))
2471

    
2472
    # check any missing OSes
2473
    missing = set(base.oslist.keys()).difference(nimg.oslist.keys())
2474
    _ErrorIf(missing, constants.CV_ENODEOS, node,
2475
             "OSes present on reference node %s but missing on this node: %s",
2476
             base.name, utils.CommaJoin(missing))
2477

    
2478
  def _VerifyOob(self, ninfo, nresult):
2479
    """Verifies out of band functionality of a node.
2480

2481
    @type ninfo: L{objects.Node}
2482
    @param ninfo: the node to check
2483
    @param nresult: the remote results for the node
2484

2485
    """
2486
    node = ninfo.name
2487
    # We just have to verify the paths on master and/or master candidates
2488
    # as the oob helper is invoked on the master
2489
    if ((ninfo.master_candidate or ninfo.master_capable) and
2490
        constants.NV_OOB_PATHS in nresult):
2491
      for path_result in nresult[constants.NV_OOB_PATHS]:
2492
        self._ErrorIf(path_result, constants.CV_ENODEOOBPATH, node, path_result)
2493

    
2494
  def _UpdateNodeVolumes(self, ninfo, nresult, nimg, vg_name):
2495
    """Verifies and updates the node volume data.
2496

2497
    This function will update a L{NodeImage}'s internal structures
2498
    with data from the remote call.
2499

2500
    @type ninfo: L{objects.Node}
2501
    @param ninfo: the node to check
2502
    @param nresult: the remote results for the node
2503
    @param nimg: the node image object
2504
    @param vg_name: the configured VG name
2505

2506
    """
2507
    node = ninfo.name
2508
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
2509

    
2510
    nimg.lvm_fail = True
2511
    lvdata = nresult.get(constants.NV_LVLIST, "Missing LV data")
2512
    if vg_name is None:
2513
      pass
2514
    elif isinstance(lvdata, basestring):
2515
      _ErrorIf(True, constants.CV_ENODELVM, node, "LVM problem on node: %s",
2516
               utils.SafeEncode(lvdata))
2517
    elif not isinstance(lvdata, dict):
2518
      _ErrorIf(True, constants.CV_ENODELVM, node,
2519
               "rpc call to node failed (lvlist)")
2520
    else:
2521
      nimg.volumes = lvdata
2522
      nimg.lvm_fail = False
2523

    
2524
  def _UpdateNodeInstances(self, ninfo, nresult, nimg):
2525
    """Verifies and updates the node instance list.
2526

2527
    If the listing was successful, then updates this node's instance
2528
    list. Otherwise, it marks the RPC call as failed for the instance
2529
    list key.
2530

2531
    @type ninfo: L{objects.Node}
2532
    @param ninfo: the node to check
2533
    @param nresult: the remote results for the node
2534
    @param nimg: the node image object
2535

2536
    """
2537
    idata = nresult.get(constants.NV_INSTANCELIST, None)
2538
    test = not isinstance(idata, list)
2539
    self._ErrorIf(test, constants.CV_ENODEHV, ninfo.name,
2540
                  "rpc call to node failed (instancelist): %s",
2541
                  utils.SafeEncode(str(idata)))
2542
    if test:
2543
      nimg.hyp_fail = True
2544
    else:
2545
      nimg.instances = idata
2546

    
2547
  def _UpdateNodeInfo(self, ninfo, nresult, nimg, vg_name):
2548
    """Verifies and computes a node information map
2549

2550
    @type ninfo: L{objects.Node}
2551
    @param ninfo: the node to check
2552
    @param nresult: the remote results for the node
2553
    @param nimg: the node image object
2554
    @param vg_name: the configured VG name
2555

2556
    """
2557
    node = ninfo.name
2558
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
2559

    
2560
    # try to read free memory (from the hypervisor)
2561
    hv_info = nresult.get(constants.NV_HVINFO, None)
2562
    test = not isinstance(hv_info, dict) or "memory_free" not in hv_info
2563
    _ErrorIf(test, constants.CV_ENODEHV, node,
2564
             "rpc call to node failed (hvinfo)")
2565
    if not test:
2566
      try:
2567
        nimg.mfree = int(hv_info["memory_free"])
2568
      except (ValueError, TypeError):
2569
        _ErrorIf(True, constants.CV_ENODERPC, node,
2570
                 "node returned invalid nodeinfo, check hypervisor")
2571

    
2572
    # FIXME: devise a free space model for file based instances as well
2573
    if vg_name is not None:
2574
      test = (constants.NV_VGLIST not in nresult or
2575
              vg_name not in nresult[constants.NV_VGLIST])
2576
      _ErrorIf(test, constants.CV_ENODELVM, node,
2577
               "node didn't return data for the volume group '%s'"
2578
               " - it is either missing or broken", vg_name)
2579
      if not test:
2580
        try:
2581
          nimg.dfree = int(nresult[constants.NV_VGLIST][vg_name])
2582
        except (ValueError, TypeError):
2583
          _ErrorIf(True, constants.CV_ENODERPC, node,
2584
                   "node returned invalid LVM info, check LVM status")
2585

    
2586
  def _CollectDiskInfo(self, nodelist, node_image, instanceinfo):
2587
    """Gets per-disk status information for all instances.
2588

2589
    @type nodelist: list of strings
2590
    @param nodelist: Node names
2591
    @type node_image: dict of (name, L{objects.Node})
2592
    @param node_image: Node objects
2593
    @type instanceinfo: dict of (name, L{objects.Instance})
2594
    @param instanceinfo: Instance objects
2595
    @rtype: {instance: {node: [(succes, payload)]}}
2596
    @return: a dictionary of per-instance dictionaries with nodes as
2597
        keys and disk information as values; the disk information is a
2598
        list of tuples (success, payload)
2599

2600
    """
2601
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
2602

    
2603
    node_disks = {}
2604
    node_disks_devonly = {}
2605
    diskless_instances = set()
2606
    diskless = constants.DT_DISKLESS
2607

    
2608
    for nname in nodelist:
2609
      node_instances = list(itertools.chain(node_image[nname].pinst,
2610
                                            node_image[nname].sinst))
2611
      diskless_instances.update(inst for inst in node_instances
2612
                                if instanceinfo[inst].disk_template == diskless)
2613
      disks = [(inst, disk)
2614
               for inst in node_instances
2615
               for disk in instanceinfo[inst].disks]
2616

    
2617
      if not disks:
2618
        # No need to collect data
2619
        continue
2620

    
2621
      node_disks[nname] = disks
2622

    
2623
      # Creating copies as SetDiskID below will modify the objects and that can
2624
      # lead to incorrect data returned from nodes
2625
      devonly = [dev.Copy() for (_, dev) in disks]
2626

    
2627
      for dev in devonly:
2628
        self.cfg.SetDiskID(dev, nname)
2629

    
2630
      node_disks_devonly[nname] = devonly
2631

    
2632
    assert len(node_disks) == len(node_disks_devonly)
2633

    
2634
    # Collect data from all nodes with disks
2635
    result = self.rpc.call_blockdev_getmirrorstatus_multi(node_disks.keys(),
2636
                                                          node_disks_devonly)
2637

    
2638
    assert len(result) == len(node_disks)
2639

    
2640
    instdisk = {}
2641

    
2642
    for (nname, nres) in result.items():
2643
      disks = node_disks[nname]
2644

    
2645
      if nres.offline:
2646
        # No data from this node
2647
        data = len(disks) * [(False, "node offline")]
2648
      else:
2649
        msg = nres.fail_msg
2650
        _ErrorIf(msg, constants.CV_ENODERPC, nname,
2651
                 "while getting disk information: %s", msg)
2652
        if msg:
2653
          # No data from this node
2654
          data = len(disks) * [(False, msg)]
2655
        else:
2656
          data = []
2657
          for idx, i in enumerate(nres.payload):
2658
            if isinstance(i, (tuple, list)) and len(i) == 2:
2659
              data.append(i)
2660
            else:
2661
              logging.warning("Invalid result from node %s, entry %d: %s",
2662
                              nname, idx, i)
2663
              data.append((False, "Invalid result from the remote node"))
2664

    
2665
      for ((inst, _), status) in zip(disks, data):
2666
        instdisk.setdefault(inst, {}).setdefault(nname, []).append(status)
2667

    
2668
    # Add empty entries for diskless instances.
2669
    for inst in diskless_instances:
2670
      assert inst not in instdisk
2671
      instdisk[inst] = {}
2672

    
2673
    assert compat.all(len(statuses) == len(instanceinfo[inst].disks) and
2674
                      len(nnames) <= len(instanceinfo[inst].all_nodes) and
2675
                      compat.all(isinstance(s, (tuple, list)) and
2676
                                 len(s) == 2 for s in statuses)
2677
                      for inst, nnames in instdisk.items()
2678
                      for nname, statuses in nnames.items())
2679
    assert set(instdisk) == set(instanceinfo), "instdisk consistency failure"
2680

    
2681
    return instdisk
2682

    
2683
  @staticmethod
2684
  def _SshNodeSelector(group_uuid, all_nodes):
2685
    """Create endless iterators for all potential SSH check hosts.
2686

2687
    """
2688
    nodes = [node for node in all_nodes
2689
             if (node.group != group_uuid and
2690
                 not node.offline)]
2691
    keyfunc = operator.attrgetter("group")
2692

    
2693
    return map(itertools.cycle,
2694
               [sorted(map(operator.attrgetter("name"), names))
2695
                for _, names in itertools.groupby(sorted(nodes, key=keyfunc),
2696
                                                  keyfunc)])
2697

    
2698
  @classmethod
2699
  def _SelectSshCheckNodes(cls, group_nodes, group_uuid, all_nodes):
2700
    """Choose which nodes should talk to which other nodes.
2701

2702
    We will make nodes contact all nodes in their group, and one node from
2703
    every other group.
2704

2705
    @warning: This algorithm has a known issue if one node group is much
2706
      smaller than others (e.g. just one node). In such a case all other
2707
      nodes will talk to the single node.
2708

2709
    """
2710
    online_nodes = sorted(node.name for node in group_nodes if not node.offline)
2711
    sel = cls._SshNodeSelector(group_uuid, all_nodes)
2712

    
2713
    return (online_nodes,
2714
            dict((name, sorted([i.next() for i in sel]))
2715
                 for name in online_nodes))
2716

    
2717
  def BuildHooksEnv(self):
2718
    """Build hooks env.
2719

2720
    Cluster-Verify hooks just ran in the post phase and their failure makes
2721
    the output be logged in the verify output and the verification to fail.
2722

2723
    """
2724
    env = {
2725
      "CLUSTER_TAGS": " ".join(self.cfg.GetClusterInfo().GetTags())
2726
      }
2727

    
2728
    env.update(("NODE_TAGS_%s" % node.name, " ".join(node.GetTags()))
2729
               for node in self.my_node_info.values())
2730

    
2731
    return env
2732

    
2733
  def BuildHooksNodes(self):
2734
    """Build hooks nodes.
2735

2736
    """
2737
    return ([], self.my_node_names)
2738

    
2739
  def Exec(self, feedback_fn):
2740
    """Verify integrity of the node group, performing various test on nodes.
2741

2742
    """
2743
    # This method has too many local variables. pylint: disable=R0914
2744
    feedback_fn("* Verifying group '%s'" % self.group_info.name)
2745

    
2746
    if not self.my_node_names:
2747
      # empty node group
2748
      feedback_fn("* Empty node group, skipping verification")
2749
      return True
2750

    
2751
    self.bad = False
2752
    _ErrorIf = self._ErrorIf # pylint: disable=C0103
2753
    verbose = self.op.verbose
2754
    self._feedback_fn = feedback_fn
2755

    
2756
    vg_name = self.cfg.GetVGName()
2757
    drbd_helper = self.cfg.GetDRBDHelper()
2758
    cluster = self.cfg.GetClusterInfo()
2759
    groupinfo = self.cfg.GetAllNodeGroupsInfo()
2760
    hypervisors = cluster.enabled_hypervisors
2761
    node_data_list = [self.my_node_info[name] for name in self.my_node_names]
2762

    
2763
    i_non_redundant = [] # Non redundant instances
2764
    i_non_a_balanced = [] # Non auto-balanced instances
2765
    i_offline = 0 # Count of offline instances
2766
    n_offline = 0 # Count of offline nodes
2767
    n_drained = 0 # Count of nodes being drained
2768
    node_vol_should = {}
2769

    
2770
    # FIXME: verify OS list
2771

    
2772
    # File verification
2773
    filemap = _ComputeAncillaryFiles(cluster, False)
2774

    
2775
    # do local checksums
2776
    master_node = self.master_node = self.cfg.GetMasterNode()
2777
    master_ip = self.cfg.GetMasterIP()
2778

    
2779
    feedback_fn("* Gathering data (%d nodes)" % len(self.my_node_names))
2780

    
2781
    user_scripts = []
2782
    if self.cfg.GetUseExternalMipScript():
2783
      user_scripts.append(constants.EXTERNAL_MASTER_SETUP_SCRIPT)
2784

    
2785
    node_verify_param = {
2786
      constants.NV_FILELIST:
2787
        utils.UniqueSequence(filename
2788
                             for files in filemap
2789
                             for filename in files),
2790
      constants.NV_NODELIST:
2791
        self._SelectSshCheckNodes(node_data_list, self.group_uuid,
2792
                                  self.all_node_info.values()),
2793
      constants.NV_HYPERVISOR: hypervisors,
2794
      constants.NV_HVPARAMS:
2795
        _GetAllHypervisorParameters(cluster, self.all_inst_info.values()),
2796
      constants.NV_NODENETTEST: [(node.name, node.primary_ip, node.secondary_ip)
2797
                                 for node in node_data_list
2798
                                 if not node.offline],
2799
      constants.NV_INSTANCELIST: hypervisors,
2800
      constants.NV_VERSION: None,
2801
      constants.NV_HVINFO: self.cfg.GetHypervisorType(),
2802
      constants.NV_NODESETUP: None,
2803
      constants.NV_TIME: None,
2804
      constants.NV_MASTERIP: (master_node, master_ip),
2805
      constants.NV_OSLIST: None,
2806
      constants.NV_VMNODES: self.cfg.GetNonVmCapableNodeList(),
2807
      constants.NV_USERSCRIPTS: user_scripts,
2808
      }
2809

    
2810
    if vg_name is not None:
2811
      node_verify_param[constants.NV_VGLIST] = None
2812
      node_verify_param[constants.NV_LVLIST] = vg_name
2813
      node_verify_param[constants.NV_PVLIST] = [vg_name]
2814
      node_verify_param[constants.NV_DRBDLIST] = None
2815

    
2816
    if drbd_helper:
2817
      node_verify_param[constants.NV_DRBDHELPER] = drbd_helper
2818

    
2819
    # bridge checks
2820
    # FIXME: this needs to be changed per node-group, not cluster-wide
2821
    bridges = set()
2822
    default_nicpp = cluster.nicparams[constants.PP_DEFAULT]
2823
    if default_nicpp[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2824
      bridges.add(default_nicpp[constants.NIC_LINK])
2825
    for instance in self.my_inst_info.values():
2826
      for nic in instance.nics:
2827
        full_nic = cluster.SimpleFillNIC(nic.nicparams)
2828
        if full_nic[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
2829
          bridges.add(full_nic[constants.NIC_LINK])
2830

    
2831
    if bridges:
2832
      node_verify_param[constants.NV_BRIDGES] = list(bridges)
2833

    
2834
    # Build our expected cluster state
2835
    node_image = dict((node.name, self.NodeImage(offline=node.offline,
2836
                                                 name=node.name,
2837
                                                 vm_capable=node.vm_capable))
2838
                      for node in node_data_list)
2839

    
2840
    # Gather OOB paths
2841
    oob_paths = []
2842
    for node in self.all_node_info.values():
2843
      path = _SupportsOob(self.cfg, node)
2844
      if path and path not in oob_paths:
2845
        oob_paths.append(path)
2846

    
2847
    if oob_paths:
2848
      node_verify_param[constants.NV_OOB_PATHS] = oob_paths
2849

    
2850
    for instance in self.my_inst_names:
2851
      inst_config = self.my_inst_info[instance]
2852

    
2853
      for nname in inst_config.all_nodes:
2854
        if nname not in node_image:
2855
          gnode = self.NodeImage(name=nname)
2856
          gnode.ghost = (nname not in self.all_node_info)
2857
          node_image[nname] = gnode
2858

    
2859
      inst_config.MapLVsByNode(node_vol_should)
2860

    
2861
      pnode = inst_config.primary_node
2862
      node_image[pnode].pinst.append(instance)
2863

    
2864
      for snode in inst_config.secondary_nodes:
2865
        nimg = node_image[snode]
2866
        nimg.sinst.append(instance)
2867
        if pnode not in nimg.sbp:
2868
          nimg.sbp[pnode] = []
2869
        nimg.sbp[pnode].append(instance)
2870

    
2871
    # At this point, we have the in-memory data structures complete,
2872
    # except for the runtime information, which we'll gather next
2873

    
2874
    # Due to the way our RPC system works, exact response times cannot be
2875
    # guaranteed (e.g. a broken node could run into a timeout). By keeping the
2876
    # time before and after executing the request, we can at least have a time
2877
    # window.
2878
    nvinfo_starttime = time.time()
2879
    all_nvinfo = self.rpc.call_node_verify(self.my_node_names,
2880
                                           node_verify_param,
2881
                                           self.cfg.GetClusterName())
2882
    nvinfo_endtime = time.time()
2883

    
2884
    if self.extra_lv_nodes and vg_name is not None:
2885
      extra_lv_nvinfo = \
2886
          self.rpc.call_node_verify(self.extra_lv_nodes,
2887
                                    {constants.NV_LVLIST: vg_name},
2888
                                    self.cfg.GetClusterName())
2889
    else:
2890
      extra_lv_nvinfo = {}
2891

    
2892
    all_drbd_map = self.cfg.ComputeDRBDMap()
2893

    
2894
    feedback_fn("* Gathering disk information (%s nodes)" %
2895
                len(self.my_node_names))
2896
    instdisk = self._CollectDiskInfo(self.my_node_names, node_image,
2897
                                     self.my_inst_info)
2898

    
2899
    feedback_fn("* Verifying configuration file consistency")
2900

    
2901
    # If not all nodes are being checked, we need to make sure the master node
2902
    # and a non-checked vm_capable node are in the list.
2903
    absent_nodes = set(self.all_node_info).difference(self.my_node_info)
2904
    if absent_nodes:
2905
      vf_nvinfo = all_nvinfo.copy()
2906
      vf_node_info = list(self.my_node_info.values())
2907
      additional_nodes = []
2908
      if master_node not in self.my_node_info:
2909
        additional_nodes.append(master_node)
2910
        vf_node_info.append(self.all_node_info[master_node])
2911
      # Add the first vm_capable node we find which is not included
2912
      for node in absent_nodes:
2913
        nodeinfo = self.all_node_info[node]
2914
        if nodeinfo.vm_capable and not nodeinfo.offline:
2915
          additional_nodes.append(node)
2916
          vf_node_info.append(self.all_node_info[node])
2917
          break
2918
      key = constants.NV_FILELIST
2919
      vf_nvinfo.update(self.rpc.call_node_verify(additional_nodes,
2920
                                                 {key: node_verify_param[key]},
2921
                                                 self.cfg.GetClusterName()))
2922
    else:
2923
      vf_nvinfo = all_nvinfo
2924
      vf_node_info = self.my_node_info.values()
2925

    
2926
    self._VerifyFiles(_ErrorIf, vf_node_info, master_node, vf_nvinfo, filemap)
2927

    
2928
    feedback_fn("* Verifying node status")
2929

    
2930
    refos_img = None
2931

    
2932
    for node_i in node_data_list:
2933
      node = node_i.name
2934
      nimg = node_image[node]
2935

    
2936
      if node_i.offline:
2937
        if verbose:
2938
          feedback_fn("* Skipping offline node %s" % (node,))
2939
        n_offline += 1
2940
        continue
2941

    
2942
      if node == master_node:
2943
        ntype = "master"
2944
      elif node_i.master_candidate:
2945
        ntype = "master candidate"
2946
      elif node_i.drained:
2947
        ntype = "drained"
2948
        n_drained += 1
2949
      else:
2950
        ntype = "regular"
2951
      if verbose:
2952
        feedback_fn("* Verifying node %s (%s)" % (node, ntype))
2953

    
2954
      msg = all_nvinfo[node].fail_msg
2955
      _ErrorIf(msg, constants.CV_ENODERPC, node, "while contacting node: %s",
2956
               msg)
2957
      if msg:
2958
        nimg.rpc_fail = True
2959
        continue
2960

    
2961
      nresult = all_nvinfo[node].payload
2962

    
2963
      nimg.call_ok = self._VerifyNode(node_i, nresult)
2964
      self._VerifyNodeTime(node_i, nresult, nvinfo_starttime, nvinfo_endtime)
2965
      self._VerifyNodeNetwork(node_i, nresult)
2966
      self._VerifyNodeUserScripts(node_i, nresult)
2967
      self._VerifyOob(node_i, nresult)
2968

    
2969
      if nimg.vm_capable:
2970
        self._VerifyNodeLVM(node_i, nresult, vg_name)
2971
        self._VerifyNodeDrbd(node_i, nresult, self.all_inst_info, drbd_helper,
2972
                             all_drbd_map)
2973

    
2974
        self._UpdateNodeVolumes(node_i, nresult, nimg, vg_name)
2975
        self._UpdateNodeInstances(node_i, nresult, nimg)
2976
        self._UpdateNodeInfo(node_i, nresult, nimg, vg_name)
2977
        self._UpdateNodeOS(node_i, nresult, nimg)
2978

    
2979
        if not nimg.os_fail:
2980
          if refos_img is None:
2981
            refos_img = nimg
2982
          self._VerifyNodeOS(node_i, nimg, refos_img)
2983
        self._VerifyNodeBridges(node_i, nresult, bridges)
2984

    
2985
        # Check whether all running instancies are primary for the node. (This
2986
        # can no longer be done from _VerifyInstance below, since some of the
2987
        # wrong instances could be from other node groups.)
2988
        non_primary_inst = set(nimg.instances).difference(nimg.pinst)
2989

    
2990
        for inst in non_primary_inst:
2991
          # FIXME: investigate best way to handle offline insts
2992
          if inst.admin_state == constants.ADMINST_OFFLINE:
2993
            if verbose:
2994
              feedback_fn("* Skipping offline instance %s" % inst.name)
2995
            i_offline += 1
2996
            continue
2997
          test = inst in self.all_inst_info
2998
          _ErrorIf(test, constants.CV_EINSTANCEWRONGNODE, inst,
2999
                   "instance should not run on node %s", node_i.name)
3000
          _ErrorIf(not test, constants.CV_ENODEORPHANINSTANCE, node_i.name,
3001
                   "node is running unknown instance %s", inst)
3002

    
3003
    for node, result in extra_lv_nvinfo.items():
3004
      self._UpdateNodeVolumes(self.all_node_info[node], result.payload,
3005
                              node_image[node], vg_name)
3006

    
3007
    feedback_fn("* Verifying instance status")
3008
    for instance in self.my_inst_names:
3009
      if verbose:
3010
        feedback_fn("* Verifying instance %s" % instance)
3011
      inst_config = self.my_inst_info[instance]
3012
      self._VerifyInstance(instance, inst_config, node_image,
3013
                           instdisk[instance])
3014
      inst_nodes_offline = []
3015

    
3016
      pnode = inst_config.primary_node
3017
      pnode_img = node_image[pnode]
3018
      _ErrorIf(pnode_img.rpc_fail and not pnode_img.offline,
3019
               constants.CV_ENODERPC, pnode, "instance %s, connection to"
3020
               " primary node failed", instance)
3021

    
3022
      _ErrorIf(inst_config.admin_state == constants.ADMINST_UP and
3023
               pnode_img.offline,
3024
               constants.CV_EINSTANCEBADNODE, instance,
3025
               "instance is marked as running and lives on offline node %s",
3026
               inst_config.primary_node)
3027

    
3028
      # If the instance is non-redundant we cannot survive losing its primary
3029
      # node, so we are not N+1 compliant. On the other hand we have no disk
3030
      # templates with more than one secondary so that situation is not well
3031
      # supported either.
3032
      # FIXME: does not support file-backed instances
3033
      if not inst_config.secondary_nodes:
3034
        i_non_redundant.append(instance)
3035

    
3036
      _ErrorIf(len(inst_config.secondary_nodes) > 1,
3037
               constants.CV_EINSTANCELAYOUT,
3038
               instance, "instance has multiple secondary nodes: %s",
3039
               utils.CommaJoin(inst_config.secondary_nodes),
3040
               code=self.ETYPE_WARNING)
3041

    
3042
      if inst_config.disk_template in constants.DTS_INT_MIRROR:
3043
        pnode = inst_config.primary_node
3044
        instance_nodes = utils.NiceSort(inst_config.all_nodes)
3045
        instance_groups = {}
3046

    
3047
        for node in instance_nodes:
3048
          instance_groups.setdefault(self.all_node_info[node].group,
3049
                                     []).append(node)
3050

    
3051
        pretty_list = [
3052
          "%s (group %s)" % (utils.CommaJoin(nodes), groupinfo[group].name)
3053
          # Sort so that we always list the primary node first.
3054
          for group, nodes in sorted(instance_groups.items(),
3055
                                     key=lambda (_, nodes): pnode in nodes,
3056
                                     reverse=True)]
3057

    
3058
        self._ErrorIf(len(instance_groups) > 1,
3059
                      constants.CV_EINSTANCESPLITGROUPS,
3060
                      instance, "instance has primary and secondary nodes in"
3061
                      " different groups: %s", utils.CommaJoin(pretty_list),
3062
                      code=self.ETYPE_WARNING)
3063

    
3064
      if not cluster.FillBE(inst_config)[constants.BE_AUTO_BALANCE]:
3065
        i_non_a_balanced.append(instance)
3066

    
3067
      for snode in inst_config.secondary_nodes:
3068
        s_img = node_image[snode]
3069
        _ErrorIf(s_img.rpc_fail and not s_img.offline, constants.CV_ENODERPC,
3070
                 snode, "instance %s, connection to secondary node failed",
3071
                 instance)
3072

    
3073
        if s_img.offline:
3074
          inst_nodes_offline.append(snode)
3075

    
3076
      # warn that the instance lives on offline nodes
3077
      _ErrorIf(inst_nodes_offline, constants.CV_EINSTANCEBADNODE, instance,
3078
               "instance has offline secondary node(s) %s",
3079
               utils.CommaJoin(inst_nodes_offline))
3080
      # ... or ghost/non-vm_capable nodes
3081
      for node in inst_config.all_nodes:
3082
        _ErrorIf(node_image[node].ghost, constants.CV_EINSTANCEBADNODE,
3083
                 instance, "instance lives on ghost node %s", node)
3084
        _ErrorIf(not node_image[node].vm_capable, constants.CV_EINSTANCEBADNODE,
3085
                 instance, "instance lives on non-vm_capable node %s", node)
3086

    
3087
    feedback_fn("* Verifying orphan volumes")
3088
    reserved = utils.FieldSet(*cluster.reserved_lvs)
3089

    
3090
    # We will get spurious "unknown volume" warnings if any node of this group
3091
    # is secondary for an instance whose primary is in another group. To avoid
3092
    # them, we find these instances and add their volumes to node_vol_should.
3093
    for inst in self.all_inst_info.values():
3094
      for secondary in inst.secondary_nodes:
3095
        if (secondary in self.my_node_info
3096
            and inst.name not in self.my_inst_info):
3097
          inst.MapLVsByNode(node_vol_should)
3098
          break
3099

    
3100
    self._VerifyOrphanVolumes(node_vol_should, node_image, reserved)
3101

    
3102
    if constants.VERIFY_NPLUSONE_MEM not in self.op.skip_checks:
3103
      feedback_fn("* Verifying N+1 Memory redundancy")
3104
      self._VerifyNPlusOneMemory(node_image, self.my_inst_info)
3105

    
3106
    feedback_fn("* Other Notes")
3107
    if i_non_redundant:
3108
      feedback_fn("  - NOTICE: %d non-redundant instance(s) found."
3109
                  % len(i_non_redundant))
3110

    
3111
    if i_non_a_balanced:
3112
      feedback_fn("  - NOTICE: %d non-auto-balanced instance(s) found."
3113
                  % len(i_non_a_balanced))
3114

    
3115
    if i_offline:
3116
      feedback_fn("  - NOTICE: %d offline instance(s) found." % i_offline)
3117

    
3118
    if n_offline:
3119
      feedback_fn("  - NOTICE: %d offline node(s) found." % n_offline)
3120

    
3121
    if n_drained:
3122
      feedback_fn("  - NOTICE: %d drained node(s) found." % n_drained)
3123

    
3124
    return not self.bad
3125

    
3126
  def HooksCallBack(self, phase, hooks_results, feedback_fn, lu_result):
3127
    """Analyze the post-hooks' result
3128

3129
    This method analyses the hook result, handles it, and sends some
3130
    nicely-formatted feedback back to the user.
3131

3132
    @param phase: one of L{constants.HOOKS_PHASE_POST} or
3133
        L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase
3134
    @param hooks_results: the results of the multi-node hooks rpc call
3135
    @param feedback_fn: function used send feedback back to the caller
3136
    @param lu_result: previous Exec result
3137
    @return: the new Exec result, based on the previous result
3138
        and hook results
3139

3140
    """
3141
    # We only really run POST phase hooks, only for non-empty groups,
3142
    # and are only interested in their results
3143
    if not self.my_node_names:
3144
      # empty node group
3145
      pass
3146
    elif phase == constants.HOOKS_PHASE_POST:
3147
      # Used to change hooks' output to proper indentation
3148
      feedback_fn("* Hooks Results")
3149
      assert hooks_results, "invalid result from hooks"
3150

    
3151
      for node_name in hooks_results:
3152
        res = hooks_results[node_name]
3153
        msg = res.fail_msg
3154
        test = msg and not res.offline
3155
        self._ErrorIf(test, constants.CV_ENODEHOOKS, node_name,
3156
                      "Communication failure in hooks execution: %s", msg)
3157
        if res.offline or msg:
3158
          # No need to investigate payload if node is offline or gave
3159
          # an error.
3160
          continue
3161
        for script, hkr, output in res.payload:
3162
          test = hkr == constants.HKR_FAIL
3163
          self._ErrorIf(test, constants.CV_ENODEHOOKS, node_name,
3164
                        "Script %s failed, output:", script)
3165
          if test:
3166
            output = self._HOOKS_INDENT_RE.sub("      ", output)
3167
            feedback_fn("%s" % output)
3168
            lu_result = False
3169

    
3170
    return lu_result
3171

    
3172

    
3173
class LUClusterVerifyDisks(NoHooksLU):
3174
  """Verifies the cluster disks status.
3175

3176
  """
3177
  REQ_BGL = False
3178

    
3179
  def ExpandNames(self):
3180
    self.share_locks = _ShareAll()
3181
    self.needed_locks = {
3182
      locking.LEVEL_NODEGROUP: locking.ALL_SET,
3183
      }
3184

    
3185
  def Exec(self, feedback_fn):
3186
    group_names = self.owned_locks(locking.LEVEL_NODEGROUP)
3187

    
3188
    # Submit one instance of L{opcodes.OpGroupVerifyDisks} per node group
3189
    return ResultWithJobs([[opcodes.OpGroupVerifyDisks(group_name=group)]
3190
                           for group in group_names])
3191

    
3192

    
3193
class LUGroupVerifyDisks(NoHooksLU):
3194
  """Verifies the status of all disks in a node group.
3195

3196
  """
3197
  REQ_BGL = False
3198

    
3199
  def ExpandNames(self):
3200
    # Raises errors.OpPrereqError on its own if group can't be found
3201
    self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name)
3202

    
3203
    self.share_locks = _ShareAll()
3204
    self.needed_locks = {
3205
      locking.LEVEL_INSTANCE: [],
3206
      locking.LEVEL_NODEGROUP: [],
3207
      locking.LEVEL_NODE: [],
3208
      }
3209

    
3210
  def DeclareLocks(self, level):
3211
    if level == locking.LEVEL_INSTANCE:
3212
      assert not self.needed_locks[locking.LEVEL_INSTANCE]
3213

    
3214
      # Lock instances optimistically, needs verification once node and group
3215
      # locks have been acquired
3216
      self.needed_locks[locking.LEVEL_INSTANCE] = \
3217
        self.cfg.GetNodeGroupInstances(self.group_uuid)
3218

    
3219
    elif level == locking.LEVEL_NODEGROUP:
3220
      assert not self.needed_locks[locking.LEVEL_NODEGROUP]
3221

    
3222
      self.needed_locks[locking.LEVEL_NODEGROUP] = \
3223
        set([self.group_uuid] +
3224
            # Lock all groups used by instances optimistically; this requires
3225
            # going via the node before it's locked, requiring verification
3226
            # later on
3227
            [group_uuid
3228
             for instance_name in self.owned_locks(locking.LEVEL_INSTANCE)
3229
             for group_uuid in self.cfg.GetInstanceNodeGroups(instance_name)])
3230

    
3231
    elif level == locking.LEVEL_NODE:
3232
      # This will only lock the nodes in the group to be verified which contain
3233
      # actual instances
3234
      self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_APPEND
3235
      self._LockInstancesNodes()
3236

    
3237
      # Lock all nodes in group to be verified
3238
      assert self.group_uuid in self.owned_locks(locking.LEVEL_NODEGROUP)
3239
      member_nodes = self.cfg.GetNodeGroup(self.group_uuid).members
3240
      self.needed_locks[locking.LEVEL_NODE].extend(member_nodes)
3241

    
3242
  def CheckPrereq(self):
3243
    owned_instances = frozenset(self.owned_locks(locking.LEVEL_INSTANCE))
3244
    owned_groups = frozenset(self.owned_locks(locking.LEVEL_NODEGROUP))
3245
    owned_nodes = frozenset(self.owned_locks(locking.LEVEL_NODE))
3246

    
3247
    assert self.group_uuid in owned_groups
3248

    
3249
    # Check if locked instances are still correct
3250
    _CheckNodeGroupInstances(self.cfg, self.group_uuid, owned_instances)
3251

    
3252
    # Get instance information
3253
    self.instances = dict(self.cfg.GetMultiInstanceInfo(owned_instances))
3254

    
3255
    # Check if node groups for locked instances are still correct
3256
    for (instance_name, inst) in self.instances.items():
3257
      assert owned_nodes.issuperset(inst.all_nodes), \
3258
        "Instance %s's nodes changed while we kept the lock" % instance_name
3259

    
3260
      inst_groups = _CheckInstanceNodeGroups(self.cfg, instance_name,
3261
                                             owned_groups)
3262

    
3263
      assert self.group_uuid in inst_groups, \
3264
        "Instance %s has no node in group %s" % (instance_name, self.group_uuid)
3265

    
3266
  def Exec(self, feedback_fn):
3267
    """Verify integrity of cluster disks.
3268

3269
    @rtype: tuple of three items
3270
    @return: a tuple of (dict of node-to-node_error, list of instances
3271
        which need activate-disks, dict of instance: (node, volume) for
3272
        missing volumes
3273

3274
    """
3275
    res_nodes = {}
3276
    res_instances = set()
3277
    res_missing = {}
3278

    
3279
    nv_dict = _MapInstanceDisksToNodes([inst
3280
            for inst in self.instances.values()
3281
            if inst.admin_state == constants.ADMINST_UP])
3282

    
3283
    if nv_dict:
3284
      nodes = utils.NiceSort(set(self.owned_locks(locking.LEVEL_NODE)) &
3285
                             set(self.cfg.GetVmCapableNodeList()))
3286

    
3287
      node_lvs = self.rpc.call_lv_list(nodes, [])
3288

    
3289
      for (node, node_res) in node_lvs.items():
3290
        if node_res.offline:
3291
          continue
3292

    
3293
        msg = node_res.fail_msg
3294
        if msg:
3295
          logging.warning("Error enumerating LVs on node %s: %s", node, msg)
3296
          res_nodes[node] = msg
3297
          continue
3298

    
3299
        for lv_name, (_, _, lv_online) in node_res.payload.items():
3300
          inst = nv_dict.pop((node, lv_name), None)
3301
          if not (lv_online or inst is None):
3302
            res_instances.add(inst)
3303

    
3304
      # any leftover items in nv_dict are missing LVs, let's arrange the data
3305
      # better
3306
      for key, inst in nv_dict.iteritems():
3307
        res_missing.setdefault(inst, []).append(list(key))
3308

    
3309
    return (res_nodes, list(res_instances), res_missing)
3310

    
3311

    
3312
class LUClusterRepairDiskSizes(NoHooksLU):
3313
  """Verifies the cluster disks sizes.
3314

3315
  """
3316
  REQ_BGL = False
3317

    
3318
  def ExpandNames(self):
3319
    if self.op.instances:
3320
      self.wanted_names = _GetWantedInstances(self, self.op.instances)
3321
      self.needed_locks = {
3322
        locking.LEVEL_NODE_RES: [],
3323
        locking.LEVEL_INSTANCE: self.wanted_names,
3324
        }
3325
      self.recalculate_locks[locking.LEVEL_NODE_RES] = constants.LOCKS_REPLACE
3326
    else:
3327
      self.wanted_names = None
3328
      self.needed_locks = {
3329
        locking.LEVEL_NODE_RES: locking.ALL_SET,
3330
        locking.LEVEL_INSTANCE: locking.ALL_SET,
3331
        }
3332
    self.share_locks = {
3333
      locking.LEVEL_NODE_RES: 1,
3334
      locking.LEVEL_INSTANCE: 0,
3335
      }
3336

    
3337
  def DeclareLocks(self, level):
3338
    if level == locking.LEVEL_NODE_RES and self.wanted_names is not None:
3339
      self._LockInstancesNodes(primary_only=True, level=level)
3340

    
3341
  def CheckPrereq(self):
3342
    """Check prerequisites.
3343

3344
    This only checks the optional instance list against the existing names.
3345

3346
    """
3347
    if self.wanted_names is None:
3348
      self.wanted_names = self.owned_locks(locking.LEVEL_INSTANCE)
3349

    
3350
    self.wanted_instances = \
3351
        map(compat.snd, self.cfg.GetMultiInstanceInfo(self.wanted_names))
3352

    
3353
  def _EnsureChildSizes(self, disk):
3354
    """Ensure children of the disk have the needed disk size.
3355

3356
    This is valid mainly for DRBD8 and fixes an issue where the
3357
    children have smaller disk size.
3358

3359
    @param disk: an L{ganeti.objects.Disk} object
3360

3361
    """
3362
    if disk.dev_type == constants.LD_DRBD8:
3363
      assert disk.children, "Empty children for DRBD8?"
3364
      fchild = disk.children[0]
3365
      mismatch = fchild.size < disk.size
3366
      if mismatch:
3367
        self.LogInfo("Child disk has size %d, parent %d, fixing",
3368
                     fchild.size, disk.size)
3369
        fchild.size = disk.size
3370

    
3371
      # and we recurse on this child only, not on the metadev
3372
      return self._EnsureChildSizes(fchild) or mismatch
3373
    else:
3374
      return False
3375

    
3376
  def Exec(self, feedback_fn):
3377
    """Verify the size of cluster disks.
3378

3379
    """
3380
    # TODO: check child disks too
3381
    # TODO: check differences in size between primary/secondary nodes
3382
    per_node_disks = {}
3383
    for instance in self.wanted_instances:
3384
      pnode = instance.primary_node
3385
      if pnode not in per_node_disks:
3386
        per_node_disks[pnode] = []
3387
      for idx, disk in enumerate(instance.disks):
3388
        per_node_disks[pnode].append((instance, idx, disk))
3389

    
3390
    assert not (frozenset(per_node_disks.keys()) -
3391
                self.owned_locks(locking.LEVEL_NODE_RES)), \
3392
      "Not owning correct locks"
3393
    assert not self.owned_locks(locking.LEVEL_NODE)
3394

    
3395
    changed = []
3396
    for node, dskl in per_node_disks.items():
3397
      newl = [v[2].Copy() for v in dskl]
3398
      for dsk in newl:
3399
        self.cfg.SetDiskID(dsk, node)
3400
      result = self.rpc.call_blockdev_getsize(node, newl)
3401
      if result.fail_msg:
3402
        self.LogWarning("Failure in blockdev_getsize call to node"
3403
                        " %s, ignoring", node)
3404
        continue
3405
      if len(result.payload) != len(dskl):
3406
        logging.warning("Invalid result from node %s: len(dksl)=%d,"
3407
                        " result.payload=%s", node, len(dskl), result.payload)
3408
        self.LogWarning("Invalid result from node %s, ignoring node results",
3409
                        node)
3410
        continue
3411
      for ((instance, idx, disk), size) in zip(dskl, result.payload):
3412
        if size is None:
3413
          self.LogWarning("Disk %d of instance %s did not return size"
3414
                          " information, ignoring", idx, instance.name)
3415
          continue
3416
        if not isinstance(size, (int, long)):
3417
          self.LogWarning("Disk %d of instance %s did not return valid"
3418
                          " size information, ignoring", idx, instance.name)
3419
          continue
3420
        size = size >> 20
3421
        if size != disk.size:
3422
          self.LogInfo("Disk %d of instance %s has mismatched size,"
3423
                       " correcting: recorded %d, actual %d", idx,
3424
                       instance.name, disk.size, size)
3425
          disk.size = size
3426
          self.cfg.Update(instance, feedback_fn)
3427
          changed.append((instance.name, idx, size))
3428
        if self._EnsureChildSizes(disk):
3429
          self.cfg.Update(instance, feedback_fn)
3430
          changed.append((instance.name, idx, disk.size))
3431
    return changed
3432

    
3433

    
3434
class LUClusterRename(LogicalUnit):
3435
  """Rename the cluster.
3436

3437
  """
3438
  HPATH = "cluster-rename"
3439
  HTYPE = constants.HTYPE_CLUSTER
3440

    
3441
  def BuildHooksEnv(self):
3442
    """Build hooks env.
3443

3444
    """
3445
    return {
3446
      "OP_TARGET": self.cfg.GetClusterName(),
3447
      "NEW_NAME": self.op.name,
3448
      }
3449

    
3450
  def BuildHooksNodes(self):
3451
    """Build hooks nodes.
3452

3453
    """
3454
    return ([self.cfg.GetMasterNode()], self.cfg.GetNodeList())
3455

    
3456
  def CheckPrereq(self):
3457
    """Verify that the passed name is a valid one.
3458

3459
    """
3460
    hostname = netutils.GetHostname(name=self.op.name,
3461
                                    family=self.cfg.GetPrimaryIPFamily())
3462

    
3463
    new_name = hostname.name
3464
    self.ip = new_ip = hostname.ip
3465
    old_name = self.cfg.GetClusterName()
3466
    old_ip = self.cfg.GetMasterIP()
3467
    if new_name == old_name and new_ip == old_ip:
3468
      raise errors.OpPrereqError("Neither the name nor the IP address of the"
3469
                                 " cluster has changed",
3470
                                 errors.ECODE_INVAL)
3471
    if new_ip != old_ip:
3472
      if netutils.TcpPing(new_ip, constants.DEFAULT_NODED_PORT):
3473
        raise errors.OpPrereqError("The given cluster IP address (%s) is"
3474
                                   " reachable on the network" %
3475
                                   new_ip, errors.ECODE_NOTUNIQUE)
3476

    
3477
    self.op.name = new_name
3478

    
3479
  def Exec(self, feedback_fn):
3480
    """Rename the cluster.
3481

3482
    """
3483
    clustername = self.op.name
3484
    new_ip = self.ip
3485

    
3486
    # shutdown the master IP
3487
    master_params = self.cfg.GetMasterNetworkParameters()
3488
    ems = self.cfg.GetUseExternalMipScript()
3489
    result = self.rpc.call_node_deactivate_master_ip(master_params.name,
3490
                                                     master_params, ems)
3491
    result.Raise("Could not disable the master role")
3492

    
3493
    try:
3494
      cluster = self.cfg.GetClusterInfo()
3495
      cluster.cluster_name = clustername
3496
      cluster.master_ip = new_ip
3497
      self.cfg.Update(cluster, feedback_fn)
3498

    
3499
      # update the known hosts file
3500
      ssh.WriteKnownHostsFile(self.cfg, constants.SSH_KNOWN_HOSTS_FILE)
3501
      node_list = self.cfg.GetOnlineNodeList()
3502
      try:
3503
        node_list.remove(master_params.name)
3504
      except ValueError:
3505
        pass
3506
      _UploadHelper(self, node_list, constants.SSH_KNOWN_HOSTS_FILE)
3507
    finally:
3508
      master_params.ip = new_ip
3509
      result = self.rpc.call_node_activate_master_ip(master_params.name,
3510
                                                     master_params, ems)
3511
      msg = result.fail_msg
3512
      if msg:
3513
        self.LogWarning("Could not re-enable the master role on"
3514
                        " the master, please restart manually: %s", msg)
3515

    
3516
    return clustername
3517

    
3518

    
3519
def _ValidateNetmask(cfg, netmask):
3520
  """Checks if a netmask is valid.
3521

3522
  @type cfg: L{config.ConfigWriter}
3523
  @param cfg: The cluster configuration
3524
  @type netmask: int
3525
  @param netmask: the netmask to be verified
3526
  @raise errors.OpPrereqError: if the validation fails
3527

3528
  """
3529
  ip_family = cfg.GetPrimaryIPFamily()
3530
  try:
3531
    ipcls = netutils.IPAddress.GetClassFromIpFamily(ip_family)
3532
  except errors.ProgrammerError:
3533
    raise errors.OpPrereqError("Invalid primary ip family: %s." %
3534
                               ip_family)
3535
  if not ipcls.ValidateNetmask(netmask):
3536
    raise errors.OpPrereqError("CIDR netmask (%s) not valid" %
3537
                                (netmask))
3538

    
3539

    
3540
class LUClusterSetParams(LogicalUnit):
3541
  """Change the parameters of the cluster.
3542

3543
  """
3544
  HPATH = "cluster-modify"
3545
  HTYPE = constants.HTYPE_CLUSTER
3546
  REQ_BGL = False
3547

    
3548
  def CheckArguments(self):
3549
    """Check parameters
3550

3551
    """
3552
    if self.op.uid_pool:
3553
      uidpool.CheckUidPool(self.op.uid_pool)
3554

    
3555
    if self.op.add_uids:
3556
      uidpool.CheckUidPool(self.op.add_uids)
3557

    
3558
    if self.op.remove_uids:
3559
      uidpool.CheckUidPool(self.op.remove_uids)
3560

    
3561
    if self.op.master_netmask is not None:
3562
      _ValidateNetmask(self.cfg, self.op.master_netmask)
3563

    
3564
    if self.op.diskparams:
3565
      for dt_params in self.op.diskparams.values():
3566
        utils.ForceDictType(dt_params, constants.DISK_DT_TYPES)
3567

    
3568
  def ExpandNames(self):
3569
    # FIXME: in the future maybe other cluster params won't require checking on
3570
    # all nodes to be modified.
3571
    self.needed_locks = {
3572
      locking.LEVEL_NODE: locking.ALL_SET,
3573
    }
3574
    self.share_locks[locking.LEVEL_NODE] = 1
3575

    
3576
  def BuildHooksEnv(self):
3577
    """Build hooks env.
3578

3579
    """
3580
    return {
3581
      "OP_TARGET": self.cfg.GetClusterName(),
3582
      "NEW_VG_NAME": self.op.vg_name,
3583
      }
3584

    
3585
  def BuildHooksNodes(self):
3586
    """Build hooks nodes.
3587

3588
    """
3589
    mn = self.cfg.GetMasterNode()
3590
    return ([mn], [mn])
3591

    
3592
  def CheckPrereq(self):
3593
    """Check prerequisites.
3594

3595
    This checks whether the given params don't conflict and
3596
    if the given volume group is valid.
3597

3598
    """
3599
    if self.op.vg_name is not None and not self.op.vg_name:
3600
      if self.cfg.HasAnyDiskOfType(constants.LD_LV):
3601
        raise errors.OpPrereqError("Cannot disable lvm storage while lvm-based"
3602
                                   " instances exist", errors.ECODE_INVAL)
3603

    
3604
    if self.op.drbd_helper is not None and not self.op.drbd_helper:
3605
      if self.cfg.HasAnyDiskOfType(constants.LD_DRBD8):
3606
        raise errors.OpPrereqError("Cannot disable drbd helper while"
3607
                                   " drbd-based instances exist",
3608
                                   errors.ECODE_INVAL)
3609

    
3610
    node_list = self.owned_locks(locking.LEVEL_NODE)
3611

    
3612
    # if vg_name not None, checks given volume group on all nodes
3613
    if self.op.vg_name:
3614
      vglist = self.rpc.call_vg_list(node_list)
3615
      for node in node_list:
3616
        msg = vglist[node].fail_msg
3617
        if msg:
3618
          # ignoring down node
3619
          self.LogWarning("Error while gathering data on node %s"
3620
                          " (ignoring node): %s", node, msg)
3621
          continue
3622
        vgstatus = utils.CheckVolumeGroupSize(vglist[node].payload,
3623
                                              self.op.vg_name,
3624
                                              constants.MIN_VG_SIZE)
3625
        if vgstatus:
3626
          raise errors.OpPrereqError("Error on node '%s': %s" %
3627
                                     (node, vgstatus), errors.ECODE_ENVIRON)
3628

    
3629
    if self.op.drbd_helper:
3630
      # checks given drbd helper on all nodes
3631
      helpers = self.rpc.call_drbd_helper(node_list)
3632
      for (node, ninfo) in self.cfg.GetMultiNodeInfo(node_list):
3633
        if ninfo.offline:
3634
          self.LogInfo("Not checking drbd helper on offline node %s", node)
3635
          continue
3636
        msg = helpers[node].fail_msg
3637
        if msg:
3638
          raise errors.OpPrereqError("Error checking drbd helper on node"
3639
                                     " '%s': %s" % (node, msg),
3640
                                     errors.ECODE_ENVIRON)
3641
        node_helper = helpers[node].payload
3642
        if node_helper != self.op.drbd_helper:
3643
          raise errors.OpPrereqError("Error on node '%s': drbd helper is %s" %
3644
                                     (node, node_helper), errors.ECODE_ENVIRON)
3645

    
3646
    self.cluster = cluster = self.cfg.GetClusterInfo()
3647
    # validate params changes
3648
    if self.op.beparams:
3649
      objects.UpgradeBeParams(self.op.beparams)
3650
      utils.ForceDictType(self.op.beparams, constants.BES_PARAMETER_TYPES)
3651
      self.new_beparams = cluster.SimpleFillBE(self.op.beparams)
3652

    
3653
    if self.op.ndparams:
3654
      utils.ForceDictType(self.op.ndparams, constants.NDS_PARAMETER_TYPES)
3655
      self.new_ndparams = cluster.SimpleFillND(self.op.ndparams)
3656

    
3657
      # TODO: we need a more general way to handle resetting
3658
      # cluster-level parameters to default values
3659
      if self.new_ndparams["oob_program"] == "":
3660
        self.new_ndparams["oob_program"] = \
3661
            constants.NDC_DEFAULTS[constants.ND_OOB_PROGRAM]
3662

    
3663
    if self.op.nicparams:
3664
      utils.ForceDictType(self.op.nicparams, constants.NICS_PARAMETER_TYPES)
3665
      self.new_nicparams = cluster.SimpleFillNIC(self.op.nicparams)
3666
      objects.NIC.CheckParameterSyntax(self.new_nicparams)
3667
      nic_errors = []
3668

    
3669
      # check all instances for consistency
3670
      for instance in self.cfg.GetAllInstancesInfo().values():
3671
        for nic_idx, nic in enumerate(instance.nics):
3672
          params_copy = copy.deepcopy(nic.nicparams)
3673
          params_filled = objects.FillDict(self.new_nicparams, params_copy)
3674

    
3675
          # check parameter syntax
3676
          try:
3677
            objects.NIC.CheckParameterSyntax(params_filled)
3678
          except errors.ConfigurationError, err:
3679
            nic_errors.append("Instance %s, nic/%d: %s" %
3680
                              (instance.name, nic_idx, err))
3681

    
3682
          # if we're moving instances to routed, check that they have an ip
3683
          target_mode = params_filled[constants.NIC_MODE]
3684
          if target_mode == constants.NIC_MODE_ROUTED and not nic.ip:
3685
            nic_errors.append("Instance %s, nic/%d: routed NIC with no ip"
3686
                              " address" % (instance.name, nic_idx))
3687
      if nic_errors:
3688
        raise errors.OpPrereqError("Cannot apply the change, errors:\n%s" %
3689
                                   "\n".join(nic_errors))
3690

    
3691
    # hypervisor list/parameters
3692
    self.new_hvparams = new_hvp = objects.FillDict(cluster.hvparams, {})
3693
    if self.op.hvparams:
3694
      for hv_name, hv_dict in self.op.hvparams.items():
3695
        if hv_name not in self.new_hvparams:
3696
          self.new_hvparams[hv_name] = hv_dict
3697
        else:
3698
          self.new_hvparams[hv_name].update(hv_dict)
3699

    
3700
    # disk template parameters
3701
    self.new_diskparams = objects.FillDict(cluster.diskparams, {})
3702
    if self.op.diskparams:
3703
      for dt_name, dt_params in self.op.diskparams.items():
3704
        if dt_name not in self.op.diskparams:
3705
          self.new_diskparams[dt_name] = dt_params
3706
        else:
3707
          self.new_diskparams[dt_name].update(dt_params)
3708

    
3709
    # os hypervisor parameters
3710
    self.new_os_hvp = objects.FillDict(cluster.os_hvp, {})
3711
    if self.op.os_hvp:
3712
      for os_name, hvs in self.op.os_hvp.items():
3713
        if os_name not in self.new_os_hvp:
3714
          self.new_os_hvp[os_name] = hvs
3715
        else:
3716
          for hv_name, hv_dict in hvs.items():
3717
            if hv_name not in self.new_os_hvp[os_name]:
3718
              self.new_os_hvp[os_name][hv_name] = hv_dict
3719
            else:
3720
              self.new_os_hvp[os_name][hv_name].update(hv_dict)
3721

    
3722
    # os parameters
3723
    self.new_osp = objects.FillDict(cluster.osparams, {})
3724
    if self.op.osparams:
3725
      for os_name, osp in self.op.osparams.items():
3726
        if os_name not in self.new_osp:
3727
          self.new_osp[os_name] = {}
3728

    
3729
        self.new_osp[os_name] = _GetUpdatedParams(self.new_osp[os_name], osp,
3730
                                                  use_none=True)
3731

    
3732
        if not self.new_osp[os_name]:
3733
          # we removed all parameters
3734
          del self.new_osp[os_name]
3735
        else:
3736
          # check the parameter validity (remote check)
3737
          _CheckOSParams(self, False, [self.cfg.GetMasterNode()],
3738
                         os_name, self.new_osp[os_name])
3739

    
3740
    # changes to the hypervisor list
3741
    if self.op.enabled_hypervisors is not None:
3742
      self.hv_list = self.op.enabled_hypervisors
3743
      for hv in self.hv_list:
3744
        # if the hypervisor doesn't already exist in the cluster
3745
        # hvparams, we initialize it to empty, and then (in both
3746
        # cases) we make sure to fill the defaults, as we might not
3747
        # have a complete defaults list if the hypervisor wasn't
3748
        # enabled before
3749
        if hv not in new_hvp:
3750
          new_hvp[hv] = {}
3751
        new_hvp[hv] = objects.FillDict(constants.HVC_DEFAULTS[hv], new_hvp[hv])
3752
        utils.ForceDictType(new_hvp[hv], constants.HVS_PARAMETER_TYPES)
3753
    else:
3754
      self.hv_list = cluster.enabled_hypervisors
3755

    
3756
    if self.op.hvparams or self.op.enabled_hypervisors is not None:
3757
      # either the enabled list has changed, or the parameters have, validate
3758
      for hv_name, hv_params in self.new_hvparams.items():
3759
        if ((self.op.hvparams and hv_name in self.op.hvparams) or
3760
            (self.op.enabled_hypervisors and
3761
             hv_name in self.op.enabled_hypervisors)):
3762
          # either this is a new hypervisor, or its parameters have changed
3763
          hv_class = hypervisor.GetHypervisor(hv_name)
3764
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3765
          hv_class.CheckParameterSyntax(hv_params)
3766
          _CheckHVParams(self, node_list, hv_name, hv_params)
3767

    
3768
    if self.op.os_hvp:
3769
      # no need to check any newly-enabled hypervisors, since the
3770
      # defaults have already been checked in the above code-block
3771
      for os_name, os_hvp in self.new_os_hvp.items():
3772
        for hv_name, hv_params in os_hvp.items():
3773
          utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
3774
          # we need to fill in the new os_hvp on top of the actual hv_p
3775
          cluster_defaults = self.new_hvparams.get(hv_name, {})
3776
          new_osp = objects.FillDict(cluster_defaults, hv_params)
3777
          hv_class = hypervisor.GetHypervisor(hv_name)
3778
          hv_class.CheckParameterSyntax(new_osp)
3779
          _CheckHVParams(self, node_list, hv_name, new_osp)
3780

    
3781
    if self.op.default_iallocator:
3782
      alloc_script = utils.FindFile(self.op.default_iallocator,
3783
                                    constants.IALLOCATOR_SEARCH_PATH,
3784
                                    os.path.isfile)
3785
      if alloc_script is None:
3786
        raise errors.OpPrereqError("Invalid default iallocator script '%s'"
3787
                                   " specified" % self.op.default_iallocator,
3788
                                   errors.ECODE_INVAL)
3789

    
3790
  def Exec(self, feedback_fn):
3791
    """Change the parameters of the cluster.
3792

3793
    """
3794
    if self.op.vg_name is not None:
3795
      new_volume = self.op.vg_name
3796
      if not new_volume:
3797
        new_volume = None
3798
      if new_volume != self.cfg.GetVGName():
3799
        self.cfg.SetVGName(new_volume)
3800
      else:
3801
        feedback_fn("Cluster LVM configuration already in desired"
3802
                    " state, not changing")
3803
    if self.op.drbd_helper is not None:
3804
      new_helper = self.op.drbd_helper
3805
      if not new_helper:
3806
        new_helper = None
3807
      if new_helper != self.cfg.GetDRBDHelper():
3808
        self.cfg.SetDRBDHelper(new_helper)
3809
      else:
3810
        feedback_fn("Cluster DRBD helper already in desired state,"
3811
                    " not changing")
3812
    if self.op.hvparams:
3813
      self.cluster.hvparams = self.new_hvparams
3814
    if self.op.os_hvp:
3815
      self.cluster.os_hvp = self.new_os_hvp
3816
    if self.op.enabled_hypervisors is not None:
3817
      self.cluster.hvparams = self.new_hvparams
3818
      self.cluster.enabled_hypervisors = self.op.enabled_hypervisors
3819
    if self.op.beparams:
3820
      self.cluster.beparams[constants.PP_DEFAULT] = self.new_beparams
3821
    if self.op.nicparams:
3822
      self.cluster.nicparams[constants.PP_DEFAULT] = self.new_nicparams
3823
    if self.op.osparams:
3824
      self.cluster.osparams = self.new_osp
3825
    if self.op.ndparams:
3826
      self.cluster.ndparams = self.new_ndparams
3827
    if self.op.diskparams:
3828
      self.cluster.diskparams = self.new_diskparams
3829

    
3830
    if self.op.candidate_pool_size is not None:
3831
      self.cluster.candidate_pool_size = self.op.candidate_pool_size
3832
      # we need to update the pool size here, otherwise the save will fail
3833
      _AdjustCandidatePool(self, [])
3834

    
3835
    if self.op.maintain_node_health is not None:
3836
      if self.op.maintain_node_health and not constants.ENABLE_CONFD:
3837
        feedback_fn("Note: CONFD was disabled at build time, node health"
3838
                    " maintenance is not useful (still enabling it)")
3839
      self.cluster.maintain_node_health = self.op.maintain_node_health
3840

    
3841
    if self.op.prealloc_wipe_disks is not None:
3842
      self.cluster.prealloc_wipe_disks = self.op.prealloc_wipe_disks
3843

    
3844
    if self.op.add_uids is not None:
3845
      uidpool.AddToUidPool(self.cluster.uid_pool, self.op.add_uids)
3846

    
3847
    if self.op.remove_uids is not None:
3848
      uidpool.RemoveFromUidPool(self.cluster.uid_pool, self.op.remove_uids)
3849

    
3850
    if self.op.uid_pool is not None:
3851
      self.cluster.uid_pool = self.op.uid_pool
3852

    
3853
    if self.op.default_iallocator is not None:
3854
      self.cluster.default_iallocator = self.op.default_iallocator
3855

    
3856
    if self.op.reserved_lvs is not None:
3857
      self.cluster.reserved_lvs = self.op.reserved_lvs
3858

    
3859
    if self.op.use_external_mip_script is not None:
3860
      self.cluster.use_external_mip_script = self.op.use_external_mip_script
3861

    
3862
    def helper_os(aname, mods, desc):
3863
      desc += " OS list"
3864
      lst = getattr(self.cluster, aname)
3865
      for key, val in mods:
3866
        if key == constants.DDM_ADD:
3867
          if val in lst:
3868
            feedback_fn("OS %s already in %s, ignoring" % (val, desc))
3869
          else:
3870
            lst.append(val)
3871
        elif key == constants.DDM_REMOVE:
3872
          if val in lst:
3873
            lst.remove(val)
3874
          else:
3875
            feedback_fn("OS %s not found in %s, ignoring" % (val, desc))
3876
        else:
3877
          raise errors.ProgrammerError("Invalid modification '%s'" % key)
3878

    
3879
    if self.op.hidden_os:
3880
      helper_os("hidden_os", self.op.hidden_os, "hidden")
3881

    
3882
    if self.op.blacklisted_os:
3883
      helper_os("blacklisted_os", self.op.blacklisted_os, "blacklisted")
3884

    
3885
    if self.op.master_netdev:
3886
      master_params = self.cfg.GetMasterNetworkParameters()
3887
      ems = self.cfg.GetUseExternalMipScript()
3888
      feedback_fn("Shutting down master ip on the current netdev (%s)" %
3889
                  self.cluster.master_netdev)
3890
      result = self.rpc.call_node_deactivate_master_ip(master_params.name,
3891
                                                       master_params, ems)
3892
      result.Raise("Could not disable the master ip")
3893
      feedback_fn("Changing master_netdev from %s to %s" %
3894
                  (master_params.netdev, self.op.master_netdev))
3895
      self.cluster.master_netdev = self.op.master_netdev
3896

    
3897
    if self.op.master_netmask:
3898
      master_params = self.cfg.GetMasterNetworkParameters()
3899
      feedback_fn("Changing master IP netmask to %s" % self.op.master_netmask)
3900
      result = self.rpc.call_node_change_master_netmask(master_params.name,
3901
                                                        master_params.netmask,
3902
                                                        self.op.master_netmask,
3903
                                                        master_params.ip,
3904
                                                        master_params.netdev)
3905
      if result.fail_msg:
3906
        msg = "Could not change the master IP netmask: %s" % result.fail_msg
3907
        feedback_fn(msg)
3908

    
3909
      self.cluster.master_netmask = self.op.master_netmask
3910

    
3911
    self.cfg.Update(self.cluster, feedback_fn)
3912

    
3913
    if self.op.master_netdev:
3914
      master_params = self.cfg.GetMasterNetworkParameters()
3915
      feedback_fn("Starting the master ip on the new master netdev (%s)" %
3916
                  self.op.master_netdev)
3917
      ems = self.cfg.GetUseExternalMipScript()
3918
      result = self.rpc.call_node_activate_master_ip(master_params.name,
3919
                                                     master_params, ems)
3920
      if result.fail_msg:
3921
        self.LogWarning("Could not re-enable the master ip on"
3922
                        " the master, please restart manually: %s",
3923
                        result.fail_msg)
3924

    
3925

    
3926
def _UploadHelper(lu, nodes, fname):
3927
  """Helper for uploading a file and showing warnings.
3928

3929
  """
3930
  if os.path.exists(fname):
3931
    result = lu.rpc.call_upload_file(nodes, fname)
3932
    for to_node, to_result in result.items():
3933
      msg = to_result.fail_msg
3934
      if msg:
3935
        msg = ("Copy of file %s to node %s failed: %s" %
3936
               (fname, to_node, msg))
3937
        lu.proc.LogWarning(msg)
3938

    
3939

    
3940
def _ComputeAncillaryFiles(cluster, redist):
3941
  """Compute files external to Ganeti which need to be consistent.
3942

3943
  @type redist: boolean
3944
  @param redist: Whether to include files which need to be redistributed
3945

3946
  """
3947
  # Compute files for all nodes
3948
  files_all = set([
3949
    constants.SSH_KNOWN_HOSTS_FILE,
3950
    constants.CONFD_HMAC_KEY,
3951
    constants.CLUSTER_DOMAIN_SECRET_FILE,
3952
    constants.SPICE_CERT_FILE,
3953
    constants.SPICE_CACERT_FILE,
3954
    constants.RAPI_USERS_FILE,
3955
    ])
3956

    
3957
  if not redist:
3958
    files_all.update(constants.ALL_CERT_FILES)
3959
    files_all.update(ssconf.SimpleStore().GetFileList())
3960
  else:
3961
    # we need to ship at least the RAPI certificate
3962
    files_all.add(constants.RAPI_CERT_FILE)
3963

    
3964
  if cluster.modify_etc_hosts:
3965
    files_all.add(constants.ETC_HOSTS)
3966

    
3967
  # Files which are optional, these must:
3968
  # - be present in one other category as well
3969
  # - either exist or not exist on all nodes of that category (mc, vm all)
3970
  files_opt = set([
3971
    constants.RAPI_USERS_FILE,
3972
    ])
3973

    
3974
  # Files which should only be on master candidates
3975
  files_mc = set()
3976

    
3977
  if not redist:
3978
    files_mc.add(constants.CLUSTER_CONF_FILE)
3979

    
3980
    # FIXME: this should also be replicated but Ganeti doesn't support files_mc
3981
    # replication
3982
    files_mc.add(constants.DEFAULT_MASTER_SETUP_SCRIPT)
3983

    
3984
  # Files which should only be on VM-capable nodes
3985
  files_vm = set(filename
3986
    for hv_name in cluster.enabled_hypervisors
3987
    for filename in hypervisor.GetHypervisor(hv_name).GetAncillaryFiles()[0])
3988

    
3989
  files_opt |= set(filename
3990
    for hv_name in cluster.enabled_hypervisors
3991
    for filename in hypervisor.GetHypervisor(hv_name).GetAncillaryFiles()[1])
3992

    
3993
  # Filenames in each category must be unique
3994
  all_files_set = files_all | files_mc | files_vm
3995
  assert (len(all_files_set) ==
3996
          sum(map(len, [files_all, files_mc, files_vm]))), \
3997
         "Found file listed in more than one file list"
3998

    
3999
  # Optional files must be present in one other category
4000
  assert all_files_set.issuperset(files_opt), \
4001
         "Optional file not in a different required list"
4002

    
4003
  return (files_all, files_opt, files_mc, files_vm)
4004

    
4005

    
4006
def _RedistributeAncillaryFiles(lu, additional_nodes=None, additional_vm=True):
4007
  """Distribute additional files which are part of the cluster configuration.
4008

4009
  ConfigWriter takes care of distributing the config and ssconf files, but
4010
  there are more files which should be distributed to all nodes. This function
4011
  makes sure those are copied.
4012

4013
  @param lu: calling logical unit
4014
  @param additional_nodes: list of nodes not in the config to distribute to
4015
  @type additional_vm: boolean
4016
  @param additional_vm: whether the additional nodes are vm-capable or not
4017

4018
  """
4019
  # Gather target nodes
4020
  cluster = lu.cfg.GetClusterInfo()
4021
  master_info = lu.cfg.GetNodeInfo(lu.cfg.GetMasterNode())
4022

    
4023
  online_nodes = lu.cfg.GetOnlineNodeList()
4024
  vm_nodes = lu.cfg.GetVmCapableNodeList()
4025

    
4026
  if additional_nodes is not None:
4027
    online_nodes.extend(additional_nodes)
4028
    if additional_vm:
4029
      vm_nodes.extend(additional_nodes)
4030

    
4031
  # Never distribute to master node
4032
  for nodelist in [online_nodes, vm_nodes]:
4033
    if master_info.name in nodelist:
4034
      nodelist.remove(master_info.name)
4035

    
4036
  # Gather file lists
4037
  (files_all, _, files_mc, files_vm) = \
4038
    _ComputeAncillaryFiles(cluster, True)
4039

    
4040
  # Never re-distribute configuration file from here
4041
  assert not (constants.CLUSTER_CONF_FILE in files_all or
4042
              constants.CLUSTER_CONF_FILE in files_vm)
4043
  assert not files_mc, "Master candidates not handled in this function"
4044

    
4045
  filemap = [
4046
    (online_nodes, files_all),
4047
    (vm_nodes, files_vm),
4048
    ]
4049

    
4050
  # Upload the files
4051
  for (node_list, files) in filemap:
4052
    for fname in files:
4053
      _UploadHelper(lu, node_list, fname)
4054

    
4055

    
4056
class LUClusterRedistConf(NoHooksLU):
4057
  """Force the redistribution of cluster configuration.
4058

4059
  This is a very simple LU.
4060

4061
  """
4062
  REQ_BGL = False
4063

    
4064
  def ExpandNames(self):
4065
    self.needed_locks = {
4066
      locking.LEVEL_NODE: locking.ALL_SET,
4067
    }
4068
    self.share_locks[locking.LEVEL_NODE] = 1
4069

    
4070
  def Exec(self, feedback_fn):
4071
    """Redistribute the configuration.
4072

4073
    """
4074
    self.cfg.Update(self.cfg.GetClusterInfo(), feedback_fn)
4075
    _RedistributeAncillaryFiles(self)
4076

    
4077

    
4078
class LUClusterActivateMasterIp(NoHooksLU):
4079
  """Activate the master IP on the master node.
4080

4081
  """
4082
  def Exec(self, feedback_fn):
4083
    """Activate the master IP.
4084

4085
    """
4086
    master_params = self.cfg.GetMasterNetworkParameters()
4087
    ems = self.cfg.GetUseExternalMipScript()
4088
    result = self.rpc.call_node_activate_master_ip(master_params.name,
4089
                                                   master_params, ems)
4090
    result.Raise("Could not activate the master IP")
4091

    
4092

    
4093
class LUClusterDeactivateMasterIp(NoHooksLU):
4094
  """Deactivate the master IP on the master node.
4095

4096
  """
4097
  def Exec(self, feedback_fn):
4098
    """Deactivate the master IP.
4099

4100
    """
4101
    master_params = self.cfg.GetMasterNetworkParameters()
4102
    ems = self.cfg.GetUseExternalMipScript()
4103
    result = self.rpc.call_node_deactivate_master_ip(master_params.name,
4104
                                                     master_params, ems)
4105
    result.Raise("Could not deactivate the master IP")
4106

    
4107

    
4108
def _WaitForSync(lu, instance, disks=None, oneshot=False):
4109
  """Sleep and poll for an instance's disk to sync.
4110

4111
  """
4112
  if not instance.disks or disks is not None and not disks:
4113