Statistics
| Branch: | Tag: | Revision:

root / lib / query.py @ 3b877f08

History | View | Annotate | Download (54.6 kB)

1
#
2
#
3

    
4
# Copyright (C) 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Module for query operations
23

24
How it works:
25

26
  - Add field definitions
27
    - See how L{NODE_FIELDS} is built
28
    - Each field gets:
29
      - Query field definition (L{objects.QueryFieldDefinition}, use
30
        L{_MakeField} for creating), containing:
31
          - Name, must be lowercase and match L{FIELD_NAME_RE}
32
          - Title for tables, must not contain whitespace and match
33
            L{TITLE_RE}
34
          - Value data type, e.g. L{constants.QFT_NUMBER}
35
          - Human-readable description, must not end with punctuation or
36
            contain newlines
37
      - Data request type, see e.g. C{NQ_*}
38
      - OR-ed flags, see C{QFF_*}
39
      - A retrieval function, see L{Query.__init__} for description
40
    - Pass list of fields through L{_PrepareFieldList} for preparation and
41
      checks
42
  - Instantiate L{Query} with prepared field list definition and selected fields
43
  - Call L{Query.RequestedData} to determine what data to collect/compute
44
  - Call L{Query.Query} or L{Query.OldStyleQuery} with collected data and use
45
    result
46
      - Data container must support iteration using C{__iter__}
47
      - Items are passed to retrieval functions and can have any format
48
  - Call L{Query.GetFields} to get list of definitions for selected fields
49

50
@attention: Retrieval functions must be idempotent. They can be called multiple
51
  times, in any order and any number of times.
52

53
"""
54

    
55
import logging
56
import operator
57
import re
58

    
59
from ganeti import constants
60
from ganeti import errors
61
from ganeti import utils
62
from ganeti import compat
63
from ganeti import objects
64
from ganeti import ht
65
from ganeti import qlang
66

    
67
from ganeti.constants import (QFT_UNKNOWN, QFT_TEXT, QFT_BOOL, QFT_NUMBER,
68
                              QFT_UNIT, QFT_TIMESTAMP, QFT_OTHER,
69
                              RS_NORMAL, RS_UNKNOWN, RS_NODATA,
70
                              RS_UNAVAIL, RS_OFFLINE)
71

    
72

    
73
# Constants for requesting data from the caller/data provider. Each property
74
# collected/computed separately by the data provider should have its own to
75
# only collect the requested data and not more.
76

    
77
(NQ_CONFIG,
78
 NQ_INST,
79
 NQ_LIVE,
80
 NQ_GROUP,
81
 NQ_OOB) = range(1, 6)
82

    
83
(IQ_CONFIG,
84
 IQ_LIVE,
85
 IQ_DISKUSAGE,
86
 IQ_CONSOLE) = range(100, 104)
87

    
88
(LQ_MODE,
89
 LQ_OWNER,
90
 LQ_PENDING) = range(10, 13)
91

    
92
(GQ_CONFIG,
93
 GQ_NODE,
94
 GQ_INST) = range(200, 203)
95

    
96
# Query field flags
97
QFF_HOSTNAME = 0x01
98
QFF_IP_ADDRESS = 0x02
99
# Next values: 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x100, 0x200
100
QFF_ALL = (QFF_HOSTNAME | QFF_IP_ADDRESS)
101

    
102
FIELD_NAME_RE = re.compile(r"^[a-z0-9/._]+$")
103
TITLE_RE = re.compile(r"^[^\s]+$")
104
DOC_RE = re.compile(r"^[A-Z].*[^.,?!]$")
105

    
106
#: Verification function for each field type
107
_VERIFY_FN = {
108
  QFT_UNKNOWN: ht.TNone,
109
  QFT_TEXT: ht.TString,
110
  QFT_BOOL: ht.TBool,
111
  QFT_NUMBER: ht.TInt,
112
  QFT_UNIT: ht.TInt,
113
  QFT_TIMESTAMP: ht.TOr(ht.TInt, ht.TFloat),
114
  QFT_OTHER: lambda _: True,
115
  }
116

    
117
# Unique objects for special field statuses
118
_FS_UNKNOWN = object()
119
_FS_NODATA = object()
120
_FS_UNAVAIL = object()
121
_FS_OFFLINE = object()
122

    
123
#: VType to QFT mapping
124
_VTToQFT = {
125
  # TODO: fix validation of empty strings
126
  constants.VTYPE_STRING: QFT_OTHER, # since VTYPE_STRINGs can be empty
127
  constants.VTYPE_MAYBE_STRING: QFT_OTHER,
128
  constants.VTYPE_BOOL: QFT_BOOL,
129
  constants.VTYPE_SIZE: QFT_UNIT,
130
  constants.VTYPE_INT: QFT_NUMBER,
131
  }
132

    
133
_SERIAL_NO_DOC = "%s object serial number, incremented on each modification"
134

    
135

    
136
def _GetUnknownField(ctx, item): # pylint: disable-msg=W0613
137
  """Gets the contents of an unknown field.
138

139
  """
140
  return _FS_UNKNOWN
141

    
142

    
143
def _GetQueryFields(fielddefs, selected):
144
  """Calculates the internal list of selected fields.
145

146
  Unknown fields are returned as L{constants.QFT_UNKNOWN}.
147

148
  @type fielddefs: dict
149
  @param fielddefs: Field definitions
150
  @type selected: list of strings
151
  @param selected: List of selected fields
152

153
  """
154
  result = []
155

    
156
  for name in selected:
157
    try:
158
      fdef = fielddefs[name]
159
    except KeyError:
160
      fdef = (_MakeField(name, name, QFT_UNKNOWN, "Unknown field '%s'" % name),
161
              None, 0, _GetUnknownField)
162

    
163
    assert len(fdef) == 4
164

    
165
    result.append(fdef)
166

    
167
  return result
168

    
169

    
170
def GetAllFields(fielddefs):
171
  """Extract L{objects.QueryFieldDefinition} from field definitions.
172

173
  @rtype: list of L{objects.QueryFieldDefinition}
174

175
  """
176
  return [fdef for (fdef, _, _, _) in fielddefs]
177

    
178

    
179
class _FilterHints:
180
  """Class for filter analytics.
181

182
  When filters are used, the user of the L{Query} class usually doesn't know
183
  exactly which items will be necessary for building the result. It therefore
184
  has to prepare and compute the input data for potentially returning
185
  everything.
186

187
  There are two ways to optimize this. The first, and simpler, is to assign
188
  each field a group of data, so that the caller can determine which
189
  computations are necessary depending on the data groups requested. The list
190
  of referenced groups must also be computed for fields referenced in the
191
  filter.
192

193
  The second is restricting the items based on a primary key. The primary key
194
  is usually a unique name (e.g. a node name). This class extracts all
195
  referenced names from a filter. If it encounters any filter condition which
196
  disallows such a list to be determined (e.g. a non-equality filter), all
197
  names will be requested.
198

199
  The end-effect is that any operation other than L{qlang.OP_OR} and
200
  L{qlang.OP_EQUAL} will make the query more expensive.
201

202
  """
203
  def __init__(self, namefield):
204
    """Initializes this class.
205

206
    @type namefield: string
207
    @param namefield: Field caller is interested in
208

209
    """
210
    self._namefield = namefield
211

    
212
    #: Whether all names need to be requested (e.g. if a non-equality operator
213
    #: has been used)
214
    self._allnames = False
215

    
216
    #: Which names to request
217
    self._names = None
218

    
219
    #: Data kinds referenced by the filter (used by L{Query.RequestedData})
220
    self._datakinds = set()
221

    
222
  def RequestedNames(self):
223
    """Returns all requested values.
224

225
    Returns C{None} if list of values can't be determined (e.g. encountered
226
    non-equality operators).
227

228
    @rtype: list
229

230
    """
231
    if self._allnames or self._names is None:
232
      return None
233

    
234
    return utils.UniqueSequence(self._names)
235

    
236
  def ReferencedData(self):
237
    """Returns all kinds of data referenced by the filter.
238

239
    """
240
    return frozenset(self._datakinds)
241

    
242
  def _NeedAllNames(self):
243
    """Changes internal state to request all names.
244

245
    """
246
    self._allnames = True
247
    self._names = None
248

    
249
  def NoteLogicOp(self, op):
250
    """Called when handling a logic operation.
251

252
    @type op: string
253
    @param op: Operator
254

255
    """
256
    if op != qlang.OP_OR:
257
      self._NeedAllNames()
258

    
259
  def NoteUnaryOp(self, op): # pylint: disable-msg=W0613
260
    """Called when handling an unary operation.
261

262
    @type op: string
263
    @param op: Operator
264

265
    """
266
    self._NeedAllNames()
267

    
268
  def NoteBinaryOp(self, op, datakind, name, value):
269
    """Called when handling a binary operation.
270

271
    @type op: string
272
    @param op: Operator
273
    @type name: string
274
    @param name: Left-hand side of operator (field name)
275
    @param value: Right-hand side of operator
276

277
    """
278
    if datakind is not None:
279
      self._datakinds.add(datakind)
280

    
281
    if self._allnames:
282
      return
283

    
284
    # If any operator other than equality was used, all names need to be
285
    # retrieved
286
    if op == qlang.OP_EQUAL and name == self._namefield:
287
      if self._names is None:
288
        self._names = []
289
      self._names.append(value)
290
    else:
291
      self._NeedAllNames()
292

    
293

    
294
def _WrapLogicOp(op_fn, sentences, ctx, item):
295
  """Wrapper for logic operator functions.
296

297
  """
298
  return op_fn(fn(ctx, item) for fn in sentences)
299

    
300

    
301
def _WrapUnaryOp(op_fn, inner, ctx, item):
302
  """Wrapper for unary operator functions.
303

304
  """
305
  return op_fn(inner(ctx, item))
306

    
307

    
308
def _WrapBinaryOp(op_fn, retrieval_fn, value, ctx, item):
309
  """Wrapper for binary operator functions.
310

311
  """
312
  return op_fn(retrieval_fn(ctx, item), value)
313

    
314

    
315
def _WrapNot(fn, lhs, rhs):
316
  """Negates the result of a wrapped function.
317

318
  """
319
  return not fn(lhs, rhs)
320

    
321

    
322
class _FilterCompilerHelper:
323
  """Converts a query filter to a callable usable for filtering.
324

325
  """
326
  # String statement has no effect, pylint: disable-msg=W0105
327

    
328
  #: How deep filters can be nested
329
  _LEVELS_MAX = 10
330

    
331
  # Unique identifiers for operator groups
332
  (_OPTYPE_LOGIC,
333
   _OPTYPE_UNARY,
334
   _OPTYPE_BINARY) = range(1, 4)
335

    
336
  """Functions for equality checks depending on field flags.
337

338
  List of tuples containing flags and a callable receiving the left- and
339
  right-hand side of the operator. The flags are an OR-ed value of C{QFF_*}
340
  (e.g. L{QFF_HOSTNAME}).
341

342
  Order matters. The first item with flags will be used. Flags are checked
343
  using binary AND.
344

345
  """
346
  _EQUALITY_CHECKS = [
347
    (QFF_HOSTNAME,
348
     lambda lhs, rhs: utils.MatchNameComponent(rhs, [lhs],
349
                                               case_sensitive=False)),
350
    (None, operator.eq),
351
    ]
352

    
353
  """Known operators
354

355
  Operator as key (C{qlang.OP_*}), value a tuple of operator group
356
  (C{_OPTYPE_*}) and a group-specific value:
357

358
    - C{_OPTYPE_LOGIC}: Callable taking any number of arguments; used by
359
      L{_HandleLogicOp}
360
    - C{_OPTYPE_UNARY}: Always C{None}; details handled by L{_HandleUnaryOp}
361
    - C{_OPTYPE_BINARY}: Callable taking exactly two parameters, the left- and
362
      right-hand side of the operator, used by L{_HandleBinaryOp}
363

364
  """
365
  _OPS = {
366
    # Logic operators
367
    qlang.OP_OR: (_OPTYPE_LOGIC, compat.any),
368
    qlang.OP_AND: (_OPTYPE_LOGIC, compat.all),
369

    
370
    # Unary operators
371
    qlang.OP_NOT: (_OPTYPE_UNARY, None),
372
    qlang.OP_TRUE: (_OPTYPE_UNARY, None),
373

    
374
    # Binary operators
375
    qlang.OP_EQUAL: (_OPTYPE_BINARY, _EQUALITY_CHECKS),
376
    qlang.OP_NOT_EQUAL:
377
      (_OPTYPE_BINARY, [(flags, compat.partial(_WrapNot, fn))
378
                        for (flags, fn) in _EQUALITY_CHECKS]),
379
    qlang.OP_GLOB: (_OPTYPE_BINARY, NotImplemented),
380
    qlang.OP_REGEXP: (_OPTYPE_BINARY, NotImplemented),
381
    qlang.OP_CONTAINS: (_OPTYPE_BINARY, [
382
      (None, operator.contains),
383
      ]),
384
    }
385

    
386
  def __init__(self, fields):
387
    """Initializes this class.
388

389
    @param fields: Field definitions (return value of L{_PrepareFieldList})
390

391
    """
392
    self._fields = fields
393
    self._hints = None
394
    self._op_handler = None
395

    
396
  def __call__(self, hints, filter_):
397
    """Converts a query filter into a callable function.
398

399
    @type hints: L{_FilterHints} or None
400
    @param hints: Callbacks doing analysis on filter
401
    @type filter_: list
402
    @param filter_: Filter structure
403
    @rtype: callable
404
    @return: Function receiving context and item as parameters, returning
405
             boolean as to whether item matches filter
406

407
    """
408
    self._op_handler = {
409
      self._OPTYPE_LOGIC:
410
        (self._HandleLogicOp, getattr(hints, "NoteLogicOp", None)),
411
      self._OPTYPE_UNARY:
412
        (self._HandleUnaryOp, getattr(hints, "NoteUnaryOp", None)),
413
      self._OPTYPE_BINARY:
414
        (self._HandleBinaryOp, getattr(hints, "NoteBinaryOp", None)),
415
      }
416

    
417
    try:
418
      filter_fn = self._Compile(filter_, 0)
419
    finally:
420
      self._op_handler = None
421

    
422
    return filter_fn
423

    
424
  def _Compile(self, filter_, level):
425
    """Inner function for converting filters.
426

427
    Calls the correct handler functions for the top-level operator. This
428
    function is called recursively (e.g. for logic operators).
429

430
    """
431
    if not (isinstance(filter_, (list, tuple)) and filter_):
432
      raise errors.ParameterError("Invalid filter on level %s" % level)
433

    
434
    # Limit recursion
435
    if level >= self._LEVELS_MAX:
436
      raise errors.ParameterError("Only up to %s levels are allowed (filter"
437
                                  " nested too deep)" % self._LEVELS_MAX)
438

    
439
    # Create copy to be modified
440
    operands = filter_[:]
441
    op = operands.pop(0)
442

    
443
    try:
444
      (kind, op_data) = self._OPS[op]
445
    except KeyError:
446
      raise errors.ParameterError("Unknown operator '%s'" % op)
447

    
448
    (handler, hints_cb) = self._op_handler[kind]
449

    
450
    return handler(hints_cb, level, op, op_data, operands)
451

    
452
  def _LookupField(self, name):
453
    """Returns a field definition by name.
454

455
    """
456
    try:
457
      return self._fields[name]
458
    except KeyError:
459
      raise errors.ParameterError("Unknown field '%s'" % name)
460

    
461
  def _HandleLogicOp(self, hints_fn, level, op, op_fn, operands):
462
    """Handles logic operators.
463

464
    @type hints_fn: callable
465
    @param hints_fn: Callback doing some analysis on the filter
466
    @type level: integer
467
    @param level: Current depth
468
    @type op: string
469
    @param op: Operator
470
    @type op_fn: callable
471
    @param op_fn: Function implementing operator
472
    @type operands: list
473
    @param operands: List of operands
474

475
    """
476
    if hints_fn:
477
      hints_fn(op)
478

    
479
    return compat.partial(_WrapLogicOp, op_fn,
480
                          [self._Compile(op, level + 1) for op in operands])
481

    
482
  def _HandleUnaryOp(self, hints_fn, level, op, op_fn, operands):
483
    """Handles unary operators.
484

485
    @type hints_fn: callable
486
    @param hints_fn: Callback doing some analysis on the filter
487
    @type level: integer
488
    @param level: Current depth
489
    @type op: string
490
    @param op: Operator
491
    @type op_fn: callable
492
    @param op_fn: Function implementing operator
493
    @type operands: list
494
    @param operands: List of operands
495

496
    """
497
    assert op_fn is None
498

    
499
    if hints_fn:
500
      hints_fn(op)
501

    
502
    if len(operands) != 1:
503
      raise errors.ParameterError("Unary operator '%s' expects exactly one"
504
                                  " operand" % op)
505

    
506
    if op == qlang.OP_TRUE:
507
      (_, _, _, retrieval_fn) = self._LookupField(operands[0])
508

    
509
      op_fn = operator.truth
510
      arg = retrieval_fn
511
    elif op == qlang.OP_NOT:
512
      op_fn = operator.not_
513
      arg = self._Compile(operands[0], level + 1)
514
    else:
515
      raise errors.ProgrammerError("Can't handle operator '%s'" % op)
516

    
517
    return compat.partial(_WrapUnaryOp, op_fn, arg)
518

    
519
  def _HandleBinaryOp(self, hints_fn, level, op, op_data, operands):
520
    """Handles binary operators.
521

522
    @type hints_fn: callable
523
    @param hints_fn: Callback doing some analysis on the filter
524
    @type level: integer
525
    @param level: Current depth
526
    @type op: string
527
    @param op: Operator
528
    @param op_data: Functions implementing operators
529
    @type operands: list
530
    @param operands: List of operands
531

532
    """
533
    # Unused arguments, pylint: disable-msg=W0613
534
    try:
535
      (name, value) = operands
536
    except (ValueError, TypeError):
537
      raise errors.ParameterError("Invalid binary operator, expected exactly"
538
                                  " two operands")
539

    
540
    (fdef, datakind, field_flags, retrieval_fn) = self._LookupField(name)
541

    
542
    assert fdef.kind != QFT_UNKNOWN
543

    
544
    # TODO: Type conversions?
545

    
546
    verify_fn = _VERIFY_FN[fdef.kind]
547
    if not verify_fn(value):
548
      raise errors.ParameterError("Unable to compare field '%s' (type '%s')"
549
                                  " with '%s', expected %s" %
550
                                  (name, fdef.kind, value.__class__.__name__,
551
                                   verify_fn))
552

    
553
    if hints_fn:
554
      hints_fn(op, datakind, name, value)
555

    
556
    for (fn_flags, fn) in op_data:
557
      if fn_flags is None or fn_flags & field_flags:
558
        return compat.partial(_WrapBinaryOp, fn, retrieval_fn, value)
559

    
560
    raise errors.ProgrammerError("Unable to find operator implementation"
561
                                 " (op '%s', flags %s)" % (op, field_flags))
562

    
563

    
564
def _CompileFilter(fields, hints, filter_):
565
  """Converts a query filter into a callable function.
566

567
  See L{_FilterCompilerHelper} for details.
568

569
  @rtype: callable
570

571
  """
572
  return _FilterCompilerHelper(fields)(hints, filter_)
573

    
574

    
575
class Query:
576
  def __init__(self, fieldlist, selected, filter_=None, namefield=None):
577
    """Initializes this class.
578

579
    The field definition is a dictionary with the field's name as a key and a
580
    tuple containing, in order, the field definition object
581
    (L{objects.QueryFieldDefinition}, the data kind to help calling code
582
    collect data and a retrieval function. The retrieval function is called
583
    with two parameters, in order, the data container and the item in container
584
    (see L{Query.Query}).
585

586
    Users of this class can call L{RequestedData} before preparing the data
587
    container to determine what data is needed.
588

589
    @type fieldlist: dictionary
590
    @param fieldlist: Field definitions
591
    @type selected: list of strings
592
    @param selected: List of selected fields
593

594
    """
595
    assert namefield is None or namefield in fieldlist
596

    
597
    self._fields = _GetQueryFields(fieldlist, selected)
598

    
599
    self._filter_fn = None
600
    self._requested_names = None
601
    self._filter_datakinds = frozenset()
602

    
603
    if filter_ is not None:
604
      # Collect requested names if wanted
605
      if namefield:
606
        hints = _FilterHints(namefield)
607
      else:
608
        hints = None
609

    
610
      # Build filter function
611
      self._filter_fn = _CompileFilter(fieldlist, hints, filter_)
612
      if hints:
613
        self._requested_names = hints.RequestedNames()
614
        self._filter_datakinds = hints.ReferencedData()
615

    
616
    if namefield is None:
617
      self._name_fn = None
618
    else:
619
      (_, _, _, self._name_fn) = fieldlist[namefield]
620

    
621
  def RequestedNames(self):
622
    """Returns all names referenced in the filter.
623

624
    If there is no filter or operators are preventing determining the exact
625
    names, C{None} is returned.
626

627
    """
628
    return self._requested_names
629

    
630
  def RequestedData(self):
631
    """Gets requested kinds of data.
632

633
    @rtype: frozenset
634

635
    """
636
    return (self._filter_datakinds |
637
            frozenset(datakind for (_, datakind, _, _) in self._fields
638
                      if datakind is not None))
639

    
640
  def GetFields(self):
641
    """Returns the list of fields for this query.
642

643
    Includes unknown fields.
644

645
    @rtype: List of L{objects.QueryFieldDefinition}
646

647
    """
648
    return GetAllFields(self._fields)
649

    
650
  def Query(self, ctx, sort_by_name=True):
651
    """Execute a query.
652

653
    @param ctx: Data container passed to field retrieval functions, must
654
      support iteration using C{__iter__}
655
    @type sort_by_name: boolean
656
    @param sort_by_name: Whether to sort by name or keep the input data's
657
      ordering
658

659
    """
660
    sort = (self._name_fn and sort_by_name)
661

    
662
    result = []
663

    
664
    for idx, item in enumerate(ctx):
665
      if not (self._filter_fn is None or self._filter_fn(ctx, item)):
666
        continue
667

    
668
      row = [_ProcessResult(fn(ctx, item)) for (_, _, _, fn) in self._fields]
669

    
670
      # Verify result
671
      if __debug__:
672
        _VerifyResultRow(self._fields, row)
673

    
674
      if sort:
675
        (status, name) = _ProcessResult(self._name_fn(ctx, item))
676
        assert status == constants.RS_NORMAL
677
        # TODO: Are there cases where we wouldn't want to use NiceSort?
678
        result.append((utils.NiceSortKey(name), idx, row))
679
      else:
680
        result.append(row)
681

    
682
    if not sort:
683
      return result
684

    
685
    # TODO: Would "heapq" be more efficient than sorting?
686

    
687
    # Sorting in-place instead of using "sorted()"
688
    result.sort()
689

    
690
    assert not result or (len(result[0]) == 3 and len(result[-1]) == 3)
691

    
692
    return map(operator.itemgetter(2), result)
693

    
694
  def OldStyleQuery(self, ctx, sort_by_name=True):
695
    """Query with "old" query result format.
696

697
    See L{Query.Query} for arguments.
698

699
    """
700
    unknown = set(fdef.name for (fdef, _, _, _) in self._fields
701
                  if fdef.kind == QFT_UNKNOWN)
702
    if unknown:
703
      raise errors.OpPrereqError("Unknown output fields selected: %s" %
704
                                 (utils.CommaJoin(unknown), ),
705
                                 errors.ECODE_INVAL)
706

    
707
    return [[value for (_, value) in row]
708
            for row in self.Query(ctx, sort_by_name=sort_by_name)]
709

    
710

    
711
def _ProcessResult(value):
712
  """Converts result values into externally-visible ones.
713

714
  """
715
  if value is _FS_UNKNOWN:
716
    return (RS_UNKNOWN, None)
717
  elif value is _FS_NODATA:
718
    return (RS_NODATA, None)
719
  elif value is _FS_UNAVAIL:
720
    return (RS_UNAVAIL, None)
721
  elif value is _FS_OFFLINE:
722
    return (RS_OFFLINE, None)
723
  else:
724
    return (RS_NORMAL, value)
725

    
726

    
727
def _VerifyResultRow(fields, row):
728
  """Verifies the contents of a query result row.
729

730
  @type fields: list
731
  @param fields: Field definitions for result
732
  @type row: list of tuples
733
  @param row: Row data
734

735
  """
736
  assert len(row) == len(fields)
737
  errs = []
738
  for ((status, value), (fdef, _, _, _)) in zip(row, fields):
739
    if status == RS_NORMAL:
740
      if not _VERIFY_FN[fdef.kind](value):
741
        errs.append("normal field %s fails validation (value is %s)" %
742
                    (fdef.name, value))
743
    elif value is not None:
744
      errs.append("abnormal field %s has a non-None value" % fdef.name)
745
  assert not errs, ("Failed validation: %s in row %s" %
746
                    (utils.CommaJoin(errors), row))
747

    
748

    
749
def _PrepareFieldList(fields, aliases):
750
  """Prepares field list for use by L{Query}.
751

752
  Converts the list to a dictionary and does some verification.
753

754
  @type fields: list of tuples; (L{objects.QueryFieldDefinition}, data
755
      kind, retrieval function)
756
  @param fields: List of fields, see L{Query.__init__} for a better
757
      description
758
  @type aliases: list of tuples; (alias, target)
759
  @param aliases: list of tuples containing aliases; for each
760
      alias/target pair, a duplicate will be created in the field list
761
  @rtype: dict
762
  @return: Field dictionary for L{Query}
763

764
  """
765
  if __debug__:
766
    duplicates = utils.FindDuplicates(fdef.title.lower()
767
                                      for (fdef, _, _, _) in fields)
768
    assert not duplicates, "Duplicate title(s) found: %r" % duplicates
769

    
770
  result = {}
771

    
772
  for field in fields:
773
    (fdef, _, flags, fn) = field
774

    
775
    assert fdef.name and fdef.title, "Name and title are required"
776
    assert FIELD_NAME_RE.match(fdef.name)
777
    assert TITLE_RE.match(fdef.title)
778
    assert (DOC_RE.match(fdef.doc) and len(fdef.doc.splitlines()) == 1 and
779
            fdef.doc.strip() == fdef.doc), \
780
           "Invalid description for field '%s'" % fdef.name
781
    assert callable(fn)
782
    assert fdef.name not in result, \
783
           "Duplicate field name '%s' found" % fdef.name
784
    assert (flags & ~QFF_ALL) == 0, "Unknown flags for field '%s'" % fdef.name
785

    
786
    result[fdef.name] = field
787

    
788
  for alias, target in aliases:
789
    assert alias not in result, "Alias %s overrides an existing field" % alias
790
    assert target in result, "Missing target %s for alias %s" % (target, alias)
791
    (fdef, k, flags, fn) = result[target]
792
    fdef = fdef.Copy()
793
    fdef.name = alias
794
    result[alias] = (fdef, k, flags, fn)
795

    
796
  assert len(result) == len(fields) + len(aliases)
797
  assert compat.all(name == fdef.name
798
                    for (name, (fdef, _, _, _)) in result.items())
799

    
800
  return result
801

    
802

    
803
def GetQueryResponse(query, ctx, sort_by_name=True):
804
  """Prepares the response for a query.
805

806
  @type query: L{Query}
807
  @param ctx: Data container, see L{Query.Query}
808
  @type sort_by_name: boolean
809
  @param sort_by_name: Whether to sort by name or keep the input data's
810
    ordering
811

812
  """
813
  return objects.QueryResponse(data=query.Query(ctx, sort_by_name=sort_by_name),
814
                               fields=query.GetFields()).ToDict()
815

    
816

    
817
def QueryFields(fielddefs, selected):
818
  """Returns list of available fields.
819

820
  @type fielddefs: dict
821
  @param fielddefs: Field definitions
822
  @type selected: list of strings
823
  @param selected: List of selected fields
824
  @return: List of L{objects.QueryFieldDefinition}
825

826
  """
827
  if selected is None:
828
    # Client requests all fields, sort by name
829
    fdefs = utils.NiceSort(GetAllFields(fielddefs.values()),
830
                           key=operator.attrgetter("name"))
831
  else:
832
    # Keep order as requested by client
833
    fdefs = Query(fielddefs, selected).GetFields()
834

    
835
  return objects.QueryFieldsResponse(fields=fdefs).ToDict()
836

    
837

    
838
def _MakeField(name, title, kind, doc):
839
  """Wrapper for creating L{objects.QueryFieldDefinition} instances.
840

841
  @param name: Field name as a regular expression
842
  @param title: Human-readable title
843
  @param kind: Field type
844
  @param doc: Human-readable description
845

846
  """
847
  return objects.QueryFieldDefinition(name=name, title=title, kind=kind,
848
                                      doc=doc)
849

    
850

    
851
def _GetNodeRole(node, master_name):
852
  """Determine node role.
853

854
  @type node: L{objects.Node}
855
  @param node: Node object
856
  @type master_name: string
857
  @param master_name: Master node name
858

859
  """
860
  if node.name == master_name:
861
    return constants.NR_MASTER
862
  elif node.master_candidate:
863
    return constants.NR_MCANDIDATE
864
  elif node.drained:
865
    return constants.NR_DRAINED
866
  elif node.offline:
867
    return constants.NR_OFFLINE
868
  else:
869
    return constants.NR_REGULAR
870

    
871

    
872
def _GetItemAttr(attr):
873
  """Returns a field function to return an attribute of the item.
874

875
  @param attr: Attribute name
876

877
  """
878
  getter = operator.attrgetter(attr)
879
  return lambda _, item: getter(item)
880

    
881

    
882
def _GetItemTimestamp(getter):
883
  """Returns function for getting timestamp of item.
884

885
  @type getter: callable
886
  @param getter: Function to retrieve timestamp attribute
887

888
  """
889
  def fn(_, item):
890
    """Returns a timestamp of item.
891

892
    """
893
    timestamp = getter(item)
894
    if timestamp is None:
895
      # Old configs might not have all timestamps
896
      return _FS_UNAVAIL
897
    else:
898
      return timestamp
899

    
900
  return fn
901

    
902

    
903
def _GetItemTimestampFields(datatype):
904
  """Returns common timestamp fields.
905

906
  @param datatype: Field data type for use by L{Query.RequestedData}
907

908
  """
909
  return [
910
    (_MakeField("ctime", "CTime", QFT_TIMESTAMP, "Creation timestamp"),
911
     datatype, 0, _GetItemTimestamp(operator.attrgetter("ctime"))),
912
    (_MakeField("mtime", "MTime", QFT_TIMESTAMP, "Modification timestamp"),
913
     datatype, 0, _GetItemTimestamp(operator.attrgetter("mtime"))),
914
    ]
915

    
916

    
917
class NodeQueryData:
918
  """Data container for node data queries.
919

920
  """
921
  def __init__(self, nodes, live_data, master_name, node_to_primary,
922
               node_to_secondary, groups, oob_support, cluster):
923
    """Initializes this class.
924

925
    """
926
    self.nodes = nodes
927
    self.live_data = live_data
928
    self.master_name = master_name
929
    self.node_to_primary = node_to_primary
930
    self.node_to_secondary = node_to_secondary
931
    self.groups = groups
932
    self.oob_support = oob_support
933
    self.cluster = cluster
934

    
935
    # Used for individual rows
936
    self.curlive_data = None
937

    
938
  def __iter__(self):
939
    """Iterate over all nodes.
940

941
    This function has side-effects and only one instance of the resulting
942
    generator should be used at a time.
943

944
    """
945
    for node in self.nodes:
946
      if self.live_data:
947
        self.curlive_data = self.live_data.get(node.name, None)
948
      else:
949
        self.curlive_data = None
950
      yield node
951

    
952

    
953
#: Fields that are direct attributes of an L{objects.Node} object
954
_NODE_SIMPLE_FIELDS = {
955
  "drained": ("Drained", QFT_BOOL, 0, "Whether node is drained"),
956
  "master_candidate": ("MasterC", QFT_BOOL, 0,
957
                       "Whether node is a master candidate"),
958
  "master_capable": ("MasterCapable", QFT_BOOL, 0,
959
                     "Whether node can become a master candidate"),
960
  "name": ("Node", QFT_TEXT, QFF_HOSTNAME, "Node name"),
961
  "offline": ("Offline", QFT_BOOL, 0, "Whether node is marked offline"),
962
  "serial_no": ("SerialNo", QFT_NUMBER, 0, _SERIAL_NO_DOC % "Node"),
963
  "uuid": ("UUID", QFT_TEXT, 0, "Node UUID"),
964
  "vm_capable": ("VMCapable", QFT_BOOL, 0, "Whether node can host instances"),
965
  }
966

    
967

    
968
#: Fields requiring talking to the node
969
# Note that none of these are available for non-vm_capable nodes
970
_NODE_LIVE_FIELDS = {
971
  "bootid": ("BootID", QFT_TEXT, "bootid",
972
             "Random UUID renewed for each system reboot, can be used"
973
             " for detecting reboots by tracking changes"),
974
  "cnodes": ("CNodes", QFT_NUMBER, "cpu_nodes",
975
             "Number of NUMA domains on node (if exported by hypervisor)"),
976
  "csockets": ("CSockets", QFT_NUMBER, "cpu_sockets",
977
               "Number of physical CPU sockets (if exported by hypervisor)"),
978
  "ctotal": ("CTotal", QFT_NUMBER, "cpu_total", "Number of logical processors"),
979
  "dfree": ("DFree", QFT_UNIT, "vg_free",
980
            "Available disk space in volume group"),
981
  "dtotal": ("DTotal", QFT_UNIT, "vg_size",
982
             "Total disk space in volume group used for instance disk"
983
             " allocation"),
984
  "mfree": ("MFree", QFT_UNIT, "memory_free",
985
            "Memory available for instance allocations"),
986
  "mnode": ("MNode", QFT_UNIT, "memory_dom0",
987
            "Amount of memory used by node (dom0 for Xen)"),
988
  "mtotal": ("MTotal", QFT_UNIT, "memory_total",
989
             "Total amount of memory of physical machine"),
990
  }
991

    
992

    
993
def _GetGroup(cb):
994
  """Build function for calling another function with an node group.
995

996
  @param cb: The callback to be called with the nodegroup
997

998
  """
999
  def fn(ctx, node):
1000
    """Get group data for a node.
1001

1002
    @type ctx: L{NodeQueryData}
1003
    @type inst: L{objects.Node}
1004
    @param inst: Node object
1005

1006
    """
1007
    ng = ctx.groups.get(node.group, None)
1008
    if ng is None:
1009
      # Nodes always have a group, or the configuration is corrupt
1010
      return _FS_UNAVAIL
1011

    
1012
    return cb(ctx, node, ng)
1013

    
1014
  return fn
1015

    
1016

    
1017
def _GetNodeGroup(ctx, node, ng): # pylint: disable-msg=W0613
1018
  """Returns the name of a node's group.
1019

1020
  @type ctx: L{NodeQueryData}
1021
  @type node: L{objects.Node}
1022
  @param node: Node object
1023
  @type ng: L{objects.NodeGroup}
1024
  @param ng: The node group this node belongs to
1025

1026
  """
1027
  return ng.name
1028

    
1029

    
1030
def _GetNodePower(ctx, node):
1031
  """Returns the node powered state
1032

1033
  @type ctx: L{NodeQueryData}
1034
  @type node: L{objects.Node}
1035
  @param node: Node object
1036

1037
  """
1038
  if ctx.oob_support[node.name]:
1039
    return node.powered
1040

    
1041
  return _FS_UNAVAIL
1042

    
1043

    
1044
def _GetNdParams(ctx, node, ng):
1045
  """Returns the ndparams for this node.
1046

1047
  @type ctx: L{NodeQueryData}
1048
  @type node: L{objects.Node}
1049
  @param node: Node object
1050
  @type ng: L{objects.NodeGroup}
1051
  @param ng: The node group this node belongs to
1052

1053
  """
1054
  return ctx.cluster.SimpleFillND(ng.FillND(node))
1055

    
1056

    
1057
def _GetLiveNodeField(field, kind, ctx, node):
1058
  """Gets the value of a "live" field from L{NodeQueryData}.
1059

1060
  @param field: Live field name
1061
  @param kind: Data kind, one of L{constants.QFT_ALL}
1062
  @type ctx: L{NodeQueryData}
1063
  @type node: L{objects.Node}
1064
  @param node: Node object
1065

1066
  """
1067
  if node.offline:
1068
    return _FS_OFFLINE
1069

    
1070
  if not node.vm_capable:
1071
    return _FS_UNAVAIL
1072

    
1073
  if not ctx.curlive_data:
1074
    return _FS_NODATA
1075

    
1076
  try:
1077
    value = ctx.curlive_data[field]
1078
  except KeyError:
1079
    return _FS_UNAVAIL
1080

    
1081
  if kind == QFT_TEXT:
1082
    return value
1083

    
1084
  assert kind in (QFT_NUMBER, QFT_UNIT)
1085

    
1086
  # Try to convert into number
1087
  try:
1088
    return int(value)
1089
  except (ValueError, TypeError):
1090
    logging.exception("Failed to convert node field '%s' (value %r) to int",
1091
                      value, field)
1092
    return _FS_UNAVAIL
1093

    
1094

    
1095
def _BuildNodeFields():
1096
  """Builds list of fields for node queries.
1097

1098
  """
1099
  fields = [
1100
    (_MakeField("pip", "PrimaryIP", QFT_TEXT, "Primary IP address"),
1101
     NQ_CONFIG, 0, _GetItemAttr("primary_ip")),
1102
    (_MakeField("sip", "SecondaryIP", QFT_TEXT, "Secondary IP address"),
1103
     NQ_CONFIG, 0, _GetItemAttr("secondary_ip")),
1104
    (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), NQ_CONFIG, 0,
1105
     lambda ctx, node: list(node.GetTags())),
1106
    (_MakeField("master", "IsMaster", QFT_BOOL, "Whether node is master"),
1107
     NQ_CONFIG, 0, lambda ctx, node: node.name == ctx.master_name),
1108
    (_MakeField("group", "Group", QFT_TEXT, "Node group"), NQ_GROUP, 0,
1109
     _GetGroup(_GetNodeGroup)),
1110
    (_MakeField("group.uuid", "GroupUUID", QFT_TEXT, "UUID of node group"),
1111
     NQ_CONFIG, 0, _GetItemAttr("group")),
1112
    (_MakeField("powered", "Powered", QFT_BOOL,
1113
                "Whether node is thought to be powered on"),
1114
     NQ_OOB, 0, _GetNodePower),
1115
    (_MakeField("ndparams", "NodeParameters", QFT_OTHER,
1116
                "Merged node parameters"),
1117
     NQ_GROUP, 0, _GetGroup(_GetNdParams)),
1118
    (_MakeField("custom_ndparams", "CustomNodeParameters", QFT_OTHER,
1119
                "Custom node parameters"),
1120
      NQ_GROUP, 0, _GetItemAttr("ndparams")),
1121
    ]
1122

    
1123
  # Node role
1124
  role_values = (constants.NR_MASTER, constants.NR_MCANDIDATE,
1125
                 constants.NR_REGULAR, constants.NR_DRAINED,
1126
                 constants.NR_OFFLINE)
1127
  role_doc = ("Node role; \"%s\" for master, \"%s\" for master candidate,"
1128
              " \"%s\" for regular, \"%s\" for a drained, \"%s\" for offline" %
1129
              role_values)
1130
  fields.append((_MakeField("role", "Role", QFT_TEXT, role_doc), NQ_CONFIG, 0,
1131
                 lambda ctx, node: _GetNodeRole(node, ctx.master_name)))
1132
  assert set(role_values) == constants.NR_ALL
1133

    
1134
  def _GetLength(getter):
1135
    return lambda ctx, node: len(getter(ctx)[node.name])
1136

    
1137
  def _GetList(getter):
1138
    return lambda ctx, node: list(getter(ctx)[node.name])
1139

    
1140
  # Add fields operating on instance lists
1141
  for prefix, titleprefix, docword, getter in \
1142
      [("p", "Pri", "primary", operator.attrgetter("node_to_primary")),
1143
       ("s", "Sec", "secondary", operator.attrgetter("node_to_secondary"))]:
1144
    # TODO: Allow filterting by hostname in list
1145
    fields.extend([
1146
      (_MakeField("%sinst_cnt" % prefix, "%sinst" % prefix.upper(), QFT_NUMBER,
1147
                  "Number of instances with this node as %s" % docword),
1148
       NQ_INST, 0, _GetLength(getter)),
1149
      (_MakeField("%sinst_list" % prefix, "%sInstances" % titleprefix,
1150
                  QFT_OTHER,
1151
                  "List of instances with this node as %s" % docword),
1152
       NQ_INST, 0, _GetList(getter)),
1153
      ])
1154

    
1155
  # Add simple fields
1156
  fields.extend([
1157
    (_MakeField(name, title, kind, doc), NQ_CONFIG, flags, _GetItemAttr(name))
1158
    for (name, (title, kind, flags, doc)) in _NODE_SIMPLE_FIELDS.items()
1159
    ])
1160

    
1161
  # Add fields requiring live data
1162
  fields.extend([
1163
    (_MakeField(name, title, kind, doc), NQ_LIVE, 0,
1164
     compat.partial(_GetLiveNodeField, nfield, kind))
1165
    for (name, (title, kind, nfield, doc)) in _NODE_LIVE_FIELDS.items()
1166
    ])
1167

    
1168
  # Add timestamps
1169
  fields.extend(_GetItemTimestampFields(NQ_CONFIG))
1170

    
1171
  return _PrepareFieldList(fields, [])
1172

    
1173

    
1174
class InstanceQueryData:
1175
  """Data container for instance data queries.
1176

1177
  """
1178
  def __init__(self, instances, cluster, disk_usage, offline_nodes, bad_nodes,
1179
               live_data, wrongnode_inst, console):
1180
    """Initializes this class.
1181

1182
    @param instances: List of instance objects
1183
    @param cluster: Cluster object
1184
    @type disk_usage: dict; instance name as key
1185
    @param disk_usage: Per-instance disk usage
1186
    @type offline_nodes: list of strings
1187
    @param offline_nodes: List of offline nodes
1188
    @type bad_nodes: list of strings
1189
    @param bad_nodes: List of faulty nodes
1190
    @type live_data: dict; instance name as key
1191
    @param live_data: Per-instance live data
1192
    @type wrongnode_inst: set
1193
    @param wrongnode_inst: Set of instances running on wrong node(s)
1194
    @type console: dict; instance name as key
1195
    @param console: Per-instance console information
1196

1197
    """
1198
    assert len(set(bad_nodes) & set(offline_nodes)) == len(offline_nodes), \
1199
           "Offline nodes not included in bad nodes"
1200
    assert not (set(live_data.keys()) & set(bad_nodes)), \
1201
           "Found live data for bad or offline nodes"
1202

    
1203
    self.instances = instances
1204
    self.cluster = cluster
1205
    self.disk_usage = disk_usage
1206
    self.offline_nodes = offline_nodes
1207
    self.bad_nodes = bad_nodes
1208
    self.live_data = live_data
1209
    self.wrongnode_inst = wrongnode_inst
1210
    self.console = console
1211

    
1212
    # Used for individual rows
1213
    self.inst_hvparams = None
1214
    self.inst_beparams = None
1215
    self.inst_nicparams = None
1216

    
1217
  def __iter__(self):
1218
    """Iterate over all instances.
1219

1220
    This function has side-effects and only one instance of the resulting
1221
    generator should be used at a time.
1222

1223
    """
1224
    for inst in self.instances:
1225
      self.inst_hvparams = self.cluster.FillHV(inst, skip_globals=True)
1226
      self.inst_beparams = self.cluster.FillBE(inst)
1227
      self.inst_nicparams = [self.cluster.SimpleFillNIC(nic.nicparams)
1228
                             for nic in inst.nics]
1229

    
1230
      yield inst
1231

    
1232

    
1233
def _GetInstOperState(ctx, inst):
1234
  """Get instance's operational status.
1235

1236
  @type ctx: L{InstanceQueryData}
1237
  @type inst: L{objects.Instance}
1238
  @param inst: Instance object
1239

1240
  """
1241
  # Can't use RS_OFFLINE here as it would describe the instance to
1242
  # be offline when we actually don't know due to missing data
1243
  if inst.primary_node in ctx.bad_nodes:
1244
    return _FS_NODATA
1245
  else:
1246
    return bool(ctx.live_data.get(inst.name))
1247

    
1248

    
1249
def _GetInstLiveData(name):
1250
  """Build function for retrieving live data.
1251

1252
  @type name: string
1253
  @param name: Live data field name
1254

1255
  """
1256
  def fn(ctx, inst):
1257
    """Get live data for an instance.
1258

1259
    @type ctx: L{InstanceQueryData}
1260
    @type inst: L{objects.Instance}
1261
    @param inst: Instance object
1262

1263
    """
1264
    if (inst.primary_node in ctx.bad_nodes or
1265
        inst.primary_node in ctx.offline_nodes):
1266
      # Can't use RS_OFFLINE here as it would describe the instance to be
1267
      # offline when we actually don't know due to missing data
1268
      return _FS_NODATA
1269

    
1270
    if inst.name in ctx.live_data:
1271
      data = ctx.live_data[inst.name]
1272
      if name in data:
1273
        return data[name]
1274

    
1275
    return _FS_UNAVAIL
1276

    
1277
  return fn
1278

    
1279

    
1280
def _GetInstStatus(ctx, inst):
1281
  """Get instance status.
1282

1283
  @type ctx: L{InstanceQueryData}
1284
  @type inst: L{objects.Instance}
1285
  @param inst: Instance object
1286

1287
  """
1288
  if inst.primary_node in ctx.offline_nodes:
1289
    return constants.INSTST_NODEOFFLINE
1290

    
1291
  if inst.primary_node in ctx.bad_nodes:
1292
    return constants.INSTST_NODEDOWN
1293

    
1294
  if bool(ctx.live_data.get(inst.name)):
1295
    if inst.name in ctx.wrongnode_inst:
1296
      return constants.INSTST_WRONGNODE
1297
    elif inst.admin_up:
1298
      return constants.INSTST_RUNNING
1299
    else:
1300
      return constants.INSTST_ERRORUP
1301

    
1302
  if inst.admin_up:
1303
    return constants.INSTST_ERRORDOWN
1304

    
1305
  return constants.INSTST_ADMINDOWN
1306

    
1307

    
1308
def _GetInstDiskSize(index):
1309
  """Build function for retrieving disk size.
1310

1311
  @type index: int
1312
  @param index: Disk index
1313

1314
  """
1315
  def fn(_, inst):
1316
    """Get size of a disk.
1317

1318
    @type inst: L{objects.Instance}
1319
    @param inst: Instance object
1320

1321
    """
1322
    try:
1323
      return inst.disks[index].size
1324
    except IndexError:
1325
      return _FS_UNAVAIL
1326

    
1327
  return fn
1328

    
1329

    
1330
def _GetInstNic(index, cb):
1331
  """Build function for calling another function with an instance NIC.
1332

1333
  @type index: int
1334
  @param index: NIC index
1335
  @type cb: callable
1336
  @param cb: Callback
1337

1338
  """
1339
  def fn(ctx, inst):
1340
    """Call helper function with instance NIC.
1341

1342
    @type ctx: L{InstanceQueryData}
1343
    @type inst: L{objects.Instance}
1344
    @param inst: Instance object
1345

1346
    """
1347
    try:
1348
      nic = inst.nics[index]
1349
    except IndexError:
1350
      return _FS_UNAVAIL
1351

    
1352
    return cb(ctx, index, nic)
1353

    
1354
  return fn
1355

    
1356

    
1357
def _GetInstNicIp(ctx, _, nic): # pylint: disable-msg=W0613
1358
  """Get a NIC's IP address.
1359

1360
  @type ctx: L{InstanceQueryData}
1361
  @type nic: L{objects.NIC}
1362
  @param nic: NIC object
1363

1364
  """
1365
  if nic.ip is None:
1366
    return _FS_UNAVAIL
1367
  else:
1368
    return nic.ip
1369

    
1370

    
1371
def _GetInstNicBridge(ctx, index, _):
1372
  """Get a NIC's bridge.
1373

1374
  @type ctx: L{InstanceQueryData}
1375
  @type index: int
1376
  @param index: NIC index
1377

1378
  """
1379
  assert len(ctx.inst_nicparams) >= index
1380

    
1381
  nicparams = ctx.inst_nicparams[index]
1382

    
1383
  if nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
1384
    return nicparams[constants.NIC_LINK]
1385
  else:
1386
    return _FS_UNAVAIL
1387

    
1388

    
1389
def _GetInstAllNicBridges(ctx, inst):
1390
  """Get all network bridges for an instance.
1391

1392
  @type ctx: L{InstanceQueryData}
1393
  @type inst: L{objects.Instance}
1394
  @param inst: Instance object
1395

1396
  """
1397
  assert len(ctx.inst_nicparams) == len(inst.nics)
1398

    
1399
  result = []
1400

    
1401
  for nicp in ctx.inst_nicparams:
1402
    if nicp[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
1403
      result.append(nicp[constants.NIC_LINK])
1404
    else:
1405
      result.append(None)
1406

    
1407
  assert len(result) == len(inst.nics)
1408

    
1409
  return result
1410

    
1411

    
1412
def _GetInstNicParam(name):
1413
  """Build function for retrieving a NIC parameter.
1414

1415
  @type name: string
1416
  @param name: Parameter name
1417

1418
  """
1419
  def fn(ctx, index, _):
1420
    """Get a NIC's bridge.
1421

1422
    @type ctx: L{InstanceQueryData}
1423
    @type inst: L{objects.Instance}
1424
    @param inst: Instance object
1425
    @type nic: L{objects.NIC}
1426
    @param nic: NIC object
1427

1428
    """
1429
    assert len(ctx.inst_nicparams) >= index
1430
    return ctx.inst_nicparams[index][name]
1431

    
1432
  return fn
1433

    
1434

    
1435
def _GetInstanceNetworkFields():
1436
  """Get instance fields involving network interfaces.
1437

1438
  @return: Tuple containing list of field definitions used as input for
1439
    L{_PrepareFieldList} and a list of aliases
1440

1441
  """
1442
  nic_mac_fn = lambda ctx, _, nic: nic.mac
1443
  nic_mode_fn = _GetInstNicParam(constants.NIC_MODE)
1444
  nic_link_fn = _GetInstNicParam(constants.NIC_LINK)
1445

    
1446
  fields = [
1447
    # All NICs
1448
    (_MakeField("nic.count", "NICs", QFT_NUMBER,
1449
                "Number of network interfaces"),
1450
     IQ_CONFIG, 0, lambda ctx, inst: len(inst.nics)),
1451
    (_MakeField("nic.macs", "NIC_MACs", QFT_OTHER,
1452
                "List containing each network interface's MAC address"),
1453
     IQ_CONFIG, 0, lambda ctx, inst: [nic.mac for nic in inst.nics]),
1454
    (_MakeField("nic.ips", "NIC_IPs", QFT_OTHER,
1455
                "List containing each network interface's IP address"),
1456
     IQ_CONFIG, 0, lambda ctx, inst: [nic.ip for nic in inst.nics]),
1457
    (_MakeField("nic.modes", "NIC_modes", QFT_OTHER,
1458
                "List containing each network interface's mode"), IQ_CONFIG, 0,
1459
     lambda ctx, inst: [nicp[constants.NIC_MODE]
1460
                        for nicp in ctx.inst_nicparams]),
1461
    (_MakeField("nic.links", "NIC_links", QFT_OTHER,
1462
                "List containing each network interface's link"), IQ_CONFIG, 0,
1463
     lambda ctx, inst: [nicp[constants.NIC_LINK]
1464
                        for nicp in ctx.inst_nicparams]),
1465
    (_MakeField("nic.bridges", "NIC_bridges", QFT_OTHER,
1466
                "List containing each network interface's bridge"),
1467
     IQ_CONFIG, 0, _GetInstAllNicBridges),
1468
    ]
1469

    
1470
  # NICs by number
1471
  for i in range(constants.MAX_NICS):
1472
    numtext = utils.FormatOrdinal(i + 1)
1473
    fields.extend([
1474
      (_MakeField("nic.ip/%s" % i, "NicIP/%s" % i, QFT_TEXT,
1475
                  "IP address of %s network interface" % numtext),
1476
       IQ_CONFIG, 0, _GetInstNic(i, _GetInstNicIp)),
1477
      (_MakeField("nic.mac/%s" % i, "NicMAC/%s" % i, QFT_TEXT,
1478
                  "MAC address of %s network interface" % numtext),
1479
       IQ_CONFIG, 0, _GetInstNic(i, nic_mac_fn)),
1480
      (_MakeField("nic.mode/%s" % i, "NicMode/%s" % i, QFT_TEXT,
1481
                  "Mode of %s network interface" % numtext),
1482
       IQ_CONFIG, 0, _GetInstNic(i, nic_mode_fn)),
1483
      (_MakeField("nic.link/%s" % i, "NicLink/%s" % i, QFT_TEXT,
1484
                  "Link of %s network interface" % numtext),
1485
       IQ_CONFIG, 0, _GetInstNic(i, nic_link_fn)),
1486
      (_MakeField("nic.bridge/%s" % i, "NicBridge/%s" % i, QFT_TEXT,
1487
                  "Bridge of %s network interface" % numtext),
1488
       IQ_CONFIG, 0, _GetInstNic(i, _GetInstNicBridge)),
1489
      ])
1490

    
1491
  aliases = [
1492
    # Legacy fields for first NIC
1493
    ("ip", "nic.ip/0"),
1494
    ("mac", "nic.mac/0"),
1495
    ("bridge", "nic.bridge/0"),
1496
    ("nic_mode", "nic.mode/0"),
1497
    ("nic_link", "nic.link/0"),
1498
    ]
1499

    
1500
  return (fields, aliases)
1501

    
1502

    
1503
def _GetInstDiskUsage(ctx, inst):
1504
  """Get disk usage for an instance.
1505

1506
  @type ctx: L{InstanceQueryData}
1507
  @type inst: L{objects.Instance}
1508
  @param inst: Instance object
1509

1510
  """
1511
  usage = ctx.disk_usage[inst.name]
1512

    
1513
  if usage is None:
1514
    usage = 0
1515

    
1516
  return usage
1517

    
1518

    
1519
def _GetInstanceConsole(ctx, inst):
1520
  """Get console information for instance.
1521

1522
  @type ctx: L{InstanceQueryData}
1523
  @type inst: L{objects.Instance}
1524
  @param inst: Instance object
1525

1526
  """
1527
  consinfo = ctx.console[inst.name]
1528

    
1529
  if consinfo is None:
1530
    return _FS_UNAVAIL
1531

    
1532
  return consinfo
1533

    
1534

    
1535
def _GetInstanceDiskFields():
1536
  """Get instance fields involving disks.
1537

1538
  @return: List of field definitions used as input for L{_PrepareFieldList}
1539

1540
  """
1541
  fields = [
1542
    (_MakeField("disk_usage", "DiskUsage", QFT_UNIT,
1543
                "Total disk space used by instance on each of its nodes;"
1544
                " this is not the disk size visible to the instance, but"
1545
                " the usage on the node"),
1546
     IQ_DISKUSAGE, 0, _GetInstDiskUsage),
1547
    (_MakeField("disk.count", "Disks", QFT_NUMBER, "Number of disks"),
1548
     IQ_CONFIG, 0, lambda ctx, inst: len(inst.disks)),
1549
    (_MakeField("disk.sizes", "Disk_sizes", QFT_OTHER, "List of disk sizes"),
1550
     IQ_CONFIG, 0, lambda ctx, inst: [disk.size for disk in inst.disks]),
1551
    ]
1552

    
1553
  # Disks by number
1554
  fields.extend([
1555
    (_MakeField("disk.size/%s" % i, "Disk/%s" % i, QFT_UNIT,
1556
                "Disk size of %s disk" % utils.FormatOrdinal(i + 1)),
1557
     IQ_CONFIG, 0, _GetInstDiskSize(i))
1558
    for i in range(constants.MAX_DISKS)
1559
    ])
1560

    
1561
  return fields
1562

    
1563

    
1564
def _GetInstanceParameterFields():
1565
  """Get instance fields involving parameters.
1566

1567
  @return: List of field definitions used as input for L{_PrepareFieldList}
1568

1569
  """
1570
  # TODO: Consider moving titles closer to constants
1571
  be_title = {
1572
    constants.BE_AUTO_BALANCE: "Auto_balance",
1573
    constants.BE_MEMORY: "ConfigMemory",
1574
    constants.BE_VCPUS: "ConfigVCPUs",
1575
    }
1576

    
1577
  hv_title = {
1578
    constants.HV_ACPI: "ACPI",
1579
    constants.HV_BOOT_ORDER: "Boot_order",
1580
    constants.HV_CDROM_IMAGE_PATH: "CDROM_image_path",
1581
    constants.HV_DISK_TYPE: "Disk_type",
1582
    constants.HV_INITRD_PATH: "Initrd_path",
1583
    constants.HV_KERNEL_PATH: "Kernel_path",
1584
    constants.HV_NIC_TYPE: "NIC_type",
1585
    constants.HV_PAE: "PAE",
1586
    constants.HV_VNC_BIND_ADDRESS: "VNC_bind_address",
1587
    }
1588

    
1589
  fields = [
1590
    # Filled parameters
1591
    (_MakeField("hvparams", "HypervisorParameters", QFT_OTHER,
1592
                "Hypervisor parameters"),
1593
     IQ_CONFIG, 0, lambda ctx, _: ctx.inst_hvparams),
1594
    (_MakeField("beparams", "BackendParameters", QFT_OTHER,
1595
                "Backend parameters"),
1596
     IQ_CONFIG, 0, lambda ctx, _: ctx.inst_beparams),
1597

    
1598
    # Unfilled parameters
1599
    (_MakeField("custom_hvparams", "CustomHypervisorParameters", QFT_OTHER,
1600
                "Custom hypervisor parameters"),
1601
     IQ_CONFIG, 0, _GetItemAttr("hvparams")),
1602
    (_MakeField("custom_beparams", "CustomBackendParameters", QFT_OTHER,
1603
                "Custom backend parameters",),
1604
     IQ_CONFIG, 0, _GetItemAttr("beparams")),
1605
    (_MakeField("custom_nicparams", "CustomNicParameters", QFT_OTHER,
1606
                "Custom network interface parameters"),
1607
     IQ_CONFIG, 0, lambda ctx, inst: [nic.nicparams for nic in inst.nics]),
1608
    ]
1609

    
1610
  # HV params
1611
  def _GetInstHvParam(name):
1612
    return lambda ctx, _: ctx.inst_hvparams.get(name, _FS_UNAVAIL)
1613

    
1614
  fields.extend([
1615
    (_MakeField("hv/%s" % name, hv_title.get(name, "hv/%s" % name),
1616
                _VTToQFT[kind], "The \"%s\" hypervisor parameter" % name),
1617
     IQ_CONFIG, 0, _GetInstHvParam(name))
1618
    for name, kind in constants.HVS_PARAMETER_TYPES.items()
1619
    if name not in constants.HVC_GLOBALS
1620
    ])
1621

    
1622
  # BE params
1623
  def _GetInstBeParam(name):
1624
    return lambda ctx, _: ctx.inst_beparams.get(name, None)
1625

    
1626
  fields.extend([
1627
    (_MakeField("be/%s" % name, be_title.get(name, "be/%s" % name),
1628
                _VTToQFT[kind], "The \"%s\" backend parameter" % name),
1629
     IQ_CONFIG, 0, _GetInstBeParam(name))
1630
    for name, kind in constants.BES_PARAMETER_TYPES.items()
1631
    ])
1632

    
1633
  return fields
1634

    
1635

    
1636
_INST_SIMPLE_FIELDS = {
1637
  "disk_template": ("Disk_template", QFT_TEXT, 0, "Instance disk template"),
1638
  "hypervisor": ("Hypervisor", QFT_TEXT, 0, "Hypervisor name"),
1639
  "name": ("Instance", QFT_TEXT, QFF_HOSTNAME, "Instance name"),
1640
  # Depending on the hypervisor, the port can be None
1641
  "network_port": ("Network_port", QFT_OTHER, 0,
1642
                   "Instance network port if available (e.g. for VNC console)"),
1643
  "os": ("OS", QFT_TEXT, 0, "Operating system"),
1644
  "serial_no": ("SerialNo", QFT_NUMBER, 0, _SERIAL_NO_DOC % "Instance"),
1645
  "uuid": ("UUID", QFT_TEXT, 0, "Instance UUID"),
1646
  }
1647

    
1648

    
1649
def _BuildInstanceFields():
1650
  """Builds list of fields for instance queries.
1651

1652
  """
1653
  fields = [
1654
    (_MakeField("pnode", "Primary_node", QFT_TEXT, "Primary node"),
1655
     IQ_CONFIG, QFF_HOSTNAME, _GetItemAttr("primary_node")),
1656
    # TODO: Allow filtering by secondary node as hostname
1657
    (_MakeField("snodes", "Secondary_Nodes", QFT_OTHER,
1658
                "Secondary nodes; usually this will just be one node"),
1659
     IQ_CONFIG, 0, lambda ctx, inst: list(inst.secondary_nodes)),
1660
    (_MakeField("admin_state", "Autostart", QFT_BOOL,
1661
                "Desired state of instance (if set, the instance should be"
1662
                " up)"),
1663
     IQ_CONFIG, 0, _GetItemAttr("admin_up")),
1664
    (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), IQ_CONFIG, 0,
1665
     lambda ctx, inst: list(inst.GetTags())),
1666
    (_MakeField("console", "Console", QFT_OTHER,
1667
                "Instance console information"), IQ_CONSOLE, 0,
1668
     _GetInstanceConsole),
1669
    ]
1670

    
1671
  # Add simple fields
1672
  fields.extend([
1673
    (_MakeField(name, title, kind, doc), IQ_CONFIG, flags, _GetItemAttr(name))
1674
    for (name, (title, kind, flags, doc)) in _INST_SIMPLE_FIELDS.items()
1675
    ])
1676

    
1677
  # Fields requiring talking to the node
1678
  fields.extend([
1679
    (_MakeField("oper_state", "Running", QFT_BOOL, "Actual state of instance"),
1680
     IQ_LIVE, 0, _GetInstOperState),
1681
    (_MakeField("oper_ram", "Memory", QFT_UNIT,
1682
                "Actual memory usage as seen by hypervisor"),
1683
     IQ_LIVE, 0, _GetInstLiveData("memory")),
1684
    (_MakeField("oper_vcpus", "VCPUs", QFT_NUMBER,
1685
                "Actual number of VCPUs as seen by hypervisor"),
1686
     IQ_LIVE, 0, _GetInstLiveData("vcpus")),
1687
    ])
1688

    
1689
  # Status field
1690
  status_values = (constants.INSTST_RUNNING, constants.INSTST_ADMINDOWN,
1691
                   constants.INSTST_WRONGNODE, constants.INSTST_ERRORUP,
1692
                   constants.INSTST_ERRORDOWN, constants.INSTST_NODEDOWN,
1693
                   constants.INSTST_NODEOFFLINE)
1694
  status_doc = ("Instance status; \"%s\" if instance is set to be running"
1695
                " and actually is, \"%s\" if instance is stopped and"
1696
                " is not running, \"%s\" if instance running, but not on its"
1697
                " designated primary node, \"%s\" if instance should be"
1698
                " stopped, but is actually running, \"%s\" if instance should"
1699
                " run, but doesn't, \"%s\" if instance's primary node is down,"
1700
                " \"%s\" if instance's primary node is marked offline" %
1701
                status_values)
1702
  fields.append((_MakeField("status", "Status", QFT_TEXT, status_doc),
1703
                 IQ_LIVE, 0, _GetInstStatus))
1704
  assert set(status_values) == constants.INSTST_ALL, \
1705
         "Status documentation mismatch"
1706

    
1707
  (network_fields, network_aliases) = _GetInstanceNetworkFields()
1708

    
1709
  fields.extend(network_fields)
1710
  fields.extend(_GetInstanceParameterFields())
1711
  fields.extend(_GetInstanceDiskFields())
1712
  fields.extend(_GetItemTimestampFields(IQ_CONFIG))
1713

    
1714
  aliases = [
1715
    ("vcpus", "be/vcpus"),
1716
    ("sda_size", "disk.size/0"),
1717
    ("sdb_size", "disk.size/1"),
1718
    ] + network_aliases
1719

    
1720
  return _PrepareFieldList(fields, aliases)
1721

    
1722

    
1723
class LockQueryData:
1724
  """Data container for lock data queries.
1725

1726
  """
1727
  def __init__(self, lockdata):
1728
    """Initializes this class.
1729

1730
    """
1731
    self.lockdata = lockdata
1732

    
1733
  def __iter__(self):
1734
    """Iterate over all locks.
1735

1736
    """
1737
    return iter(self.lockdata)
1738

    
1739

    
1740
def _GetLockOwners(_, data):
1741
  """Returns a sorted list of a lock's current owners.
1742

1743
  """
1744
  (_, _, owners, _) = data
1745

    
1746
  if owners:
1747
    owners = utils.NiceSort(owners)
1748

    
1749
  return owners
1750

    
1751

    
1752
def _GetLockPending(_, data):
1753
  """Returns a sorted list of a lock's pending acquires.
1754

1755
  """
1756
  (_, _, _, pending) = data
1757

    
1758
  if pending:
1759
    pending = [(mode, utils.NiceSort(names))
1760
               for (mode, names) in pending]
1761

    
1762
  return pending
1763

    
1764

    
1765
def _BuildLockFields():
1766
  """Builds list of fields for lock queries.
1767

1768
  """
1769
  return _PrepareFieldList([
1770
    # TODO: Lock names are not always hostnames. Should QFF_HOSTNAME be used?
1771
    (_MakeField("name", "Name", QFT_TEXT, "Lock name"), None, 0,
1772
     lambda ctx, (name, mode, owners, pending): name),
1773
    (_MakeField("mode", "Mode", QFT_OTHER,
1774
                "Mode in which the lock is currently acquired"
1775
                " (exclusive or shared)"),
1776
     LQ_MODE, 0, lambda ctx, (name, mode, owners, pending): mode),
1777
    (_MakeField("owner", "Owner", QFT_OTHER, "Current lock owner(s)"),
1778
     LQ_OWNER, 0, _GetLockOwners),
1779
    (_MakeField("pending", "Pending", QFT_OTHER,
1780
                "Threads waiting for the lock"),
1781
     LQ_PENDING, 0, _GetLockPending),
1782
    ], [])
1783

    
1784

    
1785
class GroupQueryData:
1786
  """Data container for node group data queries.
1787

1788
  """
1789
  def __init__(self, groups, group_to_nodes, group_to_instances):
1790
    """Initializes this class.
1791

1792
    @param groups: List of node group objects
1793
    @type group_to_nodes: dict; group UUID as key
1794
    @param group_to_nodes: Per-group list of nodes
1795
    @type group_to_instances: dict; group UUID as key
1796
    @param group_to_instances: Per-group list of (primary) instances
1797

1798
    """
1799
    self.groups = groups
1800
    self.group_to_nodes = group_to_nodes
1801
    self.group_to_instances = group_to_instances
1802

    
1803
  def __iter__(self):
1804
    """Iterate over all node groups.
1805

1806
    """
1807
    return iter(self.groups)
1808

    
1809

    
1810
_GROUP_SIMPLE_FIELDS = {
1811
  "alloc_policy": ("AllocPolicy", QFT_TEXT, "Allocation policy for group"),
1812
  "name": ("Group", QFT_TEXT, "Group name"),
1813
  "serial_no": ("SerialNo", QFT_NUMBER, _SERIAL_NO_DOC % "Group"),
1814
  "uuid": ("UUID", QFT_TEXT, "Group UUID"),
1815
  "ndparams": ("NDParams", QFT_OTHER, "Node parameters"),
1816
  }
1817

    
1818

    
1819
def _BuildGroupFields():
1820
  """Builds list of fields for node group queries.
1821

1822
  """
1823
  # Add simple fields
1824
  fields = [(_MakeField(name, title, kind, doc), GQ_CONFIG, 0,
1825
             _GetItemAttr(name))
1826
            for (name, (title, kind, doc)) in _GROUP_SIMPLE_FIELDS.items()]
1827

    
1828
  def _GetLength(getter):
1829
    return lambda ctx, group: len(getter(ctx)[group.uuid])
1830

    
1831
  def _GetSortedList(getter):
1832
    return lambda ctx, group: utils.NiceSort(getter(ctx)[group.uuid])
1833

    
1834
  group_to_nodes = operator.attrgetter("group_to_nodes")
1835
  group_to_instances = operator.attrgetter("group_to_instances")
1836

    
1837
  # Add fields for nodes
1838
  fields.extend([
1839
    (_MakeField("node_cnt", "Nodes", QFT_NUMBER, "Number of nodes"),
1840
     GQ_NODE, 0, _GetLength(group_to_nodes)),
1841
    (_MakeField("node_list", "NodeList", QFT_OTHER, "List of nodes"),
1842
     GQ_NODE, 0, _GetSortedList(group_to_nodes)),
1843
    ])
1844

    
1845
  # Add fields for instances
1846
  fields.extend([
1847
    (_MakeField("pinst_cnt", "Instances", QFT_NUMBER,
1848
                "Number of primary instances"),
1849
     GQ_INST, 0, _GetLength(group_to_instances)),
1850
    (_MakeField("pinst_list", "InstanceList", QFT_OTHER,
1851
                "List of primary instances"),
1852
     GQ_INST, 0, _GetSortedList(group_to_instances)),
1853
    ])
1854

    
1855
  fields.extend(_GetItemTimestampFields(GQ_CONFIG))
1856

    
1857
  return _PrepareFieldList(fields, [])
1858

    
1859

    
1860
#: Fields available for node queries
1861
NODE_FIELDS = _BuildNodeFields()
1862

    
1863
#: Fields available for instance queries
1864
INSTANCE_FIELDS = _BuildInstanceFields()
1865

    
1866
#: Fields available for lock queries
1867
LOCK_FIELDS = _BuildLockFields()
1868

    
1869
#: Fields available for node group queries
1870
GROUP_FIELDS = _BuildGroupFields()
1871

    
1872
#: All available resources
1873
ALL_FIELDS = {
1874
  constants.QR_INSTANCE: INSTANCE_FIELDS,
1875
  constants.QR_NODE: NODE_FIELDS,
1876
  constants.QR_LOCK: LOCK_FIELDS,
1877
  constants.QR_GROUP: GROUP_FIELDS,
1878
  }
1879

    
1880
#: All available field lists
1881
ALL_FIELD_LISTS = ALL_FIELDS.values()