code.grnet.gr Git - ganeti-local/blob - test/ganeti.query_unittest.py

   1 #!/usr/bin/python
   2 #
   3
   4 # Copyright (C) 2010, 2011, 2012 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Script for testing ganeti.query"""
  23
  24 import re
  25 import unittest
  26 import random
  27
  28 from ganeti import constants
  29 from ganeti import utils
  30 from ganeti import compat
  31 from ganeti import errors
  32 from ganeti import query
  33 from ganeti import objects
  34 from ganeti import cmdlib
  35
  36 import testutils
  37
  38
  39 class TestConstants(unittest.TestCase):
  40   def test(self):
  41     self.assertEqual(set(query._VERIFY_FN.keys()),
  42                      constants.QFT_ALL)
  43
  44
  45 class _QueryData:
  46   def __init__(self, data, **kwargs):
  47     self.data = data
  48
  49     for name, value in kwargs.items():
  50       setattr(self, name, value)
  51
  52   def __iter__(self):
  53     return iter(self.data)
  54
  55
  56 def _GetDiskSize(nr, ctx, item):
  57   disks = item["disks"]
  58   try:
  59     return disks[nr]
  60   except IndexError:
  61     return query._FS_UNAVAIL
  62
  63
  64 class TestQuery(unittest.TestCase):
  65   def test(self):
  66     (STATIC, DISK) = range(10, 12)
  67
  68     fielddef = query._PrepareFieldList([
  69       (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"),
  70        STATIC, 0, lambda ctx, item: item["name"]),
  71       (query._MakeField("master", "Master", constants.QFT_BOOL, "Master"),
  72        STATIC, 0, lambda ctx, item: ctx.mastername == item["name"]),
  73       ] +
  74       [(query._MakeField("disk%s.size" % i, "DiskSize%s" % i,
  75                          constants.QFT_UNIT, "Disk size %s" % i),
  76         DISK, 0, compat.partial(_GetDiskSize, i))
  77        for i in range(4)], [])
  78
  79     q = query.Query(fielddef, ["name"])
  80     self.assertEqual(q.RequestedData(), set([STATIC]))
  81     self.assertEqual(len(q._fields), 1)
  82     self.assertEqual(len(q.GetFields()), 1)
  83     self.assertEqual(q.GetFields()[0].ToDict(),
  84       objects.QueryFieldDefinition(name="name",
  85                                    title="Name",
  86                                    kind=constants.QFT_TEXT,
  87                                    doc="Name").ToDict())
  88
  89     # Create data only once query has been prepared
  90     data = [
  91       { "name": "node1", "disks": [0, 1, 2], },
  92       { "name": "node2", "disks": [3, 4], },
  93       { "name": "node3", "disks": [5, 6, 7], },
  94       ]
  95
  96     self.assertEqual(q.Query(_QueryData(data, mastername="node3")),
  97                      [[(constants.RS_NORMAL, "node1")],
  98                       [(constants.RS_NORMAL, "node2")],
  99                       [(constants.RS_NORMAL, "node3")]])
 100     self.assertEqual(q.OldStyleQuery(_QueryData(data, mastername="node3")),
 101                      [["node1"], ["node2"], ["node3"]])
 102
 103     q = query.Query(fielddef, ["name", "master"])
 104     self.assertEqual(q.RequestedData(), set([STATIC]))
 105     self.assertEqual(len(q._fields), 2)
 106     self.assertEqual(q.Query(_QueryData(data, mastername="node3")),
 107                      [[(constants.RS_NORMAL, "node1"),
 108                        (constants.RS_NORMAL, False)],
 109                       [(constants.RS_NORMAL, "node2"),
 110                        (constants.RS_NORMAL, False)],
 111                       [(constants.RS_NORMAL, "node3"),
 112                        (constants.RS_NORMAL, True)],
 113                      ])
 114
 115     q = query.Query(fielddef, ["name", "master", "disk0.size"])
 116     self.assertEqual(q.RequestedData(), set([STATIC, DISK]))
 117     self.assertEqual(len(q._fields), 3)
 118     self.assertEqual(q.Query(_QueryData(data, mastername="node2")),
 119                      [[(constants.RS_NORMAL, "node1"),
 120                        (constants.RS_NORMAL, False),
 121                        (constants.RS_NORMAL, 0)],
 122                       [(constants.RS_NORMAL, "node2"),
 123                        (constants.RS_NORMAL, True),
 124                        (constants.RS_NORMAL, 3)],
 125                       [(constants.RS_NORMAL, "node3"),
 126                        (constants.RS_NORMAL, False),
 127                        (constants.RS_NORMAL, 5)],
 128                      ])
 129
 130     # With unknown column
 131     q = query.Query(fielddef, ["disk2.size", "disk1.size", "disk99.size",
 132                                "disk0.size"])
 133     self.assertEqual(q.RequestedData(), set([DISK]))
 134     self.assertEqual(len(q._fields), 4)
 135     self.assertEqual(q.Query(_QueryData(data, mastername="node2")),
 136                      [[(constants.RS_NORMAL, 2),
 137                        (constants.RS_NORMAL, 1),
 138                        (constants.RS_UNKNOWN, None),
 139                        (constants.RS_NORMAL, 0)],
 140                       [(constants.RS_UNAVAIL, None),
 141                        (constants.RS_NORMAL, 4),
 142                        (constants.RS_UNKNOWN, None),
 143                        (constants.RS_NORMAL, 3)],
 144                       [(constants.RS_NORMAL, 7),
 145                        (constants.RS_NORMAL, 6),
 146                        (constants.RS_UNKNOWN, None),
 147                        (constants.RS_NORMAL, 5)],
 148                      ])
 149     self.assertRaises(errors.OpPrereqError, q.OldStyleQuery,
 150                       _QueryData(data, mastername="node2"))
 151     self.assertEqual([fdef.ToDict() for fdef in q.GetFields()], [
 152                      { "name": "disk2.size", "title": "DiskSize2",
 153                        "kind": constants.QFT_UNIT, "doc": "Disk size 2", },
 154                      { "name": "disk1.size", "title": "DiskSize1",
 155                        "kind": constants.QFT_UNIT, "doc": "Disk size 1", },
 156                      { "name": "disk99.size", "title": "disk99.size",
 157                        "kind": constants.QFT_UNKNOWN,
 158                        "doc": "Unknown field 'disk99.size'", },
 159                      { "name": "disk0.size", "title": "DiskSize0",
 160                        "kind": constants.QFT_UNIT, "doc": "Disk size 0", },
 161                      ])
 162
 163     # Empty query
 164     q = query.Query(fielddef, [])
 165     self.assertEqual(q.RequestedData(), set([]))
 166     self.assertEqual(len(q._fields), 0)
 167     self.assertEqual(q.Query(_QueryData(data, mastername="node2")),
 168                      [[], [], []])
 169     self.assertEqual(q.OldStyleQuery(_QueryData(data, mastername="node2")),
 170                      [[], [], []])
 171     self.assertEqual(q.GetFields(), [])
 172
 173   def testPrepareFieldList(self):
 174     # Duplicate titles
 175     for (a, b) in [("name", "name"), ("NAME", "name")]:
 176       self.assertRaises(AssertionError, query._PrepareFieldList, [
 177         (query._MakeField("name", b, constants.QFT_TEXT, "Name"), None, 0,
 178          lambda *args: None),
 179         (query._MakeField("other", a, constants.QFT_TEXT, "Other"), None, 0,
 180          lambda *args: None),
 181         ], [])
 182
 183     # Non-lowercase names
 184     self.assertRaises(AssertionError, query._PrepareFieldList, [
 185       (query._MakeField("NAME", "Name", constants.QFT_TEXT, "Name"), None, 0,
 186        lambda *args: None),
 187       ], [])
 188     self.assertRaises(AssertionError, query._PrepareFieldList, [
 189       (query._MakeField("Name", "Name", constants.QFT_TEXT, "Name"), None, 0,
 190        lambda *args: None),
 191       ], [])
 192
 193     # Empty name
 194     self.assertRaises(AssertionError, query._PrepareFieldList, [
 195       (query._MakeField("", "Name", constants.QFT_TEXT, "Name"), None, 0,
 196        lambda *args: None),
 197       ], [])
 198
 199     # Empty title
 200     self.assertRaises(AssertionError, query._PrepareFieldList, [
 201       (query._MakeField("name", "", constants.QFT_TEXT, "Name"), None, 0,
 202        lambda *args: None),
 203       ], [])
 204
 205     # Whitespace in title
 206     self.assertRaises(AssertionError, query._PrepareFieldList, [
 207       (query._MakeField("name", "Co lu mn", constants.QFT_TEXT, "Name"),
 208        None, 0, lambda *args: None),
 209       ], [])
 210
 211     # No callable function
 212     self.assertRaises(AssertionError, query._PrepareFieldList, [
 213       (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"),
 214        None, 0, None),
 215       ], [])
 216
 217     # Invalid documentation
 218     for doc in ["", ".", "Hello world\n", "Hello\nWo\nrld", "Hello World!",
 219                 "HelloWorld.", "only lowercase", ",", " x y z .\t", "  "]:
 220       self.assertRaises(AssertionError, query._PrepareFieldList, [
 221         (query._MakeField("name", "Name", constants.QFT_TEXT, doc),
 222         None, 0, lambda *args: None),
 223         ], [])
 224
 225     # Duplicate field name
 226     self.assertRaises(ValueError, query._PrepareFieldList, [
 227       (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"),
 228        None, 0, lambda *args: None),
 229       (query._MakeField("name", "Other", constants.QFT_OTHER, "Other"),
 230        None, 0, lambda *args: None),
 231       ], [])
 232
 233   def testUnknown(self):
 234     fielddef = query._PrepareFieldList([
 235       (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"),
 236        None, 0, lambda _, item: "name%s" % item),
 237       (query._MakeField("other0", "Other0", constants.QFT_TIMESTAMP, "Other"),
 238        None, 0, lambda *args: 1234),
 239       (query._MakeField("nodata", "NoData", constants.QFT_NUMBER, "No data"),
 240        None, 0, lambda *args: query._FS_NODATA ),
 241       (query._MakeField("unavail", "Unavail", constants.QFT_BOOL, "Unavail"),
 242        None, 0, lambda *args: query._FS_UNAVAIL),
 243       ], [])
 244
 245     for selected in [["foo"], ["Hello", "World"],
 246                      ["name1", "other", "foo"]]:
 247       q = query.Query(fielddef, selected)
 248       self.assertEqual(len(q._fields), len(selected))
 249       self.assert_(compat.all(len(row) == len(selected)
 250                               for row in q.Query(_QueryData(range(1, 10)))))
 251       self.assertEqual(q.Query(_QueryData(range(1, 10))),
 252                        [[(constants.RS_UNKNOWN, None)] * len(selected)
 253                         for i in range(1, 10)])
 254       self.assertEqual([fdef.ToDict() for fdef in q.GetFields()],
 255                        [{ "name": name, "title": name,
 256                           "kind": constants.QFT_UNKNOWN,
 257                           "doc": "Unknown field '%s'" % name}
 258                         for name in selected])
 259
 260     q = query.Query(fielddef, ["name", "other0", "nodata", "unavail"])
 261     self.assertEqual(len(q._fields), 4)
 262     self.assertEqual(q.OldStyleQuery(_QueryData(range(1, 10))), [
 263                      ["name%s" % i, 1234, None, None]
 264                      for i in range(1, 10)
 265                      ])
 266
 267     q = query.Query(fielddef, ["name", "other0", "nodata", "unavail", "unk"])
 268     self.assertEqual(len(q._fields), 5)
 269     self.assertEqual(q.Query(_QueryData(range(1, 10))),
 270                      [[(constants.RS_NORMAL, "name%s" % i),
 271                        (constants.RS_NORMAL, 1234),
 272                        (constants.RS_NODATA, None),
 273                        (constants.RS_UNAVAIL, None),
 274                        (constants.RS_UNKNOWN, None)]
 275                       for i in range(1, 10)])
 276
 277   def testAliases(self):
 278     fields = [
 279       (query._MakeField("a", "a-title", constants.QFT_TEXT, "Field A"),
 280        None, 0, lambda *args: None),
 281       (query._MakeField("b", "b-title", constants.QFT_TEXT, "Field B"),
 282        None, 0, lambda *args: None),
 283       ]
 284     # duplicate field
 285     self.assertRaises(AssertionError, query._PrepareFieldList, fields,
 286                       [("b", "a")])
 287     self.assertRaises(AssertionError, query._PrepareFieldList, fields,
 288                       [("c", "b"), ("c", "a")])
 289     # missing target
 290     self.assertRaises(AssertionError, query._PrepareFieldList, fields,
 291                       [("c", "d")])
 292     fdefs = query._PrepareFieldList(fields, [("c", "b")])
 293     self.assertEqual(len(fdefs), 3)
 294     self.assertEqual(fdefs["b"][1:], fdefs["c"][1:])
 295
 296
 297 class TestGetNodeRole(unittest.TestCase):
 298   def test(self):
 299     tested_role = set()
 300
 301     checks = [
 302       (constants.NR_MASTER, "node1", objects.Node(name="node1")),
 303       (constants.NR_MCANDIDATE, "master",
 304        objects.Node(name="node1", master_candidate=True)),
 305       (constants.NR_REGULAR, "master", objects.Node(name="node1")),
 306       (constants.NR_DRAINED, "master",
 307        objects.Node(name="node1", drained=True)),
 308       (constants.NR_OFFLINE,
 309        "master", objects.Node(name="node1", offline=True)),
 310       ]
 311
 312     for (role, master_name, node) in checks:
 313       result = query._GetNodeRole(node, master_name)
 314       self.assertEqual(result, role)
 315       tested_role.add(result)
 316
 317     self.assertEqual(tested_role, constants.NR_ALL)
 318
 319
 320 class TestNodeQuery(unittest.TestCase):
 321   def _Create(self, selected):
 322     return query.Query(query.NODE_FIELDS, selected)
 323
 324   def testSimple(self):
 325     cluster = objects.Cluster(cluster_name="testcluster",
 326                               ndparams=constants.NDC_DEFAULTS.copy())
 327     grp1 = objects.NodeGroup(name="default",
 328                              uuid="c0e89160-18e7-11e0-a46e-001d0904baeb",
 329                              alloc_policy=constants.ALLOC_POLICY_PREFERRED,
 330                              ipolicy=objects.MakeEmptyIPolicy(),
 331                              ndparams={},
 332                              )
 333     grp2 = objects.NodeGroup(name="group2",
 334                              uuid="c0e89160-18e7-11e0-a46e-001d0904babe",
 335                              alloc_policy=constants.ALLOC_POLICY_PREFERRED,
 336                              ipolicy=objects.MakeEmptyIPolicy(),
 337                              ndparams={constants.ND_SPINDLE_COUNT: 2},
 338                              )
 339     groups = {grp1.uuid: grp1, grp2.uuid: grp2}
 340     nodes = [
 341       objects.Node(name="node1", drained=False, group=grp1.uuid, ndparams={}),
 342       objects.Node(name="node2", drained=True, group=grp2.uuid, ndparams={}),
 343       objects.Node(name="node3", drained=False, group=grp1.uuid,
 344                    ndparams={constants.ND_SPINDLE_COUNT: 4}),
 345       ]
 346     for live_data in [None, dict.fromkeys([node.name for node in nodes], {})]:
 347       nqd = query.NodeQueryData(nodes, live_data, None, None, None,
 348                                 groups, None, cluster)
 349
 350       q = self._Create(["name", "drained"])
 351       self.assertEqual(q.RequestedData(), set([query.NQ_CONFIG]))
 352       self.assertEqual(q.Query(nqd),
 353                        [[(constants.RS_NORMAL, "node1"),
 354                          (constants.RS_NORMAL, False)],
 355                         [(constants.RS_NORMAL, "node2"),
 356                          (constants.RS_NORMAL, True)],
 357                         [(constants.RS_NORMAL, "node3"),
 358                          (constants.RS_NORMAL, False)],
 359                        ])
 360       self.assertEqual(q.OldStyleQuery(nqd),
 361                        [["node1", False],
 362                         ["node2", True],
 363                         ["node3", False]])
 364       q = self._Create(["ndp/spindle_count"])
 365       self.assertEqual(q.RequestedData(), set([query.NQ_GROUP]))
 366       self.assertEqual(q.Query(nqd),
 367                        [[(constants.RS_NORMAL,
 368                           constants.NDC_DEFAULTS[constants.ND_SPINDLE_COUNT])],
 369                         [(constants.RS_NORMAL,
 370                           grp2.ndparams[constants.ND_SPINDLE_COUNT])],
 371                         [(constants.RS_NORMAL,
 372                           nodes[2].ndparams[constants.ND_SPINDLE_COUNT])],
 373                        ])
 374
 375   def test(self):
 376     selected = query.NODE_FIELDS.keys()
 377     field_index = dict((field, idx) for idx, field in enumerate(selected))
 378
 379     q = self._Create(selected)
 380     self.assertEqual(q.RequestedData(),
 381                      set([query.NQ_CONFIG, query.NQ_LIVE, query.NQ_INST,
 382                           query.NQ_GROUP, query.NQ_OOB]))
 383
 384     cluster = objects.Cluster(cluster_name="testcluster",
 385       hvparams=constants.HVC_DEFAULTS,
 386       beparams={
 387         constants.PP_DEFAULT: constants.BEC_DEFAULTS,
 388         },
 389       nicparams={
 390         constants.PP_DEFAULT: constants.NICC_DEFAULTS,
 391         },
 392       ndparams=constants.NDC_DEFAULTS,
 393         )
 394
 395     node_names = ["node%s" % i for i in range(20)]
 396     master_name = node_names[3]
 397     nodes = [
 398       objects.Node(name=name,
 399                    primary_ip="192.0.2.%s" % idx,
 400                    secondary_ip="192.0.100.%s" % idx,
 401                    serial_no=7789 * idx,
 402                    master_candidate=(name != master_name and idx % 3 == 0),
 403                    offline=False,
 404                    drained=False,
 405                    powered=True,
 406                    vm_capable=True,
 407                    master_capable=False,
 408                    ndparams={},
 409                    group="default",
 410                    ctime=1290006900,
 411                    mtime=1290006913,
 412                    uuid="fd9ccebe-6339-43c9-a82e-94bbe575%04d" % idx)
 413       for idx, name in enumerate(node_names)
 414       ]
 415
 416     master_node = nodes[3]
 417     master_node.AddTag("masternode")
 418     master_node.AddTag("another")
 419     master_node.AddTag("tag")
 420     master_node.ctime = None
 421     master_node.mtime = None
 422     assert master_node.name == master_name
 423
 424     live_data_name = node_names[4]
 425     assert live_data_name != master_name
 426
 427     fake_live_data = {
 428       "bootid": "a2504766-498e-4b25-b21e-d23098dc3af4",
 429       "cnodes": 4,
 430       "csockets": 4,
 431       "ctotal": 8,
 432       "mnode": 128,
 433       "mfree": 100,
 434       "mtotal": 4096,
 435       "dfree": 5 * 1024 * 1024,
 436       "dtotal": 100 * 1024 * 1024,
 437       }
 438
 439     assert (sorted(query._NODE_LIVE_FIELDS.keys()) ==
 440             sorted(fake_live_data.keys()))
 441
 442     live_data = dict.fromkeys(node_names, {})
 443     live_data[live_data_name] = \
 444       dict((query._NODE_LIVE_FIELDS[name][2], value)
 445            for name, value in fake_live_data.items())
 446
 447     node_to_primary = dict((name, set()) for name in node_names)
 448     node_to_primary[master_name].update(["inst1", "inst2"])
 449
 450     node_to_secondary = dict((name, set()) for name in node_names)
 451     node_to_secondary[live_data_name].update(["instX", "instY", "instZ"])
 452
 453     ng_uuid = "492b4b74-8670-478a-b98d-4c53a76238e6"
 454     groups = {
 455       ng_uuid: objects.NodeGroup(name="ng1", uuid=ng_uuid, ndparams={}),
 456       }
 457
 458     oob_not_powered_node = node_names[0]
 459     nodes[0].powered = False
 460     oob_support = dict((name, False) for name in node_names)
 461     oob_support[master_name] = True
 462     oob_support[oob_not_powered_node] = True
 463
 464     master_node.group = ng_uuid
 465
 466     nqd = query.NodeQueryData(nodes, live_data, master_name,
 467                               node_to_primary, node_to_secondary, groups,
 468                               oob_support, cluster)
 469     result = q.Query(nqd)
 470     self.assert_(compat.all(len(row) == len(selected) for row in result))
 471     self.assertEqual([row[field_index["name"]] for row in result],
 472                      [(constants.RS_NORMAL, name) for name in node_names])
 473
 474     node_to_row = dict((row[field_index["name"]][1], idx)
 475                        for idx, row in enumerate(result))
 476
 477     master_row = result[node_to_row[master_name]]
 478     self.assert_(master_row[field_index["master"]])
 479     self.assert_(master_row[field_index["role"]], "M")
 480     self.assertEqual(master_row[field_index["group"]],
 481                      (constants.RS_NORMAL, "ng1"))
 482     self.assertEqual(master_row[field_index["group.uuid"]],
 483                      (constants.RS_NORMAL, ng_uuid))
 484     self.assertEqual(master_row[field_index["ctime"]],
 485                      (constants.RS_UNAVAIL, None))
 486     self.assertEqual(master_row[field_index["mtime"]],
 487                      (constants.RS_UNAVAIL, None))
 488
 489     self.assert_(row[field_index["pip"]] == node.primary_ip and
 490                  row[field_index["sip"]] == node.secondary_ip and
 491                  set(row[field_index["tags"]]) == node.GetTags() and
 492                  row[field_index["serial_no"]] == node.serial_no and
 493                  row[field_index["role"]] == query._GetNodeRole(node,
 494                                                                 master_name) and
 495                  (node.name == master_name or
 496                   (row[field_index["group"]] == "<unknown>" and
 497                    row[field_index["group.uuid"]] is None and
 498                    row[field_index["ctime"]] == (constants.RS_NORMAL,
 499                                                  node.ctime) and
 500                    row[field_index["mtime"]] == (constants.RS_NORMAL,
 501                                                  node.mtime) and
 502                    row[field_index["powered"]] == (constants.RS_NORMAL,
 503                                                    True))) or
 504                  (node.name == oob_not_powered_node and
 505                   row[field_index["powered"]] == (constants.RS_NORMAL,
 506                                                   False)) or
 507                  row[field_index["powered"]] == (constants.RS_UNAVAIL, None)
 508                  for row, node in zip(result, nodes))
 509
 510     live_data_row = result[node_to_row[live_data_name]]
 511
 512     for (field, value) in fake_live_data.items():
 513       self.assertEqual(live_data_row[field_index[field]],
 514                        (constants.RS_NORMAL, value))
 515
 516     self.assertEqual(master_row[field_index["pinst_cnt"]],
 517                      (constants.RS_NORMAL, 2))
 518     self.assertEqual(live_data_row[field_index["sinst_cnt"]],
 519                      (constants.RS_NORMAL, 3))
 520     self.assertEqual(master_row[field_index["pinst_list"]],
 521                      (constants.RS_NORMAL,
 522                       list(node_to_primary[master_name])))
 523     self.assertEqual(live_data_row[field_index["sinst_list"]],
 524                      (constants.RS_NORMAL,
 525                       list(node_to_secondary[live_data_name])))
 526
 527   def testGetLiveNodeField(self):
 528     nodes = [
 529       objects.Node(name="node1", drained=False, offline=False,
 530                    vm_capable=True),
 531       objects.Node(name="node2", drained=True, offline=False,
 532                    vm_capable=True),
 533       objects.Node(name="node3", drained=False, offline=False,
 534                    vm_capable=True),
 535       objects.Node(name="node4", drained=False, offline=True,
 536                    vm_capable=True),
 537       objects.Node(name="node5", drained=False, offline=False,
 538                    vm_capable=False),
 539       ]
 540     live_data = dict.fromkeys([node.name for node in nodes], {})
 541
 542     # No data
 543     nqd = query.NodeQueryData(None, None, None, None, None, None, None, None)
 544     self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
 545                                              nqd, nodes[0]),
 546                      query._FS_NODATA)
 547
 548     # Missing field
 549     ctx = _QueryData(None, curlive_data={
 550       "some": 1,
 551       "other": 2,
 552       })
 553     self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
 554                                              ctx, nodes[0]),
 555                      query._FS_UNAVAIL)
 556
 557     # Wrong format/datatype
 558     ctx = _QueryData(None, curlive_data={
 559       "hello": ["Hello World"],
 560       "other": 2,
 561       })
 562     self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
 563                                              ctx, nodes[0]),
 564                      query._FS_UNAVAIL)
 565
 566     # Offline node
 567     assert nodes[3].offline
 568     ctx = _QueryData(None, curlive_data={})
 569     self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
 570                                              ctx, nodes[3]),
 571                      query._FS_OFFLINE, None)
 572
 573     # Wrong field type
 574     ctx = _QueryData(None, curlive_data={"hello": 123})
 575     self.assertRaises(AssertionError, query._GetLiveNodeField,
 576                       "hello", constants.QFT_BOOL, ctx, nodes[0])
 577
 578     # Non-vm_capable node
 579     assert not nodes[4].vm_capable
 580     ctx = _QueryData(None, curlive_data={})
 581     self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
 582                                              ctx, nodes[4]),
 583                      query._FS_UNAVAIL, None)
 584
 585
 586 class TestInstanceQuery(unittest.TestCase):
 587   def _Create(self, selected):
 588     return query.Query(query.INSTANCE_FIELDS, selected)
 589
 590   def testSimple(self):
 591     q = self._Create(["name", "be/maxmem", "ip"])
 592     self.assertEqual(q.RequestedData(), set([query.IQ_CONFIG]))
 593
 594     cluster = objects.Cluster(cluster_name="testcluster",
 595       hvparams=constants.HVC_DEFAULTS,
 596       beparams={
 597         constants.PP_DEFAULT: constants.BEC_DEFAULTS,
 598         },
 599       nicparams={
 600         constants.PP_DEFAULT: constants.NICC_DEFAULTS,
 601         },
 602       os_hvp={},
 603       osparams={})
 604
 605     instances = [
 606       objects.Instance(name="inst1", hvparams={}, beparams={}, osparams={},
 607                        nics=[], os="deb1"),
 608       objects.Instance(name="inst2", hvparams={}, nics=[], osparams={},
 609         os="foomoo",
 610         beparams={
 611           constants.BE_MAXMEM: 512,
 612         }),
 613       objects.Instance(name="inst3", hvparams={}, beparams={}, osparams={},
 614         os="dos", nics=[objects.NIC(ip="192.0.2.99", nicparams={})]),
 615       ]
 616
 617     iqd = query.InstanceQueryData(instances, cluster, None, [], [], {},
 618                                   set(), {}, None, None)
 619     self.assertEqual(q.Query(iqd),
 620       [[(constants.RS_NORMAL, "inst1"),
 621         (constants.RS_NORMAL, 128),
 622         (constants.RS_UNAVAIL, None),
 623        ],
 624        [(constants.RS_NORMAL, "inst2"),
 625         (constants.RS_NORMAL, 512),
 626         (constants.RS_UNAVAIL, None),
 627        ],
 628        [(constants.RS_NORMAL, "inst3"),
 629         (constants.RS_NORMAL, 128),
 630         (constants.RS_NORMAL, "192.0.2.99"),
 631        ]])
 632     self.assertEqual(q.OldStyleQuery(iqd),
 633       [["inst1", 128, None],
 634        ["inst2", 512, None],
 635        ["inst3", 128, "192.0.2.99"]])
 636
 637   def test(self):
 638     selected = query.INSTANCE_FIELDS.keys()
 639     fieldidx = dict((field, idx) for idx, field in enumerate(selected))
 640
 641     macs = ["00:11:22:%02x:%02x:%02x" % (i % 255, i % 3, (i * 123) % 255)
 642             for i in range(20)]
 643
 644     q = self._Create(selected)
 645     self.assertEqual(q.RequestedData(),
 646                      set([query.IQ_CONFIG, query.IQ_LIVE, query.IQ_DISKUSAGE,
 647                           query.IQ_CONSOLE, query.IQ_NODES]))
 648
 649     cluster = objects.Cluster(cluster_name="testcluster",
 650       hvparams=constants.HVC_DEFAULTS,
 651       beparams={
 652         constants.PP_DEFAULT: constants.BEC_DEFAULTS,
 653         },
 654       nicparams={
 655         constants.PP_DEFAULT: constants.NICC_DEFAULTS,
 656         },
 657       os_hvp={},
 658       tcpudp_port_pool=set(),
 659       osparams={
 660         "deb99": {
 661           "clean_install": "yes",
 662           },
 663         })
 664
 665     offline_nodes = ["nodeoff1", "nodeoff2"]
 666     bad_nodes = ["nodebad1", "nodebad2", "nodebad3"] + offline_nodes
 667     nodes = ["node%s" % i for i in range(10)] + bad_nodes
 668
 669     instances = [
 670       objects.Instance(name="inst1", hvparams={}, beparams={}, nics=[],
 671         uuid="f90eccb3-e227-4e3c-bf2a-94a21ca8f9cd",
 672         ctime=1291244000, mtime=1291244400, serial_no=30,
 673         admin_state=constants.ADMINST_UP, hypervisor=constants.HT_XEN_PVM,
 674         os="linux1",
 675         primary_node="node1",
 676         disk_template=constants.DT_PLAIN,
 677         disks=[],
 678         osparams={}),
 679       objects.Instance(name="inst2", hvparams={}, nics=[],
 680         uuid="73a0f8a7-068c-4630-ada2-c3440015ab1a",
 681         ctime=1291211000, mtime=1291211077, serial_no=1,
 682         admin_state=constants.ADMINST_UP, hypervisor=constants.HT_XEN_HVM,
 683         os="deb99",
 684         primary_node="node5",
 685         disk_template=constants.DT_DISKLESS,
 686         disks=[],
 687         beparams={
 688           constants.BE_MAXMEM: 512,
 689           constants.BE_MINMEM: 256,
 690         },
 691         osparams={}),
 692       objects.Instance(name="inst3", hvparams={}, beparams={},
 693         uuid="11ec8dff-fb61-4850-bfe0-baa1803ff280",
 694         ctime=1291011000, mtime=1291013000, serial_no=1923,
 695         admin_state=constants.ADMINST_DOWN, hypervisor=constants.HT_KVM,
 696         os="busybox",
 697         primary_node="node6",
 698         disk_template=constants.DT_DRBD8,
 699         disks=[],
 700         nics=[
 701           objects.NIC(ip="192.0.2.99", mac=macs.pop(),
 702                       nicparams={
 703                         constants.NIC_LINK: constants.DEFAULT_BRIDGE,
 704                         }),
 705           objects.NIC(ip=None, mac=macs.pop(), nicparams={}),
 706           ],
 707         osparams={}),
 708       objects.Instance(name="inst4", hvparams={}, beparams={},
 709         uuid="68dab168-3ef5-4c9d-b4d3-801e0672068c",
 710         ctime=1291244390, mtime=1291244395, serial_no=25,
 711         admin_state=constants.ADMINST_DOWN, hypervisor=constants.HT_XEN_PVM,
 712         os="linux1",
 713         primary_node="nodeoff2",
 714         disk_template=constants.DT_DRBD8,
 715         disks=[],
 716         nics=[
 717           objects.NIC(ip="192.0.2.1", mac=macs.pop(),
 718                       nicparams={
 719                         constants.NIC_LINK: constants.DEFAULT_BRIDGE,
 720                         }),
 721           objects.NIC(ip="192.0.2.2", mac=macs.pop(), nicparams={}),
 722           objects.NIC(ip="192.0.2.3", mac=macs.pop(),
 723                       nicparams={
 724                         constants.NIC_MODE: constants.NIC_MODE_ROUTED,
 725                         }),
 726           objects.NIC(ip="192.0.2.4", mac=macs.pop(),
 727                       nicparams={
 728                         constants.NIC_MODE: constants.NIC_MODE_BRIDGED,
 729                         constants.NIC_LINK: "eth123",
 730                         }),
 731           ],
 732         osparams={}),
 733       objects.Instance(name="inst5", hvparams={}, nics=[],
 734         uuid="0e3dca12-5b42-4e24-98a2-415267545bd0",
 735         ctime=1231211000, mtime=1261200000, serial_no=3,
 736         admin_state=constants.ADMINST_UP, hypervisor=constants.HT_XEN_HVM,
 737         os="deb99",
 738         primary_node="nodebad2",
 739         disk_template=constants.DT_DISKLESS,
 740         disks=[],
 741         beparams={
 742           constants.BE_MAXMEM: 512,
 743           constants.BE_MINMEM: 512,
 744         },
 745         osparams={}),
 746       objects.Instance(name="inst6", hvparams={}, nics=[],
 747         uuid="72de6580-c8d5-4661-b902-38b5785bb8b3",
 748         ctime=7513, mtime=11501, serial_no=13390,
 749         admin_state=constants.ADMINST_DOWN, hypervisor=constants.HT_XEN_HVM,
 750         os="deb99",
 751         primary_node="node7",
 752         disk_template=constants.DT_DISKLESS,
 753         disks=[],
 754         beparams={
 755           constants.BE_MAXMEM: 768,
 756           constants.BE_MINMEM: 256,
 757         },
 758         osparams={
 759           "clean_install": "no",
 760           }),
 761       objects.Instance(name="inst7", hvparams={}, nics=[],
 762         uuid="ceec5dc4-b729-4f42-ae28-69b3cd24920e",
 763         ctime=None, mtime=None, serial_no=1947,
 764         admin_state=constants.ADMINST_DOWN, hypervisor=constants.HT_XEN_HVM,
 765         os="deb99",
 766         primary_node="node6",
 767         disk_template=constants.DT_DISKLESS,
 768         disks=[],
 769         beparams={},
 770         osparams={}),
 771       objects.Instance(name="inst8", hvparams={}, nics=[],
 772         uuid="ceec5dc4-b729-4f42-ae28-69b3cd24920f",
 773         ctime=None, mtime=None, serial_no=19478,
 774         admin_state=constants.ADMINST_OFFLINE, hypervisor=constants.HT_XEN_HVM,
 775         os="deb99",
 776         primary_node="node6",
 777         disk_template=constants.DT_DISKLESS,
 778         disks=[],
 779         beparams={},
 780         osparams={}),
 781       ]
 782
 783     assert not utils.FindDuplicates(inst.name for inst in instances)
 784
 785     instbyname = dict((inst.name, inst) for inst in instances)
 786
 787     disk_usage = dict((inst.name,
 788                        cmdlib._ComputeDiskSize(inst.disk_template,
 789                                                [{"size": disk.size}
 790                                                 for disk in inst.disks]))
 791                       for inst in instances)
 792
 793     inst_bridges = {
 794       "inst3": [constants.DEFAULT_BRIDGE, constants.DEFAULT_BRIDGE],
 795       "inst4": [constants.DEFAULT_BRIDGE, constants.DEFAULT_BRIDGE,
 796                 None, "eth123"],
 797       }
 798
 799     live_data = {
 800       "inst2": {
 801         "vcpus": 3,
 802         },
 803       "inst4": {
 804         "memory": 123,
 805         },
 806       "inst6": {
 807         "memory": 768,
 808         },
 809       "inst7": {
 810         "vcpus": 3,
 811         },
 812       }
 813     wrongnode_inst = set(["inst7"])
 814
 815     consinfo = dict((inst.name, None) for inst in instances)
 816     consinfo["inst7"] = \
 817       objects.InstanceConsole(instance="inst7", kind=constants.CONS_SSH,
 818                               host=instbyname["inst7"].primary_node,
 819                               user=constants.GANETI_RUNAS,
 820                               command=["hostname"]).ToDict()
 821
 822     iqd = query.InstanceQueryData(instances, cluster, disk_usage,
 823                                   offline_nodes, bad_nodes, live_data,
 824                                   wrongnode_inst, consinfo, {}, {})
 825     result = q.Query(iqd)
 826     self.assertEqual(len(result), len(instances))
 827     self.assert_(compat.all(len(row) == len(selected)
 828                             for row in result))
 829
 830     assert len(set(bad_nodes) & set(offline_nodes)) == len(offline_nodes), \
 831            "Offline nodes not included in bad nodes"
 832
 833     tested_status = set()
 834
 835     for (inst, row) in zip(instances, result):
 836       assert inst.primary_node in nodes
 837
 838       self.assertEqual(row[fieldidx["name"]],
 839                        (constants.RS_NORMAL, inst.name))
 840
 841       if inst.primary_node in offline_nodes:
 842         exp_status = constants.INSTST_NODEOFFLINE
 843       elif inst.primary_node in bad_nodes:
 844         exp_status = constants.INSTST_NODEDOWN
 845       elif inst.name in live_data:
 846         if inst.name in wrongnode_inst:
 847           exp_status = constants.INSTST_WRONGNODE
 848         elif inst.admin_state == constants.ADMINST_UP:
 849           exp_status = constants.INSTST_RUNNING
 850         else:
 851           exp_status = constants.INSTST_ERRORUP
 852       elif inst.admin_state == constants.ADMINST_UP:
 853         exp_status = constants.INSTST_ERRORDOWN
 854       elif inst.admin_state == constants.ADMINST_DOWN:
 855         exp_status = constants.INSTST_ADMINDOWN
 856       else:
 857         exp_status = constants.INSTST_ADMINOFFLINE
 858
 859       self.assertEqual(row[fieldidx["status"]],
 860                        (constants.RS_NORMAL, exp_status))
 861
 862       (_, status) = row[fieldidx["status"]]
 863       tested_status.add(status)
 864
 865       #FIXME(dynmem): check oper_ram vs min/max mem
 866       for (field, livefield) in [("oper_vcpus", "vcpus")]:
 867         if inst.primary_node in bad_nodes:
 868           exp = (constants.RS_NODATA, None)
 869         elif inst.name in live_data:
 870           value = live_data[inst.name].get(livefield, None)
 871           if value is None:
 872             exp = (constants.RS_UNAVAIL, None)
 873           else:
 874             exp = (constants.RS_NORMAL, value)
 875         else:
 876           exp = (constants.RS_UNAVAIL, None)
 877
 878         self.assertEqual(row[fieldidx[field]], exp)
 879
 880       bridges = inst_bridges.get(inst.name, [])
 881       self.assertEqual(row[fieldidx["nic.bridges"]],
 882                        (constants.RS_NORMAL, bridges))
 883       if bridges:
 884         self.assertEqual(row[fieldidx["bridge"]],
 885                          (constants.RS_NORMAL, bridges[0]))
 886       else:
 887         self.assertEqual(row[fieldidx["bridge"]],
 888                          (constants.RS_UNAVAIL, None))
 889
 890       for i in range(constants.MAX_NICS):
 891         if i < len(bridges) and bridges[i] is not None:
 892           exp = (constants.RS_NORMAL, bridges[i])
 893         else:
 894           exp = (constants.RS_UNAVAIL, None)
 895         self.assertEqual(row[fieldidx["nic.bridge/%s" % i]], exp)
 896
 897       if inst.primary_node in bad_nodes:
 898         exp = (constants.RS_NODATA, None)
 899       else:
 900         exp = (constants.RS_NORMAL, inst.name in live_data)
 901       self.assertEqual(row[fieldidx["oper_state"]], exp)
 902
 903       cust_exp = (constants.RS_NORMAL, {})
 904       if inst.os == "deb99":
 905         if inst.name == "inst6":
 906           exp = (constants.RS_NORMAL, {"clean_install": "no"})
 907           cust_exp = exp
 908         else:
 909           exp = (constants.RS_NORMAL, {"clean_install": "yes"})
 910       else:
 911         exp = (constants.RS_NORMAL, {})
 912       self.assertEqual(row[fieldidx["osparams"]], exp)
 913       self.assertEqual(row[fieldidx["custom_osparams"]], cust_exp)
 914
 915       usage = disk_usage[inst.name]
 916       if usage is None:
 917         usage = 0
 918       self.assertEqual(row[fieldidx["disk_usage"]],
 919                        (constants.RS_NORMAL, usage))
 920
 921       for alias, target in [("sda_size", "disk.size/0"),
 922                             ("sdb_size", "disk.size/1"),
 923                             ("vcpus", "be/vcpus"),
 924                             ("ip", "nic.ip/0"),
 925                             ("mac", "nic.mac/0"),
 926                             ("bridge", "nic.bridge/0"),
 927                             ("nic_mode", "nic.mode/0"),
 928                             ("nic_link", "nic.link/0"),
 929                             ]:
 930         self.assertEqual(row[fieldidx[alias]], row[fieldidx[target]])
 931
 932       for field in ["ctime", "mtime"]:
 933         if getattr(inst, field) is None:
 934           # No ctime/mtime
 935           exp = (constants.RS_UNAVAIL, None)
 936         else:
 937           exp = (constants.RS_NORMAL, getattr(inst, field))
 938         self.assertEqual(row[fieldidx[field]], exp)
 939
 940       self._CheckInstanceConsole(inst, row[fieldidx["console"]])
 941
 942     # Ensure all possible status' have been tested
 943     self.assertEqual(tested_status, constants.INSTST_ALL)
 944
 945   def _CheckInstanceConsole(self, instance, (status, consdata)):
 946     if instance.name == "inst7":
 947       self.assertEqual(status, constants.RS_NORMAL)
 948       console = objects.InstanceConsole.FromDict(consdata)
 949       self.assertTrue(console.Validate())
 950       self.assertEqual(console.host, instance.primary_node)
 951     else:
 952       self.assertEqual(status, constants.RS_UNAVAIL)
 953
 954
 955 class TestGroupQuery(unittest.TestCase):
 956
 957   def setUp(self):
 958     self.custom_diskparams = {
 959       constants.DT_DRBD8: {
 960         constants.DRBD_DEFAULT_METAVG: "foobar",
 961       },
 962     }
 963
 964     self.groups = [
 965       objects.NodeGroup(name="default",
 966                         uuid="c0e89160-18e7-11e0-a46e-001d0904baeb",
 967                         alloc_policy=constants.ALLOC_POLICY_PREFERRED,
 968                         ipolicy=objects.MakeEmptyIPolicy(),
 969                         ndparams={},
 970                         diskparams={},
 971                         ),
 972       objects.NodeGroup(name="restricted",
 973                         uuid="d2a40a74-18e7-11e0-9143-001d0904baeb",
 974                         alloc_policy=constants.ALLOC_POLICY_LAST_RESORT,
 975                         ipolicy=objects.MakeEmptyIPolicy(),
 976                         ndparams={},
 977                         diskparams=self.custom_diskparams,
 978                         ),
 979       ]
 980     self.cluster = objects.Cluster(cluster_name="testcluster",
 981       hvparams=constants.HVC_DEFAULTS,
 982       beparams={
 983         constants.PP_DEFAULT: constants.BEC_DEFAULTS,
 984         },
 985       nicparams={
 986         constants.PP_DEFAULT: constants.NICC_DEFAULTS,
 987         },
 988       ndparams=constants.NDC_DEFAULTS,
 989       ipolicy=constants.IPOLICY_DEFAULTS,
 990       diskparams=constants.DISK_DT_DEFAULTS,
 991       )
 992
 993   def _Create(self, selected):
 994     return query.Query(query.GROUP_FIELDS, selected)
 995
 996   def testSimple(self):
 997     q = self._Create(["name", "uuid", "alloc_policy"])
 998     gqd = query.GroupQueryData(self.cluster, self.groups, None, None, False)
 999
1000     self.assertEqual(q.RequestedData(), set([query.GQ_CONFIG]))
1001
1002     self.assertEqual(q.Query(gqd),
1003       [[(constants.RS_NORMAL, "default"),
1004         (constants.RS_NORMAL, "c0e89160-18e7-11e0-a46e-001d0904baeb"),
1005         (constants.RS_NORMAL, constants.ALLOC_POLICY_PREFERRED)
1006         ],
1007        [(constants.RS_NORMAL, "restricted"),
1008         (constants.RS_NORMAL, "d2a40a74-18e7-11e0-9143-001d0904baeb"),
1009         (constants.RS_NORMAL, constants.ALLOC_POLICY_LAST_RESORT)
1010         ],
1011        ])
1012
1013   def testNodes(self):
1014     groups_to_nodes = {
1015       "c0e89160-18e7-11e0-a46e-001d0904baeb": ["node1", "node2"],
1016       "d2a40a74-18e7-11e0-9143-001d0904baeb": ["node1", "node10", "node9"],
1017       }
1018
1019     q = self._Create(["name", "node_cnt", "node_list"])
1020     gqd = query.GroupQueryData(self.cluster, self.groups, groups_to_nodes, None,
1021                                False)
1022
1023     self.assertEqual(q.RequestedData(), set([query.GQ_CONFIG, query.GQ_NODE]))
1024
1025     self.assertEqual(q.Query(gqd),
1026                      [[(constants.RS_NORMAL, "default"),
1027                        (constants.RS_NORMAL, 2),
1028                        (constants.RS_NORMAL, ["node1", "node2"]),
1029                        ],
1030                       [(constants.RS_NORMAL, "restricted"),
1031                        (constants.RS_NORMAL, 3),
1032                        (constants.RS_NORMAL, ["node1", "node9", "node10"]),
1033                        ],
1034                       ])
1035
1036   def testInstances(self):
1037     groups_to_instances = {
1038       "c0e89160-18e7-11e0-a46e-001d0904baeb": ["inst1", "inst2"],
1039       "d2a40a74-18e7-11e0-9143-001d0904baeb": ["inst1", "inst10", "inst9"],
1040       }
1041
1042     q = self._Create(["pinst_cnt", "pinst_list"])
1043     gqd = query.GroupQueryData(self.cluster, self.groups, None,
1044       groups_to_instances, False)
1045
1046     self.assertEqual(q.RequestedData(), set([query.GQ_INST]))
1047
1048     self.assertEqual(q.Query(gqd),
1049                      [[(constants.RS_NORMAL, 2),
1050                        (constants.RS_NORMAL, ["inst1", "inst2"]),
1051                        ],
1052                       [(constants.RS_NORMAL, 3),
1053                        (constants.RS_NORMAL, ["inst1", "inst9", "inst10"]),
1054                        ],
1055                       ])
1056
1057   def testDiskparams(self):
1058     q = self._Create(["name", "uuid", "diskparams", "custom_diskparams"])
1059     gqd = query.GroupQueryData(self.cluster, self.groups, None, None, True)
1060
1061     self.assertEqual(q.RequestedData(),
1062                      set([query.GQ_CONFIG, query.GQ_DISKPARAMS]))
1063
1064     self.assertEqual(q.Query(gqd),
1065       [[(constants.RS_NORMAL, "default"),
1066         (constants.RS_NORMAL, "c0e89160-18e7-11e0-a46e-001d0904baeb"),
1067         (constants.RS_NORMAL, constants.DISK_DT_DEFAULTS),
1068         (constants.RS_NORMAL, {}),
1069         ],
1070        [(constants.RS_NORMAL, "restricted"),
1071         (constants.RS_NORMAL, "d2a40a74-18e7-11e0-9143-001d0904baeb"),
1072         (constants.RS_NORMAL, objects.FillDiskParams(constants.DISK_DT_DEFAULTS,
1073                                                      self.custom_diskparams)),
1074         (constants.RS_NORMAL, self.custom_diskparams),
1075         ],
1076        ])
1077
1078
1079 class TestOsQuery(unittest.TestCase):
1080   def _Create(self, selected):
1081     return query.Query(query.OS_FIELDS, selected)
1082
1083   def test(self):
1084     variants = ["v00", "plain", "v3", "var0", "v33", "v20"]
1085     api_versions = [10, 0, 15, 5]
1086     parameters = ["zpar3", "apar9"]
1087
1088     assert variants != sorted(variants) and variants != utils.NiceSort(variants)
1089     assert (api_versions != sorted(api_versions) and
1090             api_versions != utils.NiceSort(variants))
1091     assert (parameters != sorted(parameters) and
1092             parameters != utils.NiceSort(parameters))
1093
1094     data = [
1095       query.OsInfo(name="debian", valid=False, hidden=False, blacklisted=False,
1096                    variants=set(), api_versions=set(), parameters=set(),
1097                    node_status={ "some": "status", }),
1098       query.OsInfo(name="dos", valid=True, hidden=False, blacklisted=True,
1099                    variants=set(variants),
1100                    api_versions=set(api_versions),
1101                    parameters=set(parameters),
1102                    node_status={ "some": "other", "status": None, }),
1103       ]
1104
1105
1106     q = self._Create(["name", "valid", "hidden", "blacklisted", "variants",
1107                       "api_versions", "parameters", "node_status"])
1108     self.assertEqual(q.RequestedData(), set([]))
1109     self.assertEqual(q.Query(data),
1110                      [[(constants.RS_NORMAL, "debian"),
1111                        (constants.RS_NORMAL, False),
1112                        (constants.RS_NORMAL, False),
1113                        (constants.RS_NORMAL, False),
1114                        (constants.RS_NORMAL, []),
1115                        (constants.RS_NORMAL, []),
1116                        (constants.RS_NORMAL, []),
1117                        (constants.RS_NORMAL, {"some": "status"})],
1118                       [(constants.RS_NORMAL, "dos"),
1119                        (constants.RS_NORMAL, True),
1120                        (constants.RS_NORMAL, False),
1121                        (constants.RS_NORMAL, True),
1122                        (constants.RS_NORMAL,
1123                         ["plain", "v00", "v3", "v20", "v33", "var0"]),
1124                        (constants.RS_NORMAL, [0, 5, 10, 15]),
1125                        (constants.RS_NORMAL, ["apar9", "zpar3"]),
1126                        (constants.RS_NORMAL,
1127                         { "some": "other", "status": None, })
1128                        ]])
1129
1130
1131 class TestQueryFields(unittest.TestCase):
1132   def testAllFields(self):
1133     for fielddefs in query.ALL_FIELD_LISTS:
1134       result = query.QueryFields(fielddefs, None)
1135       self.assert_(isinstance(result, dict))
1136       response = objects.QueryFieldsResponse.FromDict(result)
1137       self.assertEqual([(fdef.name, fdef.title) for fdef in response.fields],
1138         [(fdef2.name, fdef2.title)
1139          for (fdef2, _, _, _) in utils.NiceSort(fielddefs.values(),
1140                                                 key=lambda x: x[0].name)])
1141
1142   def testSomeFields(self):
1143     rnd = random.Random(5357)
1144
1145     for _ in range(10):
1146       for fielddefs in query.ALL_FIELD_LISTS:
1147         if len(fielddefs) > 20:
1148           sample_size = rnd.randint(5, 20)
1149         else:
1150           sample_size = rnd.randint(1, max(1, len(fielddefs) - 1))
1151         fields = [fdef for (fdef, _, _, _) in rnd.sample(fielddefs.values(),
1152                                                          sample_size)]
1153         result = query.QueryFields(fielddefs, [fdef.name for fdef in fields])
1154         self.assert_(isinstance(result, dict))
1155         response = objects.QueryFieldsResponse.FromDict(result)
1156         self.assertEqual([(fdef.name, fdef.title) for fdef in response.fields],
1157                          [(fdef2.name, fdef2.title) for fdef2 in fields])
1158
1159
1160 class TestQueryFilter(unittest.TestCase):
1161   def testRequestedNames(self):
1162     for (what, fielddefs) in query.ALL_FIELDS.items():
1163       if what == constants.QR_JOB:
1164         namefield = "id"
1165         nameval = 123
1166         namevalempty = 0
1167         genval = lambda i: i * 10
1168         randvals = [17361, 22015, 13193, 15215]
1169       else:
1170         nameval = "abc"
1171         namevalempty = ""
1172         genval = lambda i: "x%s" % i
1173         randvals = ["x17361", "x22015", "x13193", "x15215"]
1174         if what == constants.QR_EXPORT:
1175           namefield = "export"
1176         else:
1177           namefield = "name"
1178
1179       assert namefield in fielddefs
1180
1181       reqnames = [genval(i) for i in range(4)]
1182       innerfilter = [["=", namefield, v] for v in reqnames]
1183
1184       # No name field
1185       q = query.Query(fielddefs, [namefield],
1186                       qfilter=["=", namefield, nameval], namefield=None)
1187       self.assertEqual(q.RequestedNames(), None)
1188
1189       # No filter
1190       q = query.Query(fielddefs, [namefield], qfilter=None, namefield=namefield)
1191       self.assertEqual(q.RequestedNames(), None)
1192
1193       # Check empty query
1194       q = query.Query(fielddefs, [namefield], qfilter=["|"],
1195                       namefield=namefield)
1196       self.assertEqual(q.RequestedNames(), None)
1197
1198       # Check order
1199       q = query.Query(fielddefs, [namefield], qfilter=["|"] + innerfilter,
1200                       namefield=namefield)
1201       self.assertEqual(q.RequestedNames(), reqnames)
1202
1203       # Check reverse order
1204       q = query.Query(fielddefs, [namefield],
1205                       qfilter=["|"] + list(reversed(innerfilter)),
1206                       namefield=namefield)
1207       self.assertEqual(q.RequestedNames(), list(reversed(reqnames)))
1208
1209       # Duplicates
1210       q = query.Query(fielddefs, [namefield],
1211                       qfilter=["|"] + innerfilter + list(reversed(innerfilter)),
1212                       namefield=namefield)
1213       self.assertEqual(q.RequestedNames(), reqnames)
1214
1215       # Unknown name field
1216       self.assertRaises(AssertionError, query.Query, fielddefs, [namefield],
1217                         namefield="_unknown_field_")
1218
1219       # Filter with AND
1220       q = query.Query(fielddefs, [namefield],
1221                       qfilter=["|", ["=", namefield, nameval],
1222                                     ["&", ["=", namefield, namevalempty]]],
1223                       namefield=namefield)
1224       self.assertTrue(q.RequestedNames() is None)
1225
1226       # Filter with NOT
1227       q = query.Query(fielddefs, [namefield],
1228                       qfilter=["|", ["=", namefield, nameval],
1229                                     ["!", ["=", namefield, namevalempty]]],
1230                       namefield=namefield)
1231       self.assertTrue(q.RequestedNames() is None)
1232
1233       # Filter with only OR (names must be in correct order)
1234       q = query.Query(fielddefs, [namefield],
1235                       qfilter=["|", ["=", namefield, randvals[0]],
1236                                     ["|", ["=", namefield, randvals[1]]],
1237                                     ["|", ["|", ["=", namefield, randvals[2]]]],
1238                                     ["=", namefield, randvals[3]]],
1239                       namefield=namefield)
1240       self.assertEqual(q.RequestedNames(), randvals)
1241
1242   @staticmethod
1243   def _GenNestedFilter(namefield, op, depth, nameval):
1244     nested = ["=", namefield, nameval]
1245     for i in range(depth):
1246       nested = [op, nested]
1247     return nested
1248
1249   def testCompileFilter(self):
1250     levels_max = query._FilterCompilerHelper._LEVELS_MAX
1251
1252     for (what, fielddefs) in query.ALL_FIELDS.items():
1253       if what == constants.QR_JOB:
1254         namefield = "id"
1255         nameval = 123
1256       elif what == constants.QR_EXPORT:
1257         namefield = "export"
1258         nameval = "value"
1259       else:
1260         namefield = "name"
1261         nameval = "value"
1262
1263       checks = [
1264         [], ["="], ["=", "foo"], ["unknownop"], ["!"],
1265         ["=", "_unknown_field", "value"],
1266         self._GenNestedFilter(namefield, "|", levels_max, nameval),
1267         self._GenNestedFilter(namefield, "|", levels_max * 3, nameval),
1268         self._GenNestedFilter(namefield, "!", levels_max, nameval),
1269         ]
1270
1271       for qfilter in checks:
1272         self.assertRaises(errors.ParameterError, query._CompileFilter,
1273                           fielddefs, None, qfilter)
1274
1275       for op in ["|", "!"]:
1276         qfilter = self._GenNestedFilter(namefield, op, levels_max - 1, nameval)
1277         self.assertTrue(callable(query._CompileFilter(fielddefs, None,
1278                                                       qfilter)))
1279
1280   def testQueryInputOrder(self):
1281     fielddefs = query._PrepareFieldList([
1282       (query._MakeField("pnode", "PNode", constants.QFT_TEXT, "Primary"),
1283        None, 0, lambda ctx, item: item["pnode"]),
1284       (query._MakeField("snode", "SNode", constants.QFT_TEXT, "Secondary"),
1285        None, 0, lambda ctx, item: item["snode"]),
1286       ], [])
1287
1288     data = [
1289       { "pnode": "node1", "snode": "node44", },
1290       { "pnode": "node30", "snode": "node90", },
1291       { "pnode": "node25", "snode": "node1", },
1292       { "pnode": "node20", "snode": "node1", },
1293       ]
1294
1295     qfilter = ["|", ["=", "pnode", "node1"], ["=", "snode", "node1"]]
1296
1297     q = query.Query(fielddefs, ["pnode", "snode"], namefield="pnode",
1298                     qfilter=qfilter)
1299     self.assertTrue(q.RequestedNames() is None)
1300     self.assertFalse(q.RequestedData())
1301     self.assertEqual(q.Query(data),
1302       [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")],
1303        [(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")],
1304        [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")]])
1305
1306     # Try again with reversed input data
1307     self.assertEqual(q.Query(reversed(data)),
1308       [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")],
1309        [(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")],
1310        [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")]])
1311
1312     # No name field, result must be in incoming order
1313     q = query.Query(fielddefs, ["pnode", "snode"], namefield=None,
1314                     qfilter=qfilter)
1315     self.assertFalse(q.RequestedData())
1316     self.assertEqual(q.Query(data),
1317       [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")],
1318        [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")],
1319        [(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")]])
1320     self.assertEqual(q.OldStyleQuery(data), [
1321       ["node1", "node44"],
1322       ["node25", "node1"],
1323       ["node20", "node1"],
1324       ])
1325     self.assertEqual(q.Query(reversed(data)),
1326       [[(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")],
1327        [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")],
1328        [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")]])
1329     self.assertEqual(q.OldStyleQuery(reversed(data)), [
1330       ["node20", "node1"],
1331       ["node25", "node1"],
1332       ["node1", "node44"],
1333       ])
1334
1335     # Name field, but no sorting, result must be in incoming order
1336     q = query.Query(fielddefs, ["pnode", "snode"], namefield="pnode")
1337     self.assertFalse(q.RequestedData())
1338     self.assertEqual(q.Query(data, sort_by_name=False),
1339       [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")],
1340        [(constants.RS_NORMAL, "node30"), (constants.RS_NORMAL, "node90")],
1341        [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")],
1342        [(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")]])
1343     self.assertEqual(q.OldStyleQuery(data, sort_by_name=False), [
1344       ["node1", "node44"],
1345       ["node30", "node90"],
1346       ["node25", "node1"],
1347       ["node20", "node1"],
1348       ])
1349     self.assertEqual(q.Query(reversed(data), sort_by_name=False),
1350       [[(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")],
1351        [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")],
1352        [(constants.RS_NORMAL, "node30"), (constants.RS_NORMAL, "node90")],
1353        [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")]])
1354     self.assertEqual(q.OldStyleQuery(reversed(data), sort_by_name=False), [
1355       ["node20", "node1"],
1356       ["node25", "node1"],
1357       ["node30", "node90"],
1358       ["node1", "node44"],
1359       ])
1360
1361   def testEqualNamesOrder(self):
1362     fielddefs = query._PrepareFieldList([
1363       (query._MakeField("pnode", "PNode", constants.QFT_TEXT, "Primary"),
1364        None, 0, lambda ctx, item: item["pnode"]),
1365       (query._MakeField("num", "Num", constants.QFT_NUMBER, "Num"),
1366        None, 0, lambda ctx, item: item["num"]),
1367       ], [])
1368
1369     data = [
1370       { "pnode": "node1", "num": 100, },
1371       { "pnode": "node1", "num": 25, },
1372       { "pnode": "node2", "num": 90, },
1373       { "pnode": "node2", "num": 30, },
1374       ]
1375
1376     q = query.Query(fielddefs, ["pnode", "num"], namefield="pnode",
1377                     qfilter=["|", ["=", "pnode", "node1"],
1378                                   ["=", "pnode", "node2"],
1379                                   ["=", "pnode", "node1"]])
1380     self.assertEqual(q.RequestedNames(), ["node1", "node2"],
1381                      msg="Did not return unique names")
1382     self.assertFalse(q.RequestedData())
1383     self.assertEqual(q.Query(data),
1384       [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, 100)],
1385        [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, 25)],
1386        [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, 90)],
1387        [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, 30)]])
1388     self.assertEqual(q.Query(data, sort_by_name=False),
1389       [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, 100)],
1390        [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, 25)],
1391        [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, 90)],
1392        [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, 30)]])
1393
1394     data = [
1395       { "pnode": "nodeX", "num": 50, },
1396       { "pnode": "nodeY", "num": 40, },
1397       { "pnode": "nodeX", "num": 30, },
1398       { "pnode": "nodeX", "num": 20, },
1399       { "pnode": "nodeM", "num": 10, },
1400       ]
1401
1402     q = query.Query(fielddefs, ["pnode", "num"], namefield="pnode",
1403                     qfilter=["|", ["=", "pnode", "nodeX"],
1404                                   ["=", "pnode", "nodeY"],
1405                                   ["=", "pnode", "nodeY"],
1406                                   ["=", "pnode", "nodeY"],
1407                                   ["=", "pnode", "nodeM"]])
1408     self.assertEqual(q.RequestedNames(), ["nodeX", "nodeY", "nodeM"],
1409                      msg="Did not return unique names")
1410     self.assertFalse(q.RequestedData())
1411
1412     # First sorted by name, then input order
1413     self.assertEqual(q.Query(data, sort_by_name=True),
1414       [[(constants.RS_NORMAL, "nodeM"), (constants.RS_NORMAL, 10)],
1415        [(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 50)],
1416        [(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 30)],
1417        [(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 20)],
1418        [(constants.RS_NORMAL, "nodeY"), (constants.RS_NORMAL, 40)]])
1419
1420     # Input order
1421     self.assertEqual(q.Query(data, sort_by_name=False),
1422       [[(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 50)],
1423        [(constants.RS_NORMAL, "nodeY"), (constants.RS_NORMAL, 40)],
1424        [(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 30)],
1425        [(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 20)],
1426        [(constants.RS_NORMAL, "nodeM"), (constants.RS_NORMAL, 10)]])
1427
1428   def testFilter(self):
1429     (DK_A, DK_B) = range(1000, 1002)
1430
1431     fielddefs = query._PrepareFieldList([
1432       (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"),
1433        DK_A, 0, lambda ctx, item: item["name"]),
1434       (query._MakeField("other", "Other", constants.QFT_TEXT, "Other"),
1435        DK_B, 0, lambda ctx, item: item["other"]),
1436       ], [])
1437
1438     data = [
1439       { "name": "node1", "other": "foo", },
1440       { "name": "node2", "other": "bar", },
1441       { "name": "node3", "other": "Hello", },
1442       ]
1443
1444     # Empty filter
1445     q = query.Query(fielddefs, ["name", "other"], namefield="name",
1446                     qfilter=["|"])
1447     self.assertTrue(q.RequestedNames() is None)
1448     self.assertEqual(q.RequestedData(), set([DK_A, DK_B]))
1449     self.assertEqual(q.Query(data), [])
1450
1451     # Normal filter
1452     q = query.Query(fielddefs, ["name", "other"], namefield="name",
1453                     qfilter=["=", "name", "node1"])
1454     self.assertEqual(q.RequestedNames(), ["node1"])
1455     self.assertEqual(q.Query(data),
1456       [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "foo")]])
1457
1458     q = query.Query(fielddefs, ["name", "other"], namefield="name",
1459                     qfilter=(["|", ["=", "name", "node1"],
1460                                    ["=", "name", "node3"]]))
1461     self.assertEqual(q.RequestedNames(), ["node1", "node3"])
1462     self.assertEqual(q.Query(data),
1463       [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "foo")],
1464        [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, "Hello")]])
1465
1466     # Complex filter
1467     q = query.Query(fielddefs, ["name", "other"], namefield="name",
1468                     qfilter=(["|", ["=", "name", "node1"],
1469                                    ["|", ["=", "name", "node3"],
1470                                          ["=", "name", "node2"]],
1471                                    ["=", "name", "node3"]]))
1472     self.assertEqual(q.RequestedNames(), ["node1", "node3", "node2"])
1473     self.assertEqual(q.RequestedData(), set([DK_A, DK_B]))
1474     self.assertEqual(q.Query(data),
1475       [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "foo")],
1476        [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, "bar")],
1477        [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, "Hello")]])
1478
1479     # Filter data type mismatch
1480     for i in [-1, 0, 1, 123, [], None, True, False]:
1481       self.assertRaises(errors.ParameterError, query.Query,
1482                         fielddefs, ["name", "other"], namefield="name",
1483                         qfilter=["=", "name", i])
1484
1485     # Negative filter
1486     q = query.Query(fielddefs, ["name", "other"], namefield="name",
1487                     qfilter=["!", ["|", ["=", "name", "node1"],
1488                                         ["=", "name", "node3"]]])
1489     self.assertTrue(q.RequestedNames() is None)
1490     self.assertEqual(q.Query(data),
1491       [[(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, "bar")]])
1492
1493     # Not equal
1494     q = query.Query(fielddefs, ["name", "other"], namefield="name",
1495                     qfilter=["!=", "name", "node3"])
1496     self.assertTrue(q.RequestedNames() is None)
1497     self.assertEqual(q.Query(data),
1498       [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "foo")],
1499        [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, "bar")]])
1500
1501     # Data type
1502     q = query.Query(fielddefs, [], namefield="name",
1503                     qfilter=["|", ["=", "other", "bar"],
1504                                   ["=", "name", "foo"]])
1505     self.assertTrue(q.RequestedNames() is None)
1506     self.assertEqual(q.RequestedData(), set([DK_A, DK_B]))
1507     self.assertEqual(q.Query(data), [[]])
1508
1509     # Only one data type
1510     q = query.Query(fielddefs, ["other"], namefield="name",
1511                     qfilter=["=", "other", "bar"])
1512     self.assertTrue(q.RequestedNames() is None)
1513     self.assertEqual(q.RequestedData(), set([DK_B]))
1514     self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, "bar")]])
1515
1516     q = query.Query(fielddefs, [], namefield="name",
1517                     qfilter=["=", "other", "bar"])
1518     self.assertTrue(q.RequestedNames() is None)
1519     self.assertEqual(q.RequestedData(), set([DK_B]))
1520     self.assertEqual(q.Query(data), [[]])
1521
1522   def testFilterContains(self):
1523     fielddefs = query._PrepareFieldList([
1524       (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"),
1525        None, 0, lambda ctx, item: item["name"]),
1526       (query._MakeField("other", "Other", constants.QFT_OTHER, "Other"),
1527        None, 0, lambda ctx, item: item["other"]),
1528       ], [])
1529
1530     data = [
1531       { "name": "node2", "other": ["x", "y", "bar"], },
1532       { "name": "node3", "other": "Hello", },
1533       { "name": "node1", "other": ["a", "b", "foo"], },
1534       { "name": "empty", "other": []},
1535       ]
1536
1537     q = query.Query(fielddefs, ["name", "other"], namefield="name",
1538                     qfilter=["=[]", "other", "bar"])
1539     self.assertTrue(q.RequestedNames() is None)
1540     self.assertEqual(q.Query(data), [
1541       [(constants.RS_NORMAL, "node2"),
1542        (constants.RS_NORMAL, ["x", "y", "bar"])],
1543       ])
1544
1545     q = query.Query(fielddefs, ["name", "other"], namefield="name",
1546                     qfilter=["|", ["=[]", "other", "bar"],
1547                                   ["=[]", "other", "a"],
1548                                   ["=[]", "other", "b"]])
1549     self.assertTrue(q.RequestedNames() is None)
1550     self.assertEqual(q.Query(data), [
1551       [(constants.RS_NORMAL, "node1"),
1552        (constants.RS_NORMAL, ["a", "b", "foo"])],
1553       [(constants.RS_NORMAL, "node2"),
1554        (constants.RS_NORMAL, ["x", "y", "bar"])],
1555       ])
1556     self.assertEqual(q.OldStyleQuery(data), [
1557       ["node1", ["a", "b", "foo"]],
1558       ["node2", ["x", "y", "bar"]],
1559       ])
1560
1561     # Boolean test
1562     q = query.Query(fielddefs, ["name", "other"], namefield="name",
1563                     qfilter=["?", "other"])
1564     self.assertEqual(q.OldStyleQuery(data), [
1565       ["node1", ["a", "b", "foo"]],
1566       ["node2", ["x", "y", "bar"]],
1567       ["node3", "Hello"],
1568       ])
1569
1570     q = query.Query(fielddefs, ["name", "other"], namefield="name",
1571                     qfilter=["!", ["?", "other"]])
1572     self.assertEqual(q.OldStyleQuery(data), [
1573       ["empty", []],
1574       ])
1575
1576   def testFilterHostname(self):
1577     fielddefs = query._PrepareFieldList([
1578       (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"),
1579        None, query.QFF_HOSTNAME, lambda ctx, item: item["name"]),
1580       ], [])
1581
1582     data = [
1583       { "name": "node1.example.com", },
1584       { "name": "node2.example.com", },
1585       { "name": "node2.example.net", },
1586       ]
1587
1588     q = query.Query(fielddefs, ["name"], namefield="name",
1589                     qfilter=["=", "name", "node2"])
1590     self.assertEqual(q.RequestedNames(), ["node2"])
1591     self.assertEqual(q.Query(data), [
1592       [(constants.RS_NORMAL, "node2.example.com")],
1593       [(constants.RS_NORMAL, "node2.example.net")],
1594       ])
1595
1596     q = query.Query(fielddefs, ["name"], namefield="name",
1597                     qfilter=["=", "name", "node1"])
1598     self.assertEqual(q.RequestedNames(), ["node1"])
1599     self.assertEqual(q.Query(data), [
1600       [(constants.RS_NORMAL, "node1.example.com")],
1601       ])
1602
1603     q = query.Query(fielddefs, ["name"], namefield="name",
1604                     qfilter=["=", "name", "othername"])
1605     self.assertEqual(q.RequestedNames(), ["othername"])
1606     self.assertEqual(q.Query(data), [])
1607
1608     q = query.Query(fielddefs, ["name"], namefield="name",
1609                     qfilter=["|", ["=", "name", "node1.example.com"],
1610                                   ["=", "name", "node2"]])
1611     self.assertEqual(q.RequestedNames(), ["node1.example.com", "node2"])
1612     self.assertEqual(q.Query(data), [
1613       [(constants.RS_NORMAL, "node1.example.com")],
1614       [(constants.RS_NORMAL, "node2.example.com")],
1615       [(constants.RS_NORMAL, "node2.example.net")],
1616       ])
1617     self.assertEqual(q.OldStyleQuery(data), [
1618       ["node1.example.com"],
1619       ["node2.example.com"],
1620       ["node2.example.net"],
1621       ])
1622
1623     q = query.Query(fielddefs, ["name"], namefield="name",
1624                     qfilter=["!=", "name", "node1"])
1625     self.assertTrue(q.RequestedNames() is None)
1626     self.assertEqual(q.Query(data), [
1627       [(constants.RS_NORMAL, "node2.example.com")],
1628       [(constants.RS_NORMAL, "node2.example.net")],
1629       ])
1630     self.assertEqual(q.OldStyleQuery(data), [
1631       ["node2.example.com"],
1632       ["node2.example.net"],
1633       ])
1634
1635   def testFilterBoolean(self):
1636     fielddefs = query._PrepareFieldList([
1637       (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"),
1638        None, query.QFF_HOSTNAME, lambda ctx, item: item["name"]),
1639       (query._MakeField("value", "Value", constants.QFT_BOOL, "Value"),
1640        None, 0, lambda ctx, item: item["value"]),
1641       ], [])
1642
1643     data = [
1644       { "name": "node1", "value": False, },
1645       { "name": "node2", "value": True, },
1646       { "name": "node3", "value": True, },
1647       ]
1648
1649     q = query.Query(fielddefs, ["name", "value"],
1650                     qfilter=["|", ["=", "value", False],
1651                                   ["=", "value", True]])
1652     self.assertTrue(q.RequestedNames() is None)
1653     self.assertEqual(q.Query(data), [
1654       [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, False)],
1655       [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, True)],
1656       [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, True)],
1657       ])
1658
1659     q = query.Query(fielddefs, ["name", "value"],
1660                     qfilter=["|", ["=", "value", False],
1661                                   ["!", ["=", "value", False]]])
1662     self.assertTrue(q.RequestedNames() is None)
1663     self.assertEqual(q.Query(data), [
1664       [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, False)],
1665       [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, True)],
1666       [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, True)],
1667       ])
1668
1669     # Comparing bool with string
1670     for i in ["False", "True", "0", "1", "no", "yes", "N", "Y"]:
1671       self.assertRaises(errors.ParameterError, query.Query,
1672                         fielddefs, ["name", "value"],
1673                         qfilter=["=", "value", i])
1674
1675     # Truth filter
1676     q = query.Query(fielddefs, ["name", "value"], qfilter=["?", "value"])
1677     self.assertTrue(q.RequestedNames() is None)
1678     self.assertEqual(q.Query(data), [
1679       [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, True)],
1680       [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, True)],
1681       ])
1682
1683     # Negative bool filter
1684     q = query.Query(fielddefs, ["name", "value"], qfilter=["!", ["?", "value"]])
1685     self.assertTrue(q.RequestedNames() is None)
1686     self.assertEqual(q.Query(data), [
1687       [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, False)],
1688       ])
1689
1690     # Complex truth filter
1691     q = query.Query(fielddefs, ["name", "value"],
1692                     qfilter=["|", ["&", ["=", "name", "node1"],
1693                                         ["!", ["?", "value"]]],
1694                                   ["?", "value"]])
1695     self.assertTrue(q.RequestedNames() is None)
1696     self.assertEqual(q.Query(data), [
1697       [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, False)],
1698       [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, True)],
1699       [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, True)],
1700       ])
1701
1702   def testFilterRegex(self):
1703     fielddefs = query._PrepareFieldList([
1704       (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"),
1705        None, 0, lambda ctx, item: item["name"]),
1706       ], [])
1707
1708     data = [
1709       { "name": "node1.example.com", },
1710       { "name": "node2.site.example.com", },
1711       { "name": "node2.example.net", },
1712
1713       # Empty name
1714       { "name": "", },
1715       ]
1716
1717     q = query.Query(fielddefs, ["name"], namefield="name",
1718                     qfilter=["=~", "name", "site"])
1719     self.assertTrue(q.RequestedNames() is None)
1720     self.assertEqual(q.Query(data), [
1721       [(constants.RS_NORMAL, "node2.site.example.com")],
1722       ])
1723
1724     q = query.Query(fielddefs, ["name"], namefield="name",
1725                     qfilter=["=~", "name", "^node2"])
1726     self.assertTrue(q.RequestedNames() is None)
1727     self.assertEqual(q.Query(data), [
1728       [(constants.RS_NORMAL, "node2.example.net")],
1729       [(constants.RS_NORMAL, "node2.site.example.com")],
1730       ])
1731
1732     q = query.Query(fielddefs, ["name"], namefield="name",
1733                     qfilter=["=~", "name", r"(?i)\.COM$"])
1734     self.assertTrue(q.RequestedNames() is None)
1735     self.assertEqual(q.Query(data), [
1736       [(constants.RS_NORMAL, "node1.example.com")],
1737       [(constants.RS_NORMAL, "node2.site.example.com")],
1738       ])
1739
1740     q = query.Query(fielddefs, ["name"], namefield="name",
1741                     qfilter=["=~", "name", r"."])
1742     self.assertTrue(q.RequestedNames() is None)
1743     self.assertEqual(q.Query(data), [
1744       [(constants.RS_NORMAL, "node1.example.com")],
1745       [(constants.RS_NORMAL, "node2.example.net")],
1746       [(constants.RS_NORMAL, "node2.site.example.com")],
1747       ])
1748
1749     q = query.Query(fielddefs, ["name"], namefield="name",
1750                     qfilter=["=~", "name", r"^$"])
1751     self.assertTrue(q.RequestedNames() is None)
1752     self.assertEqual(q.Query(data), [
1753       [(constants.RS_NORMAL, "")],
1754       ])
1755
1756     # Invalid regular expression
1757     self.assertRaises(errors.ParameterError, query.Query, fielddefs, ["name"],
1758                       qfilter=["=~", "name", r"["])
1759
1760   def testFilterLessGreater(self):
1761     fielddefs = query._PrepareFieldList([
1762       (query._MakeField("value", "Value", constants.QFT_NUMBER, "Value"),
1763        None, 0, lambda ctx, item: item),
1764       ], [])
1765
1766     data = range(100)
1767
1768     q = query.Query(fielddefs, ["value"],
1769                     qfilter=["<", "value", 20])
1770     self.assertTrue(q.RequestedNames() is None)
1771     self.assertEqual(q.Query(data),
1772                      [[(constants.RS_NORMAL, i)] for i in range(20)])
1773
1774     q = query.Query(fielddefs, ["value"],
1775                     qfilter=["<=", "value", 30])
1776     self.assertTrue(q.RequestedNames() is None)
1777     self.assertEqual(q.Query(data),
1778                      [[(constants.RS_NORMAL, i)] for i in range(31)])
1779
1780     q = query.Query(fielddefs, ["value"],
1781                     qfilter=[">", "value", 40])
1782     self.assertTrue(q.RequestedNames() is None)
1783     self.assertEqual(q.Query(data),
1784                      [[(constants.RS_NORMAL, i)] for i in range(41, 100)])
1785
1786     q = query.Query(fielddefs, ["value"],
1787                     qfilter=[">=", "value", 50])
1788     self.assertTrue(q.RequestedNames() is None)
1789     self.assertEqual(q.Query(data),
1790                      [[(constants.RS_NORMAL, i)] for i in range(50, 100)])
1791
1792   def testFilterLessGreaterJobId(self):
1793     fielddefs = query._PrepareFieldList([
1794       (query._MakeField("id", "ID", constants.QFT_TEXT, "Job ID"),
1795        None, query.QFF_JOB_ID, lambda ctx, item: item),
1796       ], [])
1797
1798     data = ["1", "2", "3", "10", "102", "120", "125", "15", "100", "7"]
1799
1800     assert data != utils.NiceSort(data), "Test data should not be sorted"
1801
1802     q = query.Query(fielddefs, ["id"], qfilter=["<", "id", "20"])
1803     self.assertTrue(q.RequestedNames() is None)
1804     self.assertEqual(q.Query(data), [
1805       [(constants.RS_NORMAL, "1")],
1806       [(constants.RS_NORMAL, "2")],
1807       [(constants.RS_NORMAL, "3")],
1808       [(constants.RS_NORMAL, "10")],
1809       [(constants.RS_NORMAL, "15")],
1810       [(constants.RS_NORMAL, "7")],
1811       ])
1812
1813     q = query.Query(fielddefs, ["id"], qfilter=[">=", "id", "100"])
1814     self.assertTrue(q.RequestedNames() is None)
1815     self.assertEqual(q.Query(data), [
1816       [(constants.RS_NORMAL, "102")],
1817       [(constants.RS_NORMAL, "120")],
1818       [(constants.RS_NORMAL, "125")],
1819       [(constants.RS_NORMAL, "100")],
1820       ])
1821
1822     # Integers are no valid job IDs
1823     self.assertRaises(errors.ParameterError, query.Query,
1824                       fielddefs, ["id"], qfilter=[">=", "id", 10])
1825
1826   def testFilterLessGreaterSplitTimestamp(self):
1827     fielddefs = query._PrepareFieldList([
1828       (query._MakeField("ts", "Timestamp", constants.QFT_OTHER, "Timestamp"),
1829        None, query.QFF_SPLIT_TIMESTAMP, lambda ctx, item: item),
1830       ], [])
1831
1832     data = [
1833       utils.SplitTime(0),
1834       utils.SplitTime(0.1),
1835       utils.SplitTime(18224.7872),
1836       utils.SplitTime(919896.12623),
1837       utils.SplitTime(999),
1838       utils.SplitTime(989.9999),
1839       ]
1840
1841     for i in [0, [0, 0]]:
1842       q = query.Query(fielddefs, ["ts"], qfilter=["<", "ts", i])
1843       self.assertTrue(q.RequestedNames() is None)
1844       self.assertEqual(q.Query(data), [])
1845
1846     q = query.Query(fielddefs, ["ts"], qfilter=["<", "ts", 1000])
1847     self.assertTrue(q.RequestedNames() is None)
1848     self.assertEqual(q.Query(data), [
1849       [(constants.RS_NORMAL, (0, 0))],
1850       [(constants.RS_NORMAL, (0, 100000))],
1851       [(constants.RS_NORMAL, (999, 0))],
1852       [(constants.RS_NORMAL, (989, 999900))],
1853       ])
1854
1855     q = query.Query(fielddefs, ["ts"], qfilter=[">=", "ts", 5000.3])
1856     self.assertTrue(q.RequestedNames() is None)
1857     self.assertEqual(q.Query(data), [
1858       [(constants.RS_NORMAL, (18224, 787200))],
1859       [(constants.RS_NORMAL, (919896, 126230))],
1860       ])
1861
1862     for i in [18224.7772, utils.SplitTime(18224.7772)]:
1863       q = query.Query(fielddefs, ["ts"], qfilter=[">=", "ts", i])
1864       self.assertTrue(q.RequestedNames() is None)
1865       self.assertEqual(q.Query(data), [
1866         [(constants.RS_NORMAL, (18224, 787200))],
1867         [(constants.RS_NORMAL, (919896, 126230))],
1868         ])
1869
1870     q = query.Query(fielddefs, ["ts"], qfilter=[">", "ts", 18224.7880])
1871     self.assertTrue(q.RequestedNames() is None)
1872     self.assertEqual(q.Query(data), [
1873       [(constants.RS_NORMAL, (919896, 126230))],
1874       ])
1875
1876
1877 if __name__ == "__main__":
1878   testutils.GanetiTestProgram()