Add node query definition
authorMichael Hanselmann <hansmi@google.com>
Fri, 12 Nov 2010 15:17:43 +0000 (16:17 +0100)
committerMichael Hanselmann <hansmi@google.com>
Mon, 29 Nov 2010 20:00:25 +0000 (21:00 +0100)
This includes a bunch of helper functions which can be helpful for other
queries, too. Unittests are included.

Signed-off-by: Michael Hanselmann <hansmi@google.com>
Reviewed-by: Iustin Pop <iustin@google.com>

lib/query.py
test/ganeti.query_unittest.py

index 9be8c54..11f1025 100644 (file)
@@ -21,6 +21,7 @@
 
 """Module for query operations"""
 
+import logging
 import operator
 import re
 
@@ -32,6 +33,12 @@ from ganeti import objects
 from ganeti import ht
 
 
+(NQ_CONFIG,
+ NQ_INST,
+ NQ_LIVE,
+ NQ_GROUP) = range(1, 5)
+
+
 FIELD_NAME_RE = re.compile(r"^[a-z0-9/._]+$")
 TITLE_RE = re.compile(r"^[^\s]+$")
 
@@ -231,3 +238,203 @@ def _MakeField(name, title, kind):
 
   """
   return objects.QueryFieldDefinition(name=name, title=title, kind=kind)
+
+
+def _GetNodeRole(node, master_name):
+  """Determine node role.
+
+  @type node: L{objects.Node}
+  @param node: Node object
+  @type master_name: string
+  @param master_name: Master node name
+
+  """
+  if node.name == master_name:
+    return "M"
+  elif node.master_candidate:
+    return "C"
+  elif node.drained:
+    return "D"
+  elif node.offline:
+    return "O"
+  else:
+    return "R"
+
+
+def _GetItemAttr(attr):
+  """Returns a field function to return an attribute of the item.
+
+  @param attr: Attribute name
+
+  """
+  getter = operator.attrgetter(attr)
+  return lambda _, item: (constants.QRFS_NORMAL, getter(item))
+
+
+class NodeQueryData:
+  """Data container for node data queries.
+
+  """
+  def __init__(self, nodes, live_data, master_name, node_to_primary,
+               node_to_secondary, groups):
+    """Initializes this class.
+
+    """
+    self.nodes = nodes
+    self.live_data = live_data
+    self.master_name = master_name
+    self.node_to_primary = node_to_primary
+    self.node_to_secondary = node_to_secondary
+    self.groups = groups
+
+    # Used for individual rows
+    self.curlive_data = None
+
+  def __iter__(self):
+    """Iterate over all nodes.
+
+    This function has side-effects and only one instance of the resulting
+    generator should be used at a time.
+
+    """
+    for node in self.nodes:
+      if self.live_data:
+        self.curlive_data = self.live_data.get(node.name, None)
+      else:
+        self.curlive_data = None
+      yield node
+
+
+#: Fields that are direct attributes of an L{objects.Node} object
+_NODE_SIMPLE_FIELDS = {
+  "ctime": ("CTime", constants.QFT_TIMESTAMP),
+  "drained": ("Drained", constants.QFT_BOOL),
+  "master_candidate": ("MasterC", constants.QFT_BOOL),
+  "master_capable": ("MasterCapable", constants.QFT_BOOL),
+  "mtime": ("MTime", constants.QFT_TIMESTAMP),
+  "name": ("Node", constants.QFT_TEXT),
+  "offline": ("Offline", constants.QFT_BOOL),
+  "serial_no": ("SerialNo", constants.QFT_NUMBER),
+  "uuid": ("UUID", constants.QFT_TEXT),
+  "vm_capable": ("VMCapable", constants.QFT_BOOL),
+  }
+
+
+#: Fields requiring talking to the node
+_NODE_LIVE_FIELDS = {
+  "bootid": ("BootID", constants.QFT_TEXT, "bootid"),
+  "cnodes": ("CNodes", constants.QFT_NUMBER, "cpu_nodes"),
+  "csockets": ("CSockets", constants.QFT_NUMBER, "cpu_sockets"),
+  "ctotal": ("CTotal", constants.QFT_NUMBER, "cpu_total"),
+  "dfree": ("DFree", constants.QFT_UNIT, "vg_free"),
+  "dtotal": ("DTotal", constants.QFT_UNIT, "vg_size"),
+  "mfree": ("MFree", constants.QFT_UNIT, "memory_free"),
+  "mnode": ("MNode", constants.QFT_UNIT, "memory_dom0"),
+  "mtotal": ("MTotal", constants.QFT_UNIT, "memory_total"),
+  }
+
+
+def _GetNodeGroup(ctx, node):
+  """Returns the name of a node's group.
+
+  @type ctx: L{NodeQueryData}
+  @type node: L{objects.Node}
+  @param node: Node object
+
+  """
+  ng = ctx.groups.get(node.group, None)
+  if ng is None:
+    # Nodes always have a group, or the configuration is corrupt
+    return (constants.QRFS_UNAVAIL, None)
+
+  return (constants.QRFS_NORMAL, ng.name)
+
+
+def _GetLiveNodeField(field, kind, ctx, _):
+  """Gets the value of a "live" field from L{NodeQueryData}.
+
+  @param field: Live field name
+  @param kind: Data kind, one of L{constants.QFT_ALL}
+  @type ctx: L{NodeQueryData}
+
+  """
+  if not ctx.curlive_data:
+    return (constants.QRFS_NODATA, None)
+
+  try:
+    value = ctx.curlive_data[field]
+  except KeyError:
+    return (constants.QRFS_UNAVAIL, None)
+
+  if kind == constants.QFT_TEXT:
+    return (constants.QRFS_NORMAL, value)
+
+  assert kind in (constants.QFT_NUMBER, constants.QFT_UNIT)
+
+  # Try to convert into number
+  try:
+    return (constants.QRFS_NORMAL, int(value))
+  except (ValueError, TypeError):
+    logging.exception("Failed to convert node field '%s' (value %r) to int",
+                      value, field)
+    return (constants.QRFS_UNAVAIL, None)
+
+
+def _BuildNodeFields():
+  """Builds list of fields for node queries.
+
+  """
+  fields = [
+    (_MakeField("pip", "PrimaryIP", constants.QFT_TEXT), NQ_CONFIG,
+     lambda ctx, node: (constants.QRFS_NORMAL, node.primary_ip)),
+    (_MakeField("sip", "SecondaryIP", constants.QFT_TEXT), NQ_CONFIG,
+     lambda ctx, node: (constants.QRFS_NORMAL, node.secondary_ip)),
+    (_MakeField("tags", "Tags", constants.QFT_OTHER), NQ_CONFIG,
+     lambda ctx, node: (constants.QRFS_NORMAL, list(node.GetTags()))),
+    (_MakeField("master", "IsMaster", constants.QFT_BOOL), NQ_CONFIG,
+     lambda ctx, node: (constants.QRFS_NORMAL, node.name == ctx.master_name)),
+    (_MakeField("role", "Role", constants.QFT_TEXT), NQ_CONFIG,
+     lambda ctx, node: (constants.QRFS_NORMAL,
+                        _GetNodeRole(node, ctx.master_name))),
+    (_MakeField("group", "Group", constants.QFT_TEXT), NQ_GROUP, _GetNodeGroup),
+    (_MakeField("group.uuid", "GroupUUID", constants.QFT_TEXT),
+     NQ_CONFIG, lambda ctx, node: (constants.QRFS_NORMAL, node.group)),
+    ]
+
+  def _GetLength(getter):
+    return lambda ctx, node: (constants.QRFS_NORMAL,
+                              len(getter(ctx)[node.name]))
+
+  def _GetList(getter):
+    return lambda ctx, node: (constants.QRFS_NORMAL,
+                              list(getter(ctx)[node.name]))
+
+  # Add fields operating on instance lists
+  for prefix, titleprefix, getter in \
+      [("p", "Pri", operator.attrgetter("node_to_primary")),
+       ("s", "Sec", operator.attrgetter("node_to_secondary"))]:
+    fields.extend([
+      (_MakeField("%sinst_cnt" % prefix, "%sinst" % prefix.upper(),
+                  constants.QFT_NUMBER),
+       NQ_INST, _GetLength(getter)),
+      (_MakeField("%sinst_list" % prefix, "%sInstances" % titleprefix,
+                  constants.QFT_OTHER),
+       NQ_INST, _GetList(getter)),
+      ])
+
+  # Add simple fields
+  fields.extend([(_MakeField(name, title, kind), NQ_CONFIG, _GetItemAttr(name))
+                 for (name, (title, kind)) in _NODE_SIMPLE_FIELDS.items()])
+
+  # Add fields requiring live data
+  fields.extend([
+    (_MakeField(name, title, kind), NQ_LIVE,
+     compat.partial(_GetLiveNodeField, nfield, kind))
+    for (name, (title, kind, nfield)) in _NODE_LIVE_FIELDS.items()
+    ])
+
+  return _PrepareFieldList(fields)
+
+
+#: Fields available for node queries
+NODE_FIELDS = _BuildNodeFields()
index 0986457..2e37fd8 100755 (executable)
@@ -254,5 +254,209 @@ class TestQuery(unittest.TestCase):
                       for i in range(1, 10)])
 
 
+class TestGetNodeRole(unittest.TestCase):
+  def testMaster(self):
+    node = objects.Node(name="node1")
+    self.assertEqual(query._GetNodeRole(node, "node1"), "M")
+
+  def testMasterCandidate(self):
+    node = objects.Node(name="node1", master_candidate=True)
+    self.assertEqual(query._GetNodeRole(node, "master"), "C")
+
+  def testRegular(self):
+    node = objects.Node(name="node1")
+    self.assertEqual(query._GetNodeRole(node, "master"), "R")
+
+  def testDrained(self):
+    node = objects.Node(name="node1", drained=True)
+    self.assertEqual(query._GetNodeRole(node, "master"), "D")
+
+  def testOffline(self):
+    node = objects.Node(name="node1", offline=True)
+    self.assertEqual(query._GetNodeRole(node, "master"), "O")
+
+
+class TestNodeQuery(unittest.TestCase):
+  def _Create(self, selected):
+    return query.Query(query.NODE_FIELDS, selected)
+
+  def testSimple(self):
+    nodes = [
+      objects.Node(name="node1", drained=False),
+      objects.Node(name="node2", drained=True),
+      objects.Node(name="node3", drained=False),
+      ]
+    for live_data in [None, dict.fromkeys([node.name for node in nodes], {})]:
+      nqd = query.NodeQueryData(nodes, live_data, None, None, None, None)
+
+      q = self._Create(["name", "drained"])
+      self.assertEqual(q.RequestedData(), set([query.NQ_CONFIG]))
+      self.assertEqual(q.Query(nqd),
+                       [[(constants.QRFS_NORMAL, "node1"),
+                         (constants.QRFS_NORMAL, False)],
+                        [(constants.QRFS_NORMAL, "node2"),
+                         (constants.QRFS_NORMAL, True)],
+                        [(constants.QRFS_NORMAL, "node3"),
+                         (constants.QRFS_NORMAL, False)],
+                       ])
+      self.assertEqual(q.OldStyleQuery(nqd),
+                       [["node1", False],
+                        ["node2", True],
+                        ["node3", False]])
+
+  def test(self):
+    selected = query.NODE_FIELDS.keys()
+    field_index = dict((field, idx) for idx, field in enumerate(selected))
+
+    q = self._Create(selected)
+    self.assertEqual(q.RequestedData(),
+                     set([query.NQ_CONFIG, query.NQ_LIVE, query.NQ_INST,
+                          query.NQ_GROUP]))
+
+    node_names = ["node%s" % i for i in range(20)]
+    master_name = node_names[3]
+    nodes = [
+      objects.Node(name=name,
+                   primary_ip="192.0.2.%s" % idx,
+                   secondary_ip="192.0.100.%s" % idx,
+                   serial_no=7789 * idx,
+                   master_candidate=(name != master_name and idx % 3 == 0),
+                   offline=False,
+                   drained=False,
+                   vm_capable=False,
+                   master_capable=False,
+                   group="default",
+                   ctime=1290006900,
+                   mtime=1290006913,
+                   uuid="fd9ccebe-6339-43c9-a82e-94bbe575%04d" % idx)
+      for idx, name in enumerate(node_names)
+      ]
+
+    master_node = nodes[3]
+    master_node.AddTag("masternode")
+    master_node.AddTag("another")
+    master_node.AddTag("tag")
+    assert master_node.name == master_name
+
+    live_data_name = node_names[4]
+    assert live_data_name != master_name
+
+    fake_live_data = {
+      "bootid": "a2504766-498e-4b25-b21e-d23098dc3af4",
+      "cnodes": 4,
+      "csockets": 4,
+      "ctotal": 8,
+      "mnode": 128,
+      "mfree": 100,
+      "mtotal": 4096,
+      "dfree": 5 * 1024 * 1024,
+      "dtotal": 100 * 1024 * 1024,
+      }
+
+    assert (sorted(query._NODE_LIVE_FIELDS.keys()) ==
+            sorted(fake_live_data.keys()))
+
+    live_data = dict.fromkeys(node_names, {})
+    live_data[live_data_name] = \
+      dict((query._NODE_LIVE_FIELDS[name][2], value)
+           for name, value in fake_live_data.items())
+
+    node_to_primary = dict((name, set()) for name in node_names)
+    node_to_primary[master_name].update(["inst1", "inst2"])
+
+    node_to_secondary = dict((name, set()) for name in node_names)
+    node_to_secondary[live_data_name].update(["instX", "instY", "instZ"])
+
+    ng_uuid = "492b4b74-8670-478a-b98d-4c53a76238e6"
+    groups = {
+      ng_uuid: objects.NodeGroup(name="ng1", uuid=ng_uuid),
+      }
+
+    master_node.group = ng_uuid
+
+    nqd = query.NodeQueryData(nodes, live_data, master_name,
+                              node_to_primary, node_to_secondary, groups)
+    result = q.Query(nqd)
+    self.assert_(compat.all(len(row) == len(selected) for row in result))
+    self.assertEqual([row[field_index["name"]] for row in result],
+                     [(constants.QRFS_NORMAL, name) for name in node_names])
+
+    node_to_row = dict((row[field_index["name"]][1], idx)
+                       for idx, row in enumerate(result))
+
+    master_row = result[node_to_row[master_name]]
+    self.assert_(master_row[field_index["master"]])
+    self.assert_(master_row[field_index["role"]], "M")
+    self.assertEqual(master_row[field_index["group"]],
+                     (constants.QRFS_NORMAL, "ng1"))
+    self.assertEqual(master_row[field_index["group.uuid"]],
+                     (constants.QRFS_NORMAL, ng_uuid))
+
+    self.assert_(row[field_index["pip"]] == node.primary_ip and
+                 row[field_index["sip"]] == node.secondary_ip and
+                 set(row[field_index["tags"]]) == node.GetTags() and
+                 row[field_index["serial_no"]] == node.serial_no and
+                 row[field_index["role"]] == query._GetNodeRole(node,
+                                                                master_name) and
+                 (node.name == master_name or
+                  (row[field_index["group"]] == "<unknown>" and
+                   row[field_index["group.uuid"]] is None))
+                 for row, node in zip(result, nodes))
+
+    live_data_row = result[node_to_row[live_data_name]]
+
+    for (field, value) in fake_live_data.items():
+      self.assertEqual(live_data_row[field_index[field]],
+                       (constants.QRFS_NORMAL, value))
+
+    self.assertEqual(master_row[field_index["pinst_cnt"]],
+                     (constants.QRFS_NORMAL, 2))
+    self.assertEqual(live_data_row[field_index["sinst_cnt"]],
+                     (constants.QRFS_NORMAL, 3))
+    self.assertEqual(master_row[field_index["pinst_list"]],
+                     (constants.QRFS_NORMAL,
+                      list(node_to_primary[master_name])))
+    self.assertEqual(live_data_row[field_index["sinst_list"]],
+                     (constants.QRFS_NORMAL,
+                      list(node_to_secondary[live_data_name])))
+
+  def testGetLiveNodeField(self):
+    nodes = [
+      objects.Node(name="node1", drained=False),
+      objects.Node(name="node2", drained=True),
+      objects.Node(name="node3", drained=False),
+      ]
+    live_data = dict.fromkeys([node.name for node in nodes], {})
+
+    # No data
+    nqd = query.NodeQueryData(None, None, None, None, None, None)
+    self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
+                                             nqd, None),
+                     (constants.QRFS_NODATA, None))
+
+    # Missing field
+    ctx = _QueryData(None, curlive_data={
+      "some": 1,
+      "other": 2,
+      })
+    self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
+                                             ctx, None),
+                     (constants.QRFS_UNAVAIL, None))
+
+    # Wrong format/datatype
+    ctx = _QueryData(None, curlive_data={
+      "hello": ["Hello World"],
+      "other": 2,
+      })
+    self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER,
+                                             ctx, None),
+                     (constants.QRFS_UNAVAIL, None))
+
+    # Wrong field type
+    ctx = _QueryData(None, curlive_data={"hello": 123})
+    self.assertRaises(AssertionError, query._GetLiveNodeField,
+                      "hello", constants.QFT_BOOL, ctx, None)
+
+
 if __name__ == "__main__":
   testutils.GanetiTestProgram()