Revision 7578ab0a

b/INSTALL
28 28
- `Python <http://www.python.org/>`_, version 2.4 or above, not 3.0
29 29
- `Python OpenSSL bindings <http://pyopenssl.sourceforge.net/>`_
30 30
- `simplejson Python module <http://code.google.com/p/simplejson/>`_
31
- `pyparsing Python module <http://pyparsing.wikispaces.com/>`_
31
- `pyparsing Python module <http://pyparsing.wikispaces.com/>`_, version
32
  1.4.6 or above
32 33
- `pyinotify Python module <http://trac.dbzteam.org/pyinotify/>`_
33 34
- `PycURL Python module <http://pycurl.sourceforge.net/>`_
34 35
- `ctypes Python module
b/NEWS
16 16
- Support for the undocumented and deprecated RAPI instance creation
17 17
  request format version 0 has been dropped. Use version 1, supported
18 18
  since Ganeti 2.1.3 and :doc:`documented <rapi>`, instead.
19
- Pyparsing 1.4.6 or above is required, see :doc:`installation
20
  documentation <install>`
19 21

  
20 22

  
21 23
Version 2.4.1
b/lib/errors.py
391 391
  """
392 392

  
393 393

  
394
class QueryFilterParseError(ParseError):
395
  """Error while parsing query filter.
396

  
397
  """
398
  def GetDetails(self):
399
    """Returns a list of strings with details about the error.
400

  
401
    """
402
    try:
403
      (_, inner) = self.args
404
    except IndexError:
405
      return None
406

  
407
    return [str(inner.line),
408
            (" " * (inner.column - 1)) + "^",
409
            str(inner)]
410

  
411

  
394 412
# errors should be added above
395 413

  
396 414

  
b/lib/qlang.py
31 31

  
32 32
"""
33 33

  
34
import re
35

  
36
import pyparsing as pyp
37

  
38
from ganeti import errors
39

  
40

  
34 41
# Logic operators with one or more operands, each of which is a filter on its
35 42
# own
36 43
OP_OR = "|"
......
61 68
    return [OP_OR] + [[OP_EQUAL, namefield, i] for i in values]
62 69

  
63 70
  return None
71

  
72

  
73
def _ConvertLogicOp(op):
74
  """Creates parsing action function for logic operator.
75

  
76
  @type op: string
77
  @param op: Operator for data structure, e.g. L{OP_AND}
78

  
79
  """
80
  def fn(toks):
81
    """Converts parser tokens to query operator structure.
82

  
83
    @rtype: list
84
    @return: Query operator structure, e.g. C{[OP_AND, ["=", "foo", "bar"]]}
85

  
86
    """
87
    operands = toks[0]
88

  
89
    if len(operands) == 1:
90
      return operands[0]
91

  
92
    # Build query operator structure
93
    return [[op] + operands.asList()]
94

  
95
  return fn
96

  
97

  
98
_KNOWN_REGEXP_DELIM = "/#^|"
99
_KNOWN_REGEXP_FLAGS = frozenset("si")
100

  
101

  
102
def _ConvertRegexpValue(_, loc, toks):
103
  """Regular expression value for condition.
104

  
105
  """
106
  (regexp, flags) = toks[0]
107

  
108
  # Ensure only whitelisted flags are used
109
  unknown_flags = (frozenset(flags) - _KNOWN_REGEXP_FLAGS)
110
  if unknown_flags:
111
    raise pyp.ParseFatalException("Unknown regular expression flags: '%s'" %
112
                                  "".join(unknown_flags), loc)
113

  
114
  if flags:
115
    re_flags = "(?%s)" % "".join(sorted(flags))
116
  else:
117
    re_flags = ""
118

  
119
  re_cond = re_flags + regexp
120

  
121
  # Test if valid
122
  try:
123
    re.compile(re_cond)
124
  except re.error, err:
125
    raise pyp.ParseFatalException("Invalid regular expression (%s)" % err, loc)
126

  
127
  return [re_cond]
128

  
129

  
130
def BuildFilterParser():
131
  """Builds a parser for query filter strings.
132

  
133
  @rtype: pyparsing.ParserElement
134

  
135
  """
136
  field_name = pyp.Word(pyp.alphas, pyp.alphanums + "_/.")
137

  
138
  # Integer
139
  num_sign = pyp.Word("-+", exact=1)
140
  number = pyp.Combine(pyp.Optional(num_sign) + pyp.Word(pyp.nums))
141
  number.setParseAction(lambda toks: int(toks[0]))
142

  
143
  # Right-hand-side value
144
  rval = (number | pyp.quotedString.setParseAction(pyp.removeQuotes))
145

  
146
  # Boolean condition
147
  bool_cond = field_name.copy()
148
  bool_cond.setParseAction(lambda (fname, ): [[OP_TRUE, fname]])
149

  
150
  # Simple binary conditions
151
  binopstbl = {
152
    "==": OP_EQUAL,
153
    "!=": OP_NOT_EQUAL,
154
    }
155

  
156
  binary_cond = (field_name + pyp.oneOf(binopstbl.keys()) + rval)
157
  binary_cond.setParseAction(lambda (lhs, op, rhs): [[binopstbl[op], lhs, rhs]])
158

  
159
  # "in" condition
160
  in_cond = (rval + pyp.Suppress("in") + field_name)
161
  in_cond.setParseAction(lambda (value, field): [[OP_CONTAINS, field, value]])
162

  
163
  # "not in" condition
164
  not_in_cond = (rval + pyp.Suppress("not") + pyp.Suppress("in") + field_name)
165
  not_in_cond.setParseAction(lambda (value, field): [[OP_NOT, [OP_CONTAINS,
166
                                                               field, value]]])
167

  
168
  # Regular expression, e.g. m/foobar/i
169
  regexp_val = pyp.Group(pyp.Optional("m").suppress() +
170
                         pyp.MatchFirst([pyp.QuotedString(i, escChar="\\")
171
                                         for i in _KNOWN_REGEXP_DELIM]) +
172
                         pyp.Optional(pyp.Word(pyp.alphas), default=""))
173
  regexp_val.setParseAction(_ConvertRegexpValue)
174
  regexp_cond = (field_name + pyp.Suppress("=~") + regexp_val)
175
  regexp_cond.setParseAction(lambda (field, value): [[OP_REGEXP, field, value]])
176

  
177
  not_regexp_cond = (field_name + pyp.Suppress("!~") + regexp_val)
178
  not_regexp_cond.setParseAction(lambda (field, value):
179
                                 [[OP_NOT, [OP_REGEXP, field, value]]])
180

  
181
  # All possible conditions
182
  condition = (binary_cond ^ bool_cond ^
183
               in_cond ^ not_in_cond ^
184
               regexp_cond ^ not_regexp_cond)
185

  
186
  # Associativity operators
187
  filter_expr = pyp.operatorPrecedence(condition, [
188
    (pyp.Keyword("not").suppress(), 1, pyp.opAssoc.RIGHT,
189
     lambda toks: [[OP_NOT, toks[0][0]]]),
190
    (pyp.Keyword("and").suppress(), 2, pyp.opAssoc.LEFT,
191
     _ConvertLogicOp(OP_AND)),
192
    (pyp.Keyword("or").suppress(), 2, pyp.opAssoc.LEFT,
193
     _ConvertLogicOp(OP_OR)),
194
    ])
195

  
196
  parser = pyp.StringStart() + filter_expr + pyp.StringEnd()
197
  parser.parseWithTabs()
198

  
199
  # Originally C{parser.validate} was called here, but there seems to be some
200
  # issue causing it to fail whenever the "not" operator is included above.
201

  
202
  return parser
203

  
204

  
205
def ParseFilter(text, parser=None):
206
  """Parses a query filter.
207

  
208
  @type text: string
209
  @param text: Query filter
210
  @type parser: pyparsing.ParserElement
211
  @param parser: Pyparsing object
212
  @rtype: list
213

  
214
  """
215
  if parser is None:
216
    parser = BuildFilterParser()
217

  
218
  try:
219
    return parser.parseString(text)[0]
220
  except pyp.ParseBaseException, err:
221
    raise errors.QueryFilterParseError("Failed to parse query filter"
222
                                       " '%s': %s" % (text, err), err)
b/test/ganeti.qlang_unittest.py
26 26
from ganeti import utils
27 27
from ganeti import errors
28 28
from ganeti import qlang
29
from ganeti import query
29 30

  
30 31
import testutils
31 32

  
......
47 48
               ["|", ["=", "xyz", "a"], ["=", "xyz", "b"], ["=", "xyz", "c"]])
48 49

  
49 50

  
51
class TestParseFilter(unittest.TestCase):
52
  def setUp(self):
53
    self.parser = qlang.BuildFilterParser()
54

  
55
  def _Test(self, filter_, expected):
56
    self.assertEqual(qlang.ParseFilter(filter_, parser=self.parser), expected)
57

  
58
  def test(self):
59
    self._Test("name==\"foobar\"", [qlang.OP_EQUAL, "name", "foobar"])
60
    self._Test("name=='foobar'", [qlang.OP_EQUAL, "name", "foobar"])
61

  
62
    self._Test("valA==1 and valB==2 or valC==3",
63
               [qlang.OP_OR,
64
                [qlang.OP_AND, [qlang.OP_EQUAL, "valA", 1],
65
                               [qlang.OP_EQUAL, "valB", 2]],
66
                [qlang.OP_EQUAL, "valC", 3]])
67

  
68
    self._Test(("(name\n==\"foobar\") and (xyz==\"va)ue\" and k == 256 or"
69
                " x ==\t\"y\"\n) and mc"),
70
               [qlang.OP_AND,
71
                [qlang.OP_EQUAL, "name", "foobar"],
72
                [qlang.OP_OR,
73
                 [qlang.OP_AND, [qlang.OP_EQUAL, "xyz", "va)ue"],
74
                                [qlang.OP_EQUAL, "k", 256]],
75
                 [qlang.OP_EQUAL, "x", "y"]],
76
                [qlang.OP_TRUE, "mc"]])
77

  
78
    self._Test("(xyz==\"v\" or k == 256 and x == \"y\")",
79
               [qlang.OP_OR,
80
                [qlang.OP_EQUAL, "xyz", "v"],
81
                [qlang.OP_AND, [qlang.OP_EQUAL, "k", 256],
82
                               [qlang.OP_EQUAL, "x", "y"]]])
83

  
84
    self._Test("valA==1 and valB==2 and valC==3",
85
               [qlang.OP_AND, [qlang.OP_EQUAL, "valA", 1],
86
                              [qlang.OP_EQUAL, "valB", 2],
87
                              [qlang.OP_EQUAL, "valC", 3]])
88
    self._Test("master or field",
89
               [qlang.OP_OR, [qlang.OP_TRUE, "master"],
90
                             [qlang.OP_TRUE, "field"]])
91
    self._Test("mem == 128", [qlang.OP_EQUAL, "mem", 128])
92
    self._Test("negfield != -1", [qlang.OP_NOT_EQUAL, "negfield", -1])
93
    self._Test("master", [qlang.OP_TRUE, "master"])
94
    self._Test("not master", [qlang.OP_NOT, [qlang.OP_TRUE, "master"]])
95
    for op in ["not", "and", "or"]:
96
      self._Test("%sxyz" % op, [qlang.OP_TRUE, "%sxyz" % op])
97
      self._Test("not %sxyz" % op,
98
                 [qlang.OP_NOT, [qlang.OP_TRUE, "%sxyz" % op]])
99
      self._Test("  not \t%sfoo" % op,
100
                 [qlang.OP_NOT, [qlang.OP_TRUE, "%sfoo" % op]])
101
      self._Test("%sname =~ m/abc/" % op,
102
                 [qlang.OP_REGEXP, "%sname" % op, "abc"])
103
    self._Test("master and not other",
104
               [qlang.OP_AND, [qlang.OP_TRUE, "master"],
105
                              [qlang.OP_NOT, [qlang.OP_TRUE, "other"]]])
106
    self._Test("not (master or other == 4)",
107
               [qlang.OP_NOT,
108
                [qlang.OP_OR, [qlang.OP_TRUE, "master"],
109
                              [qlang.OP_EQUAL, "other", 4]]])
110
    self._Test("some==\"val\\\"ue\"", [qlang.OP_EQUAL, "some", "val\\\"ue"])
111
    self._Test("123 in ips", [qlang.OP_CONTAINS, "ips", 123])
112
    self._Test("99 not in ips", [qlang.OP_NOT, [qlang.OP_CONTAINS, "ips", 99]])
113
    self._Test("\"a\" in valA and \"b\" not in valB",
114
               [qlang.OP_AND, [qlang.OP_CONTAINS, "valA", "a"],
115
                              [qlang.OP_NOT, [qlang.OP_CONTAINS, "valB", "b"]]])
116

  
117
    self._Test("name =~ m/test/", [qlang.OP_REGEXP, "name", "test"])
118
    self._Test("name =~ m/^node.*example.com$/i",
119
               [qlang.OP_REGEXP, "name", "(?i)^node.*example.com$"])
120
    self._Test("(name =~ m/^node.*example.com$/s and master) or pip =~ |^3.*|",
121
               [qlang.OP_OR,
122
                [qlang.OP_AND,
123
                 [qlang.OP_REGEXP, "name", "(?s)^node.*example.com$"],
124
                 [qlang.OP_TRUE, "master"]],
125
                [qlang.OP_REGEXP, "pip", "^3.*"]])
126
    for flags in ["si", "is", "ssss", "iiiisiii"]:
127
      self._Test("name =~ m/gi/%s" % flags,
128
                 [qlang.OP_REGEXP, "name", "(?%s)gi" % "".join(sorted(flags))])
129

  
130
    for i in qlang._KNOWN_REGEXP_DELIM:
131
      self._Test("name =~ m%stest%s" % (i, i),
132
                 [qlang.OP_REGEXP, "name", "test"])
133
      self._Test("name !~ m%stest%s" % (i, i),
134
                 [qlang.OP_NOT, [qlang.OP_REGEXP, "name", "test"]])
135
      self._Test("not\tname =~ m%stest%s" % (i, i),
136
                 [qlang.OP_NOT, [qlang.OP_REGEXP, "name", "test"]])
137
      self._Test("notname =~ m%stest%s" % (i, i),
138
                 [qlang.OP_REGEXP, "notname", "test"])
139

  
140
  def testAllFields(self):
141
    for name in frozenset(i for d in query.ALL_FIELD_LISTS for i in d.keys()):
142
      self._Test("%s == \"value\"" % name, [qlang.OP_EQUAL, name, "value"])
143

  
144
  def testError(self):
145
    # Invalid field names, meaning no boolean check is done
146
    tests = ["#invalid!filter#", "m/x/,"]
147

  
148
    # Unknown regexp flag
149
    tests.append("name=~m#a#g")
150

  
151
    # Incomplete regexp group
152
    tests.append("name=~^[^")
153

  
154
    # Valid flag, but in uppercase
155
    tests.append("asdf =~ m|abc|I")
156

  
157
    # Non-matching regexp delimiters
158
    tests.append("name =~ /foobarbaz#")
159

  
160
    for filter_ in tests:
161
      try:
162
        qlang.ParseFilter(filter_, parser=self.parser)
163
      except errors.QueryFilterParseError, err:
164
        self.assertEqual(len(err.GetDetails()), 3)
165
      else:
166
        self.fail("Invalid filter '%s' did not raise exception" % filter_)
167

  
168

  
50 169
if __name__ == "__main__":
51 170
  testutils.GanetiTestProgram()

Also available in: Unified diff