Statistics
| Branch: | Tag: | Revision:

root / lib / serializer.py @ 560ef132

History | View | Annotate | Download (10.7 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2008, 2014 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21
"""Serializer abstraction module
22

23
This module introduces a simple abstraction over the serialization
24
backend (currently json).
25

26
"""
27
# pylint: disable=C0103
28

    
29
# C0103: Invalid name, since pylint doesn't see that Dump points to a
30
# function and not a constant
31

    
32
import re
33

    
34
# Python 2.6 and above contain a JSON module based on simplejson. Unfortunately
35
# the standard library version is significantly slower than the external
36
# module. While it should be better from at least Python 3.2 on (see Python
37
# issue 7451), for now Ganeti needs to work well with older Python versions
38
# too.
39
import simplejson
40

    
41
from ganeti import errors
42
from ganeti import utils
43
from ganeti import constants
44

    
45
_RE_EOLSP = re.compile("[ \t]+$", re.MULTILINE)
46

    
47

    
48
def DumpJson(data, private_encoder=None):
49
  """Serialize a given object.
50

51
  @param data: the data to serialize
52
  @return: the string representation of data
53
  @param private_encoder: specify L{serializer.EncodeWithPrivateFields} if you
54
                          require the produced JSON to also contain private
55
                          parameters. Otherwise, they will encode to null.
56

57
  """
58
  if private_encoder is None:
59
    # Do not leak private fields by default.
60
    private_encoder = EncodeWithoutPrivateFields
61
  encoded = simplejson.dumps(data, default=private_encoder)
62

    
63
  txt = _RE_EOLSP.sub("", encoded)
64
  if not txt.endswith("\n"):
65
    txt += "\n"
66

    
67
  return txt
68

    
69

    
70
def LoadJson(txt):
71
  """Unserialize data from a string.
72

73
  @param txt: the json-encoded form
74
  @return: the original data
75
  @raise JSONDecodeError: if L{txt} is not a valid JSON document
76

77
  """
78
  values = simplejson.loads(txt)
79

    
80
  # Hunt and seek for Private fields and wrap them.
81
  WrapPrivateValues(values)
82

    
83
  return values
84

    
85

    
86
def WrapPrivateValues(json):
87
  """Crawl a JSON decoded structure for private values and wrap them.
88

89
  @param json: the json-decoded value to protect.
90

91
  """
92
  # This function used to be recursive. I use this list to avoid actual
93
  # recursion, however, since this is a very high-traffic area.
94
  todo = [json]
95

    
96
  while todo:
97
    data = todo.pop()
98

    
99
    if isinstance(data, list): # Array
100
      for item in data:
101
        todo.append(item)
102
    elif isinstance(data, dict): # Object
103

    
104
      # This is kind of a kludge, but the only place where we know what should
105
      # be protected is in ganeti.opcodes, and not in a way that is helpful to
106
      # us, especially in such a high traffic method; on the other hand, the
107
      # Haskell `py_compat_fields` test should complain whenever this check
108
      # does not protect fields properly.
109
      for field in data:
110
        value = data[field]
111
        if field in constants.PRIVATE_PARAMETERS_BLACKLIST:
112
          if not field.endswith("_cluster"):
113
            data[field] = PrivateDict(value)
114
          else:
115
            for os in data[field]:
116
              value[os] = PrivateDict(value[os])
117
        else:
118
          todo.append(value)
119
    else: # Values
120
      pass
121

    
122

    
123
def DumpSignedJson(data, key, salt=None, key_selector=None,
124
                   private_encoder=None):
125
  """Serialize a given object and authenticate it.
126

127
  @param data: the data to serialize
128
  @param key: shared hmac key
129
  @param key_selector: name/id that identifies the key (in case there are
130
    multiple keys in use, e.g. in a multi-cluster environment)
131
  @param private_encoder: see L{DumpJson}
132
  @return: the string representation of data signed by the hmac key
133

134
  """
135
  txt = DumpJson(data, private_encoder=private_encoder)
136
  if salt is None:
137
    salt = ""
138
  signed_dict = {
139
    "msg": txt,
140
    "salt": salt,
141
    }
142

    
143
  if key_selector:
144
    signed_dict["key_selector"] = key_selector
145
  else:
146
    key_selector = ""
147

    
148
  signed_dict["hmac"] = utils.Sha1Hmac(key, txt, salt=salt + key_selector)
149

    
150
  return DumpJson(signed_dict)
151

    
152

    
153
def LoadSignedJson(txt, key):
154
  """Verify that a given message was signed with the given key, and load it.
155

156
  @param txt: json-encoded hmac-signed message
157
  @param key: the shared hmac key or a callable taking one argument (the key
158
    selector), which returns the hmac key belonging to the key selector.
159
    Typical usage is to pass a reference to the get method of a dict.
160
  @rtype: tuple of original data, string
161
  @return: original data, salt
162
  @raises errors.SignatureError: if the message signature doesn't verify
163

164
  """
165
  signed_dict = LoadJson(txt)
166

    
167
  WrapPrivateValues(signed_dict)
168

    
169
  if not isinstance(signed_dict, dict):
170
    raise errors.SignatureError("Invalid external message")
171
  try:
172
    msg = signed_dict["msg"]
173
    salt = signed_dict["salt"]
174
    hmac_sign = signed_dict["hmac"]
175
  except KeyError:
176
    raise errors.SignatureError("Invalid external message")
177

    
178
  if callable(key):
179
    # pylint: disable=E1103
180
    key_selector = signed_dict.get("key_selector", None)
181
    hmac_key = key(key_selector)
182
    if not hmac_key:
183
      raise errors.SignatureError("No key with key selector '%s' found" %
184
                                  key_selector)
185
  else:
186
    key_selector = ""
187
    hmac_key = key
188

    
189
  if not utils.VerifySha1Hmac(hmac_key, msg, hmac_sign,
190
                              salt=salt + key_selector):
191
    raise errors.SignatureError("Invalid Signature")
192

    
193
  return LoadJson(msg), salt
194

    
195

    
196
def LoadAndVerifyJson(raw, verify_fn):
197
  """Parses and verifies JSON data.
198

199
  @type raw: string
200
  @param raw: Input data in JSON format
201
  @type verify_fn: callable
202
  @param verify_fn: Verification function, usually from L{ht}
203
  @return: De-serialized data
204

205
  """
206
  try:
207
    data = LoadJson(raw)
208
  except Exception, err:
209
    raise errors.ParseError("Can't parse input data: %s" % err)
210

    
211
  if not verify_fn(data):
212
    raise errors.ParseError("Data does not match expected format: %s" %
213
                            verify_fn)
214

    
215
  return data
216

    
217

    
218
Dump = DumpJson
219
Load = LoadJson
220
DumpSigned = DumpSignedJson
221
LoadSigned = LoadSignedJson
222

    
223

    
224
class Private(object):
225
  """Wrap a value so it is hard to leak it accidentally.
226

227
  >>> x = Private("foo")
228
  >>> print "Value: %s" % x
229
  Value: <redacted>
230
  >>> print "Value: {0}".format(x)
231
  Value: <redacted>
232
  >>> x.upper() == "FOO"
233
  True
234

235
  """
236
  def __init__(self, item, descr="redacted"):
237
    if isinstance(item, Private):
238
      raise ValueError("Attempted to nest Private values.")
239
    self._item = item
240
    self._descr = descr
241

    
242
  def Get(self):
243
    "Return the wrapped value."
244
    return self._item
245

    
246
  def __str__(self):
247
    return "<{._descr}>".format(self)
248

    
249
  def __repr__(self):
250
    return "Private(?, descr='{._descr}')".format(self)
251

    
252
  # pylint: disable=W0212
253
  # If it doesn't access _item directly, the call will go through __getattr__
254
  # because this class defines __slots__ and "item" is not in it.
255
  # OTOH, if we do add it there, we'd risk shadowing an "item" attribute.
256
  def __eq__(self, other):
257
    if isinstance(other, Private):
258
      return self._item == other._item
259
    else:
260
      return self._item == other
261

    
262
  def __hash__(self):
263
    return hash(self._item)
264

    
265
  def __format__(self, *_1, **_2):
266
    return self.__str__()
267

    
268
  def __getattr__(self, attr):
269
    return Private(getattr(self._item, attr),
270
                   descr="%s.%s" % (self._descr, attr))
271

    
272
  def __call__(self, *args, **kwargs):
273
    return Private(self._item(*args, **kwargs),
274
                   descr="%s()" % self._descr)
275

    
276
  # pylint: disable=R0201
277
  # While this could get away with being a function, it needs to be a method.
278
  # Required by the copy.deepcopy function used by FillDict.
279
  def __getnewargs__(self):
280
    return tuple()
281

    
282
  def __nonzero__(self):
283
    return bool(self._item)
284

    
285
  # Get in the way of Pickle by implementing __slots__ but not __getstate__
286
  # ...and get a performance boost, too.
287
  __slots__ = ["_item", "_descr"]
288

    
289

    
290
class PrivateDict(dict):
291
  """A dictionary that turns its values to private fields.
292

293
  >>> PrivateDict()
294
  {}
295
  >>> supersekkrit = PrivateDict({"password": "foobar"})
296
  >>> print supersekkrit["password"]
297
  <password>
298
  >>> supersekkrit["password"].Get()
299
  'foobar'
300
  >>> supersekkrit.GetPrivate("password")
301
  'foobar'
302
  >>> supersekkrit["user"] = "eggspam"
303
  >>> supersekkrit.Unprivate()
304
  {'password': 'foobar', 'user': 'eggspam'}
305

306
  """
307
  def __init__(self, data=None):
308
    dict.__init__(self)
309
    self.update(data)
310

    
311
  def __setitem__(self, item, value):
312
    if not isinstance(value, Private):
313
      if not isinstance(item, dict):
314
        value = Private(value, descr=item)
315
      else:
316
        value = PrivateDict(value)
317
    dict.__setitem__(self, item, value)
318

    
319
  # The actual conversion to Private containers is done by __setitem__
320

    
321
  # copied straight from cpython/Lib/UserDict.py
322
  # Copyright (c) 2001-2014 Python Software Foundation; All Rights Reserved
323
  def update(self, other=None, **kwargs):
324
    # Make progressively weaker assumptions about "other"
325
    if other is None:
326
      pass
327
    elif hasattr(other, 'iteritems'):  # iteritems saves memory and lookups
328
      for k, v in other.iteritems():
329
        self[k] = v
330
    elif hasattr(other, 'keys'):
331
      for k in other.keys():
332
        self[k] = other[k]
333
    else:
334
      for k, v in other:
335
        self[k] = v
336
    if kwargs:
337
      self.update(kwargs)
338

    
339
  def GetPrivate(self, *args):
340
    """Like dict.get, but extracting the value in the process.
341

342
    Arguments are semantically equivalent to ``dict.get``
343

344
    >>> PrivateDict({"foo": "bar"}).GetPrivate("foo")
345
    'bar'
346
    >>> PrivateDict({"foo": "bar"}).GetPrivate("baz", "spam")
347
    'spam'
348

349
    """
350
    if len(args) == 1:
351
      key, = args
352
      return self[key].Get()
353
    elif len(args) == 2:
354
      key, default = args
355
      if key not in self:
356
        return default
357
      else:
358
        return self[key].Get()
359
    else:
360
      raise TypeError("GetPrivate() takes 2 arguments (%d given)" % len(args))
361

    
362
  def Unprivate(self):
363
    """Turn this dict of Private() values to a dict of values.
364

365
    >>> PrivateDict({"foo": "bar"}).Unprivate()
366
    {'foo': 'bar'}
367

368
    @rtype: dict
369

370
    """
371
    returndict = {}
372
    for key in self:
373
      returndict[key] = self[key].Get()
374
    return returndict
375

    
376

    
377
def EncodeWithoutPrivateFields(obj):
378
  if isinstance(obj, Private):
379
    return None
380
  raise TypeError(repr(obj) + " is not JSON serializable")
381

    
382

    
383
def EncodeWithPrivateFields(obj):
384
  if isinstance(obj, Private):
385
    return obj.Get()
386
  raise TypeError(repr(obj) + " is not JSON serializable")