Statistics
| Branch: | Tag: | Revision:

root / lib / utils / text.py @ dce20078

History | View | Annotate | Download (14.1 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21
"""Utility functions for manipulating or working with text.
22

23
"""
24

    
25

    
26
import re
27
import os
28
import time
29
import collections
30

    
31
from ganeti import errors
32

    
33

    
34
#: Unit checker regexp
35
_PARSEUNIT_REGEX = re.compile(r"^([.\d]+)\s*([a-zA-Z]+)?$")
36

    
37
#: Characters which don't need to be quoted for shell commands
38
_SHELL_UNQUOTED_RE = re.compile("^[-.,=:/_+@A-Za-z0-9]+$")
39

    
40
#: MAC checker regexp
41
_MAC_CHECK_RE = re.compile("^([0-9a-f]{2}:){5}[0-9a-f]{2}$", re.I)
42

    
43
#: Shell param checker regexp
44
_SHELLPARAM_REGEX = re.compile(r"^[-a-zA-Z0-9._+/:%@]+$")
45

    
46

    
47
def MatchNameComponent(key, name_list, case_sensitive=True):
48
  """Try to match a name against a list.
49

50
  This function will try to match a name like test1 against a list
51
  like C{['test1.example.com', 'test2.example.com', ...]}. Against
52
  this list, I{'test1'} as well as I{'test1.example'} will match, but
53
  not I{'test1.ex'}. A multiple match will be considered as no match
54
  at all (e.g. I{'test1'} against C{['test1.example.com',
55
  'test1.example.org']}), except when the key fully matches an entry
56
  (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
57

58
  @type key: str
59
  @param key: the name to be searched
60
  @type name_list: list
61
  @param name_list: the list of strings against which to search the key
62
  @type case_sensitive: boolean
63
  @param case_sensitive: whether to provide a case-sensitive match
64

65
  @rtype: None or str
66
  @return: None if there is no match I{or} if there are multiple matches,
67
      otherwise the element from the list which matches
68

69
  """
70
  if key in name_list:
71
    return key
72

    
73
  re_flags = 0
74
  if not case_sensitive:
75
    re_flags |= re.IGNORECASE
76
    key = key.upper()
77

    
78
  name_re = re.compile(r"^%s(\..*)?$" % re.escape(key), re_flags)
79

    
80
  names_filtered = []
81
  string_matches = []
82
  for name in name_list:
83
    if name_re.match(name) is not None:
84
      names_filtered.append(name)
85
      if not case_sensitive and key == name.upper():
86
        string_matches.append(name)
87

    
88
  if len(string_matches) == 1:
89
    return string_matches[0]
90
  if len(names_filtered) == 1:
91
    return names_filtered[0]
92

    
93
  return None
94

    
95

    
96
def _DnsNameGlobHelper(match):
97
  """Helper function for L{DnsNameGlobPattern}.
98

99
  Returns regular expression pattern for parts of the pattern.
100

101
  """
102
  text = match.group(0)
103

    
104
  if text == "*":
105
    return "[^.]*"
106
  elif text == "?":
107
    return "[^.]"
108
  else:
109
    return re.escape(text)
110

    
111

    
112
def DnsNameGlobPattern(pattern):
113
  """Generates regular expression from DNS name globbing pattern.
114

115
  A DNS name globbing pattern (e.g. C{*.site}) is converted to a regular
116
  expression. Escape sequences or ranges (e.g. [a-z]) are not supported.
117

118
  Matching always starts at the leftmost part. An asterisk (*) matches all
119
  characters except the dot (.) separating DNS name parts. A question mark (?)
120
  matches a single character except the dot (.).
121

122
  @type pattern: string
123
  @param pattern: DNS name globbing pattern
124
  @rtype: string
125
  @return: Regular expression
126

127
  """
128
  return r"^%s(\..*)?$" % re.sub(r"\*|\?|[^*?]*", _DnsNameGlobHelper, pattern)
129

    
130

    
131
def FormatUnit(value, units):
132
  """Formats an incoming number of MiB with the appropriate unit.
133

134
  @type value: int
135
  @param value: integer representing the value in MiB (1048576)
136
  @type units: char
137
  @param units: the type of formatting we should do:
138
      - 'h' for automatic scaling
139
      - 'm' for MiBs
140
      - 'g' for GiBs
141
      - 't' for TiBs
142
  @rtype: str
143
  @return: the formatted value (with suffix)
144

145
  """
146
  if units not in ("m", "g", "t", "h"):
147
    raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
148

    
149
  suffix = ""
150

    
151
  if units == "m" or (units == "h" and value < 1024):
152
    if units == "h":
153
      suffix = "M"
154
    return "%d%s" % (round(value, 0), suffix)
155

    
156
  elif units == "g" or (units == "h" and value < (1024 * 1024)):
157
    if units == "h":
158
      suffix = "G"
159
    return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
160

    
161
  else:
162
    if units == "h":
163
      suffix = "T"
164
    return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
165

    
166

    
167
def ParseUnit(input_string):
168
  """Tries to extract number and scale from the given string.
169

170
  Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
171
  [UNIT]}. If no unit is specified, it defaults to MiB. Return value
172
  is always an int in MiB.
173

174
  """
175
  m = _PARSEUNIT_REGEX.match(str(input_string))
176
  if not m:
177
    raise errors.UnitParseError("Invalid format")
178

    
179
  value = float(m.groups()[0])
180

    
181
  unit = m.groups()[1]
182
  if unit:
183
    lcunit = unit.lower()
184
  else:
185
    lcunit = "m"
186

    
187
  if lcunit in ("m", "mb", "mib"):
188
    # Value already in MiB
189
    pass
190

    
191
  elif lcunit in ("g", "gb", "gib"):
192
    value *= 1024
193

    
194
  elif lcunit in ("t", "tb", "tib"):
195
    value *= 1024 * 1024
196

    
197
  else:
198
    raise errors.UnitParseError("Unknown unit: %s" % unit)
199

    
200
  # Make sure we round up
201
  if int(value) < value:
202
    value += 1
203

    
204
  # Round up to the next multiple of 4
205
  value = int(value)
206
  if value % 4:
207
    value += 4 - value % 4
208

    
209
  return value
210

    
211

    
212
def ShellQuote(value):
213
  """Quotes shell argument according to POSIX.
214

215
  @type value: str
216
  @param value: the argument to be quoted
217
  @rtype: str
218
  @return: the quoted value
219

220
  """
221
  if _SHELL_UNQUOTED_RE.match(value):
222
    return value
223
  else:
224
    return "'%s'" % value.replace("'", "'\\''")
225

    
226

    
227
def ShellQuoteArgs(args):
228
  """Quotes a list of shell arguments.
229

230
  @type args: list
231
  @param args: list of arguments to be quoted
232
  @rtype: str
233
  @return: the quoted arguments concatenated with spaces
234

235
  """
236
  return " ".join([ShellQuote(i) for i in args])
237

    
238

    
239
class ShellWriter:
240
  """Helper class to write scripts with indentation.
241

242
  """
243
  INDENT_STR = "  "
244

    
245
  def __init__(self, fh):
246
    """Initializes this class.
247

248
    """
249
    self._fh = fh
250
    self._indent = 0
251

    
252
  def IncIndent(self):
253
    """Increase indentation level by 1.
254

255
    """
256
    self._indent += 1
257

    
258
  def DecIndent(self):
259
    """Decrease indentation level by 1.
260

261
    """
262
    assert self._indent > 0
263
    self._indent -= 1
264

    
265
  def Write(self, txt, *args):
266
    """Write line to output file.
267

268
    """
269
    assert self._indent >= 0
270

    
271
    if args:
272
      line = txt % args
273
    else:
274
      line = txt
275

    
276
    if line:
277
      # Indent only if there's something on the line
278
      self._fh.write(self._indent * self.INDENT_STR)
279

    
280
    self._fh.write(line)
281

    
282
    self._fh.write("\n")
283

    
284

    
285
def GenerateSecret(numbytes=20):
286
  """Generates a random secret.
287

288
  This will generate a pseudo-random secret returning an hex string
289
  (so that it can be used where an ASCII string is needed).
290

291
  @param numbytes: the number of bytes which will be represented by the returned
292
      string (defaulting to 20, the length of a SHA1 hash)
293
  @rtype: str
294
  @return: an hex representation of the pseudo-random sequence
295

296
  """
297
  return os.urandom(numbytes).encode("hex")
298

    
299

    
300
def NormalizeAndValidateMac(mac):
301
  """Normalizes and check if a MAC address is valid.
302

303
  Checks whether the supplied MAC address is formally correct, only
304
  accepts colon separated format. Normalize it to all lower.
305

306
  @type mac: str
307
  @param mac: the MAC to be validated
308
  @rtype: str
309
  @return: returns the normalized and validated MAC.
310

311
  @raise errors.OpPrereqError: If the MAC isn't valid
312

313
  """
314
  if not _MAC_CHECK_RE.match(mac):
315
    raise errors.OpPrereqError("Invalid MAC address '%s'" % mac,
316
                               errors.ECODE_INVAL)
317

    
318
  return mac.lower()
319

    
320

    
321
def SafeEncode(text):
322
  """Return a 'safe' version of a source string.
323

324
  This function mangles the input string and returns a version that
325
  should be safe to display/encode as ASCII. To this end, we first
326
  convert it to ASCII using the 'backslashreplace' encoding which
327
  should get rid of any non-ASCII chars, and then we process it
328
  through a loop copied from the string repr sources in the python; we
329
  don't use string_escape anymore since that escape single quotes and
330
  backslashes too, and that is too much; and that escaping is not
331
  stable, i.e. string_escape(string_escape(x)) != string_escape(x).
332

333
  @type text: str or unicode
334
  @param text: input data
335
  @rtype: str
336
  @return: a safe version of text
337

338
  """
339
  if isinstance(text, unicode):
340
    # only if unicode; if str already, we handle it below
341
    text = text.encode("ascii", "backslashreplace")
342
  resu = ""
343
  for char in text:
344
    c = ord(char)
345
    if char == "\t":
346
      resu += r"\t"
347
    elif char == "\n":
348
      resu += r"\n"
349
    elif char == "\r":
350
      resu += r'\'r'
351
    elif c < 32 or c >= 127: # non-printable
352
      resu += "\\x%02x" % (c & 0xff)
353
    else:
354
      resu += char
355
  return resu
356

    
357

    
358
def UnescapeAndSplit(text, sep=","):
359
  """Split and unescape a string based on a given separator.
360

361
  This function splits a string based on a separator where the
362
  separator itself can be escape in order to be an element of the
363
  elements. The escaping rules are (assuming coma being the
364
  separator):
365
    - a plain , separates the elements
366
    - a sequence \\\\, (double backslash plus comma) is handled as a
367
      backslash plus a separator comma
368
    - a sequence \, (backslash plus comma) is handled as a
369
      non-separator comma
370

371
  @type text: string
372
  @param text: the string to split
373
  @type sep: string
374
  @param text: the separator
375
  @rtype: string
376
  @return: a list of strings
377

378
  """
379
  # we split the list by sep (with no escaping at this stage)
380
  slist = text.split(sep)
381
  # next, we revisit the elements and if any of them ended with an odd
382
  # number of backslashes, then we join it with the next
383
  rlist = []
384
  while slist:
385
    e1 = slist.pop(0)
386
    if e1.endswith("\\"):
387
      num_b = len(e1) - len(e1.rstrip("\\"))
388
      if num_b % 2 == 1 and slist:
389
        e2 = slist.pop(0)
390
        # here the backslashes remain (all), and will be reduced in
391
        # the next step
392
        rlist.append(e1 + sep + e2)
393
        continue
394
    rlist.append(e1)
395
  # finally, replace backslash-something with something
396
  rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist]
397
  return rlist
398

    
399

    
400
def CommaJoin(names):
401
  """Nicely join a set of identifiers.
402

403
  @param names: set, list or tuple
404
  @return: a string with the formatted results
405

406
  """
407
  return ", ".join([str(val) for val in names])
408

    
409

    
410
def FormatTime(val):
411
  """Formats a time value.
412

413
  @type val: float or None
414
  @param val: Timestamp as returned by time.time() (seconds since Epoch,
415
    1970-01-01 00:00:00 UTC)
416
  @return: a string value or N/A if we don't have a valid timestamp
417

418
  """
419
  if val is None or not isinstance(val, (int, float)):
420
    return "N/A"
421
  # these two codes works on Linux, but they are not guaranteed on all
422
  # platforms
423
  return time.strftime("%F %T", time.localtime(val))
424

    
425

    
426
def FormatSeconds(secs):
427
  """Formats seconds for easier reading.
428

429
  @type secs: number
430
  @param secs: Number of seconds
431
  @rtype: string
432
  @return: Formatted seconds (e.g. "2d 9h 19m 49s")
433

434
  """
435
  parts = []
436

    
437
  secs = round(secs, 0)
438

    
439
  if secs > 0:
440
    # Negative values would be a bit tricky
441
    for unit, one in [("d", 24 * 60 * 60), ("h", 60 * 60), ("m", 60)]:
442
      (complete, secs) = divmod(secs, one)
443
      if complete or parts:
444
        parts.append("%d%s" % (complete, unit))
445

    
446
  parts.append("%ds" % secs)
447

    
448
  return " ".join(parts)
449

    
450

    
451
class LineSplitter:
452
  """Splits data chunks into lines separated by newline.
453

454
  Instances provide a file-like interface.
455

456
  """
457
  def __init__(self, line_fn, *args):
458
    """Initializes this class.
459

460
    @type line_fn: callable
461
    @param line_fn: Function called for each line, first parameter is line
462
    @param args: Extra arguments for L{line_fn}
463

464
    """
465
    assert callable(line_fn)
466

    
467
    if args:
468
      # Python 2.4 doesn't have functools.partial yet
469
      self._line_fn = \
470
        lambda line: line_fn(line, *args) # pylint: disable=W0142
471
    else:
472
      self._line_fn = line_fn
473

    
474
    self._lines = collections.deque()
475
    self._buffer = ""
476

    
477
  def write(self, data):
478
    parts = (self._buffer + data).split("\n")
479
    self._buffer = parts.pop()
480
    self._lines.extend(parts)
481

    
482
  def flush(self):
483
    while self._lines:
484
      self._line_fn(self._lines.popleft().rstrip("\r\n"))
485

    
486
  def close(self):
487
    self.flush()
488
    if self._buffer:
489
      self._line_fn(self._buffer)
490

    
491

    
492
def IsValidShellParam(word):
493
  """Verifies is the given word is safe from the shell's p.o.v.
494

495
  This means that we can pass this to a command via the shell and be
496
  sure that it doesn't alter the command line and is passed as such to
497
  the actual command.
498

499
  Note that we are overly restrictive here, in order to be on the safe
500
  side.
501

502
  @type word: str
503
  @param word: the word to check
504
  @rtype: boolean
505
  @return: True if the word is 'safe'
506

507
  """
508
  return bool(_SHELLPARAM_REGEX.match(word))
509

    
510

    
511
def BuildShellCmd(template, *args):
512
  """Build a safe shell command line from the given arguments.
513

514
  This function will check all arguments in the args list so that they
515
  are valid shell parameters (i.e. they don't contain shell
516
  metacharacters). If everything is ok, it will return the result of
517
  template % args.
518

519
  @type template: str
520
  @param template: the string holding the template for the
521
      string formatting
522
  @rtype: str
523
  @return: the expanded command line
524

525
  """
526
  for word in args:
527
    if not IsValidShellParam(word):
528
      raise errors.ProgrammerError("Shell argument '%s' contains"
529
                                   " invalid characters" % word)
530
  return template % args
531

    
532

    
533
def FormatOrdinal(value):
534
  """Formats a number as an ordinal in the English language.
535

536
  E.g. the number 1 becomes "1st", 22 becomes "22nd".
537

538
  @type value: integer
539
  @param value: Number
540
  @rtype: string
541

542
  """
543
  tens = value % 10
544

    
545
  if value > 10 and value < 20:
546
    suffix = "th"
547
  elif tens == 1:
548
    suffix = "st"
549
  elif tens == 2:
550
    suffix = "nd"
551
  elif tens == 3:
552
    suffix = "rd"
553
  else:
554
    suffix = "th"
555

    
556
  return "%s%s" % (value, suffix)