4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 """Utility functions for manipulating or working with text.
31 from ganeti import errors
34 #: Unit checker regexp
35 _PARSEUNIT_REGEX = re.compile(r"^([.\d]+)\s*([a-zA-Z]+)?$")
37 #: Characters which don't need to be quoted for shell commands
38 _SHELL_UNQUOTED_RE = re.compile("^[-.,=:/_+@A-Za-z0-9]+$")
41 _MAC_CHECK_RE = re.compile("^([0-9a-f]{2}:){5}[0-9a-f]{2}$", re.I)
43 #: Shell param checker regexp
44 _SHELLPARAM_REGEX = re.compile(r"^[-a-zA-Z0-9._+/:%@]+$")
47 def MatchNameComponent(key, name_list, case_sensitive=True):
48 """Try to match a name against a list.
50 This function will try to match a name like test1 against a list
51 like C{['test1.example.com', 'test2.example.com', ...]}. Against
52 this list, I{'test1'} as well as I{'test1.example'} will match, but
53 not I{'test1.ex'}. A multiple match will be considered as no match
54 at all (e.g. I{'test1'} against C{['test1.example.com',
55 'test1.example.org']}), except when the key fully matches an entry
56 (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
59 @param key: the name to be searched
61 @param name_list: the list of strings against which to search the key
62 @type case_sensitive: boolean
63 @param case_sensitive: whether to provide a case-sensitive match
66 @return: None if there is no match I{or} if there are multiple matches,
67 otherwise the element from the list which matches
74 if not case_sensitive:
75 re_flags |= re.IGNORECASE
78 name_re = re.compile(r"^%s(\..*)?$" % re.escape(key), re_flags)
82 for name in name_list:
83 if name_re.match(name) is not None:
84 names_filtered.append(name)
85 if not case_sensitive and key == name.upper():
86 string_matches.append(name)
88 if len(string_matches) == 1:
89 return string_matches[0]
90 if len(names_filtered) == 1:
91 return names_filtered[0]
96 def _DnsNameGlobHelper(match):
97 """Helper function for L{DnsNameGlobPattern}.
99 Returns regular expression pattern for parts of the pattern.
102 text = match.group(0)
109 return re.escape(text)
112 def DnsNameGlobPattern(pattern):
113 """Generates regular expression from DNS name globbing pattern.
115 A DNS name globbing pattern (e.g. C{*.site}) is converted to a regular
116 expression. Escape sequences or ranges (e.g. [a-z]) are not supported.
118 Matching always starts at the leftmost part. An asterisk (*) matches all
119 characters except the dot (.) separating DNS name parts. A question mark (?)
120 matches a single character except the dot (.).
122 @type pattern: string
123 @param pattern: DNS name globbing pattern
125 @return: Regular expression
128 return r"^%s(\..*)?$" % re.sub(r"\*|\?|[^*?]*", _DnsNameGlobHelper, pattern)
131 def FormatUnit(value, units):
132 """Formats an incoming number of MiB with the appropriate unit.
135 @param value: integer representing the value in MiB (1048576)
137 @param units: the type of formatting we should do:
138 - 'h' for automatic scaling
143 @return: the formatted value (with suffix)
146 if units not in ("m", "g", "t", "h"):
147 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
151 if units == "m" or (units == "h" and value < 1024):
154 return "%d%s" % (round(value, 0), suffix)
156 elif units == "g" or (units == "h" and value < (1024 * 1024)):
159 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
164 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
167 def ParseUnit(input_string):
168 """Tries to extract number and scale from the given string.
170 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
171 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
172 is always an int in MiB.
175 m = _PARSEUNIT_REGEX.match(str(input_string))
177 raise errors.UnitParseError("Invalid format")
179 value = float(m.groups()[0])
183 lcunit = unit.lower()
187 if lcunit in ("m", "mb", "mib"):
188 # Value already in MiB
191 elif lcunit in ("g", "gb", "gib"):
194 elif lcunit in ("t", "tb", "tib"):
198 raise errors.UnitParseError("Unknown unit: %s" % unit)
200 # Make sure we round up
201 if int(value) < value:
204 # Round up to the next multiple of 4
207 value += 4 - value % 4
212 def ShellQuote(value):
213 """Quotes shell argument according to POSIX.
216 @param value: the argument to be quoted
218 @return: the quoted value
221 if _SHELL_UNQUOTED_RE.match(value):
224 return "'%s'" % value.replace("'", "'\\''")
227 def ShellQuoteArgs(args):
228 """Quotes a list of shell arguments.
231 @param args: list of arguments to be quoted
233 @return: the quoted arguments concatenated with spaces
236 return " ".join([ShellQuote(i) for i in args])
240 """Helper class to write scripts with indentation.
245 def __init__(self, fh):
246 """Initializes this class.
253 """Increase indentation level by 1.
259 """Decrease indentation level by 1.
262 assert self._indent > 0
265 def Write(self, txt, *args):
266 """Write line to output file.
269 assert self._indent >= 0
277 # Indent only if there's something on the line
278 self._fh.write(self._indent * self.INDENT_STR)
285 def GenerateSecret(numbytes=20):
286 """Generates a random secret.
288 This will generate a pseudo-random secret returning an hex string
289 (so that it can be used where an ASCII string is needed).
291 @param numbytes: the number of bytes which will be represented by the returned
292 string (defaulting to 20, the length of a SHA1 hash)
294 @return: an hex representation of the pseudo-random sequence
297 return os.urandom(numbytes).encode("hex")
300 def NormalizeAndValidateMac(mac):
301 """Normalizes and check if a MAC address is valid.
303 Checks whether the supplied MAC address is formally correct, only
304 accepts colon separated format. Normalize it to all lower.
307 @param mac: the MAC to be validated
309 @return: returns the normalized and validated MAC.
311 @raise errors.OpPrereqError: If the MAC isn't valid
314 if not _MAC_CHECK_RE.match(mac):
315 raise errors.OpPrereqError("Invalid MAC address '%s'" % mac,
321 def SafeEncode(text):
322 """Return a 'safe' version of a source string.
324 This function mangles the input string and returns a version that
325 should be safe to display/encode as ASCII. To this end, we first
326 convert it to ASCII using the 'backslashreplace' encoding which
327 should get rid of any non-ASCII chars, and then we process it
328 through a loop copied from the string repr sources in the python; we
329 don't use string_escape anymore since that escape single quotes and
330 backslashes too, and that is too much; and that escaping is not
331 stable, i.e. string_escape(string_escape(x)) != string_escape(x).
333 @type text: str or unicode
334 @param text: input data
336 @return: a safe version of text
339 if isinstance(text, unicode):
340 # only if unicode; if str already, we handle it below
341 text = text.encode("ascii", "backslashreplace")
351 elif c < 32 or c >= 127: # non-printable
352 resu += "\\x%02x" % (c & 0xff)
358 def UnescapeAndSplit(text, sep=","):
359 """Split and unescape a string based on a given separator.
361 This function splits a string based on a separator where the
362 separator itself can be escape in order to be an element of the
363 elements. The escaping rules are (assuming coma being the
365 - a plain , separates the elements
366 - a sequence \\\\, (double backslash plus comma) is handled as a
367 backslash plus a separator comma
368 - a sequence \, (backslash plus comma) is handled as a
372 @param text: the string to split
374 @param text: the separator
376 @return: a list of strings
379 # we split the list by sep (with no escaping at this stage)
380 slist = text.split(sep)
381 # next, we revisit the elements and if any of them ended with an odd
382 # number of backslashes, then we join it with the next
386 if e1.endswith("\\"):
387 num_b = len(e1) - len(e1.rstrip("\\"))
388 if num_b % 2 == 1 and slist:
390 # here the backslashes remain (all), and will be reduced in
392 rlist.append(e1 + sep + e2)
395 # finally, replace backslash-something with something
396 rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist]
400 def CommaJoin(names):
401 """Nicely join a set of identifiers.
403 @param names: set, list or tuple
404 @return: a string with the formatted results
407 return ", ".join([str(val) for val in names])
411 """Formats a time value.
413 @type val: float or None
414 @param val: Timestamp as returned by time.time() (seconds since Epoch,
415 1970-01-01 00:00:00 UTC)
416 @return: a string value or N/A if we don't have a valid timestamp
419 if val is None or not isinstance(val, (int, float)):
421 # these two codes works on Linux, but they are not guaranteed on all
423 return time.strftime("%F %T", time.localtime(val))
426 def FormatSeconds(secs):
427 """Formats seconds for easier reading.
430 @param secs: Number of seconds
432 @return: Formatted seconds (e.g. "2d 9h 19m 49s")
437 secs = round(secs, 0)
440 # Negative values would be a bit tricky
441 for unit, one in [("d", 24 * 60 * 60), ("h", 60 * 60), ("m", 60)]:
442 (complete, secs) = divmod(secs, one)
443 if complete or parts:
444 parts.append("%d%s" % (complete, unit))
446 parts.append("%ds" % secs)
448 return " ".join(parts)
452 """Splits data chunks into lines separated by newline.
454 Instances provide a file-like interface.
457 def __init__(self, line_fn, *args):
458 """Initializes this class.
460 @type line_fn: callable
461 @param line_fn: Function called for each line, first parameter is line
462 @param args: Extra arguments for L{line_fn}
465 assert callable(line_fn)
468 # Python 2.4 doesn't have functools.partial yet
470 lambda line: line_fn(line, *args) # pylint: disable=W0142
472 self._line_fn = line_fn
474 self._lines = collections.deque()
477 def write(self, data):
478 parts = (self._buffer + data).split("\n")
479 self._buffer = parts.pop()
480 self._lines.extend(parts)
484 self._line_fn(self._lines.popleft().rstrip("\r\n"))
489 self._line_fn(self._buffer)
492 def IsValidShellParam(word):
493 """Verifies is the given word is safe from the shell's p.o.v.
495 This means that we can pass this to a command via the shell and be
496 sure that it doesn't alter the command line and is passed as such to
499 Note that we are overly restrictive here, in order to be on the safe
503 @param word: the word to check
505 @return: True if the word is 'safe'
508 return bool(_SHELLPARAM_REGEX.match(word))
511 def BuildShellCmd(template, *args):
512 """Build a safe shell command line from the given arguments.
514 This function will check all arguments in the args list so that they
515 are valid shell parameters (i.e. they don't contain shell
516 metacharacters). If everything is ok, it will return the result of
520 @param template: the string holding the template for the
523 @return: the expanded command line
527 if not IsValidShellParam(word):
528 raise errors.ProgrammerError("Shell argument '%s' contains"
529 " invalid characters" % word)
530 return template % args
533 def FormatOrdinal(value):
534 """Formats a number as an ordinal in the English language.
536 E.g. the number 1 becomes "1st", 22 becomes "22nd".
545 if value > 10 and value < 20:
556 return "%s%s" % (value, suffix)