4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 """Utility functions for manipulating or working with text.
31 from ganeti import errors
34 #: Unit checker regexp
35 _PARSEUNIT_REGEX = re.compile(r"^([.\d]+)\s*([a-zA-Z]+)?$")
37 #: Characters which don't need to be quoted for shell commands
38 _SHELL_UNQUOTED_RE = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
41 _MAC_CHECK_RE = re.compile("^([0-9a-f]{2}:){5}[0-9a-f]{2}$", re.I)
43 #: Shell param checker regexp
44 _SHELLPARAM_REGEX = re.compile(r"^[-a-zA-Z0-9._+/:%@]+$")
47 def MatchNameComponent(key, name_list, case_sensitive=True):
48 """Try to match a name against a list.
50 This function will try to match a name like test1 against a list
51 like C{['test1.example.com', 'test2.example.com', ...]}. Against
52 this list, I{'test1'} as well as I{'test1.example'} will match, but
53 not I{'test1.ex'}. A multiple match will be considered as no match
54 at all (e.g. I{'test1'} against C{['test1.example.com',
55 'test1.example.org']}), except when the key fully matches an entry
56 (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
59 @param key: the name to be searched
61 @param name_list: the list of strings against which to search the key
62 @type case_sensitive: boolean
63 @param case_sensitive: whether to provide a case-sensitive match
66 @return: None if there is no match I{or} if there are multiple matches,
67 otherwise the element from the list which matches
74 if not case_sensitive:
75 re_flags |= re.IGNORECASE
77 mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
80 for name in name_list:
81 if mo.match(name) is not None:
82 names_filtered.append(name)
83 if not case_sensitive and key == name.upper():
84 string_matches.append(name)
86 if len(string_matches) == 1:
87 return string_matches[0]
88 if len(names_filtered) == 1:
89 return names_filtered[0]
93 def FormatUnit(value, units):
94 """Formats an incoming number of MiB with the appropriate unit.
97 @param value: integer representing the value in MiB (1048576)
99 @param units: the type of formatting we should do:
100 - 'h' for automatic scaling
105 @return: the formatted value (with suffix)
108 if units not in ('m', 'g', 't', 'h'):
109 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
113 if units == 'm' or (units == 'h' and value < 1024):
116 return "%d%s" % (round(value, 0), suffix)
118 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
121 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
126 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
129 def ParseUnit(input_string):
130 """Tries to extract number and scale from the given string.
132 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
133 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
134 is always an int in MiB.
137 m = _PARSEUNIT_REGEX.match(str(input_string))
139 raise errors.UnitParseError("Invalid format")
141 value = float(m.groups()[0])
145 lcunit = unit.lower()
149 if lcunit in ('m', 'mb', 'mib'):
150 # Value already in MiB
153 elif lcunit in ('g', 'gb', 'gib'):
156 elif lcunit in ('t', 'tb', 'tib'):
160 raise errors.UnitParseError("Unknown unit: %s" % unit)
162 # Make sure we round up
163 if int(value) < value:
166 # Round up to the next multiple of 4
169 value += 4 - value % 4
174 def ShellQuote(value):
175 """Quotes shell argument according to POSIX.
178 @param value: the argument to be quoted
180 @return: the quoted value
183 if _SHELL_UNQUOTED_RE.match(value):
186 return "'%s'" % value.replace("'", "'\\''")
189 def ShellQuoteArgs(args):
190 """Quotes a list of shell arguments.
193 @param args: list of arguments to be quoted
195 @return: the quoted arguments concatenated with spaces
198 return " ".join([ShellQuote(i) for i in args])
202 """Helper class to write scripts with indentation.
207 def __init__(self, fh):
208 """Initializes this class.
215 """Increase indentation level by 1.
221 """Decrease indentation level by 1.
224 assert self._indent > 0
227 def Write(self, txt, *args):
228 """Write line to output file.
231 assert self._indent >= 0
233 self._fh.write(self._indent * self.INDENT_STR)
236 self._fh.write(txt % args)
243 def GenerateSecret(numbytes=20):
244 """Generates a random secret.
246 This will generate a pseudo-random secret returning an hex string
247 (so that it can be used where an ASCII string is needed).
249 @param numbytes: the number of bytes which will be represented by the returned
250 string (defaulting to 20, the length of a SHA1 hash)
252 @return: an hex representation of the pseudo-random sequence
255 return os.urandom(numbytes).encode("hex")
258 def NormalizeAndValidateMac(mac):
259 """Normalizes and check if a MAC address is valid.
261 Checks whether the supplied MAC address is formally correct, only
262 accepts colon separated format. Normalize it to all lower.
265 @param mac: the MAC to be validated
267 @return: returns the normalized and validated MAC.
269 @raise errors.OpPrereqError: If the MAC isn't valid
272 if not _MAC_CHECK_RE.match(mac):
273 raise errors.OpPrereqError("Invalid MAC address '%s'" % mac,
279 def SafeEncode(text):
280 """Return a 'safe' version of a source string.
282 This function mangles the input string and returns a version that
283 should be safe to display/encode as ASCII. To this end, we first
284 convert it to ASCII using the 'backslashreplace' encoding which
285 should get rid of any non-ASCII chars, and then we process it
286 through a loop copied from the string repr sources in the python; we
287 don't use string_escape anymore since that escape single quotes and
288 backslashes too, and that is too much; and that escaping is not
289 stable, i.e. string_escape(string_escape(x)) != string_escape(x).
291 @type text: str or unicode
292 @param text: input data
294 @return: a safe version of text
297 if isinstance(text, unicode):
298 # only if unicode; if str already, we handle it below
299 text = text.encode('ascii', 'backslashreplace')
309 elif c < 32 or c >= 127: # non-printable
310 resu += "\\x%02x" % (c & 0xff)
316 def UnescapeAndSplit(text, sep=","):
317 """Split and unescape a string based on a given separator.
319 This function splits a string based on a separator where the
320 separator itself can be escape in order to be an element of the
321 elements. The escaping rules are (assuming coma being the
323 - a plain , separates the elements
324 - a sequence \\\\, (double backslash plus comma) is handled as a
325 backslash plus a separator comma
326 - a sequence \, (backslash plus comma) is handled as a
330 @param text: the string to split
332 @param text: the separator
334 @return: a list of strings
337 # we split the list by sep (with no escaping at this stage)
338 slist = text.split(sep)
339 # next, we revisit the elements and if any of them ended with an odd
340 # number of backslashes, then we join it with the next
344 if e1.endswith("\\"):
345 num_b = len(e1) - len(e1.rstrip("\\"))
348 # here the backslashes remain (all), and will be reduced in
350 rlist.append(e1 + sep + e2)
353 # finally, replace backslash-something with something
354 rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist]
358 def CommaJoin(names):
359 """Nicely join a set of identifiers.
361 @param names: set, list or tuple
362 @return: a string with the formatted results
365 return ", ".join([str(val) for val in names])
369 """Formats a time value.
371 @type val: float or None
372 @param val: Timestamp as returned by time.time() (seconds since Epoch,
373 1970-01-01 00:00:00 UTC)
374 @return: a string value or N/A if we don't have a valid timestamp
377 if val is None or not isinstance(val, (int, float)):
379 # these two codes works on Linux, but they are not guaranteed on all
381 return time.strftime("%F %T", time.localtime(val))
384 def FormatSeconds(secs):
385 """Formats seconds for easier reading.
388 @param secs: Number of seconds
390 @return: Formatted seconds (e.g. "2d 9h 19m 49s")
395 secs = round(secs, 0)
398 # Negative values would be a bit tricky
399 for unit, one in [("d", 24 * 60 * 60), ("h", 60 * 60), ("m", 60)]:
400 (complete, secs) = divmod(secs, one)
401 if complete or parts:
402 parts.append("%d%s" % (complete, unit))
404 parts.append("%ds" % secs)
406 return " ".join(parts)
410 """Splits data chunks into lines separated by newline.
412 Instances provide a file-like interface.
415 def __init__(self, line_fn, *args):
416 """Initializes this class.
418 @type line_fn: callable
419 @param line_fn: Function called for each line, first parameter is line
420 @param args: Extra arguments for L{line_fn}
423 assert callable(line_fn)
426 # Python 2.4 doesn't have functools.partial yet
428 lambda line: line_fn(line, *args) # pylint: disable-msg=W0142
430 self._line_fn = line_fn
432 self._lines = collections.deque()
435 def write(self, data):
436 parts = (self._buffer + data).split("\n")
437 self._buffer = parts.pop()
438 self._lines.extend(parts)
442 self._line_fn(self._lines.popleft().rstrip("\r\n"))
447 self._line_fn(self._buffer)
450 def IsValidShellParam(word):
451 """Verifies is the given word is safe from the shell's p.o.v.
453 This means that we can pass this to a command via the shell and be
454 sure that it doesn't alter the command line and is passed as such to
457 Note that we are overly restrictive here, in order to be on the safe
461 @param word: the word to check
463 @return: True if the word is 'safe'
466 return bool(_SHELLPARAM_REGEX.match(word))
469 def BuildShellCmd(template, *args):
470 """Build a safe shell command line from the given arguments.
472 This function will check all arguments in the args list so that they
473 are valid shell parameters (i.e. they don't contain shell
474 metacharacters). If everything is ok, it will return the result of
478 @param template: the string holding the template for the
481 @return: the expanded command line
485 if not IsValidShellParam(word):
486 raise errors.ProgrammerError("Shell argument '%s' contains"
487 " invalid characters" % word)
488 return template % args