X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/26288e6803bc2976ce51805f2af43dfd28e32d1d..cc0d88e95afa0be2cf4aa563904016c5a067d4cd:/lib/utils.py?ds=sidebyside diff --git a/lib/utils.py b/lib/utils.py index 131f4c1..c32ad20 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -43,6 +43,8 @@ import resource import logging import logging.handlers import signal +import datetime +import calendar from cStringIO import StringIO @@ -363,20 +365,29 @@ def RenameFile(old, new, mkdir=False, mkdir_mode=0750): # as efficient. if mkdir and err.errno == errno.ENOENT: # Create directory and try again - dirname = os.path.dirname(new) - try: - os.makedirs(dirname, mode=mkdir_mode) - except OSError, err: - # Ignore EEXIST. This is only handled in os.makedirs as included in - # Python 2.5 and above. - if err.errno != errno.EEXIST or not os.path.exists(dirname): - raise + Makedirs(os.path.dirname(new), mode=mkdir_mode) return os.rename(old, new) raise +def Makedirs(path, mode=0750): + """Super-mkdir; create a leaf directory and all intermediate ones. + + This is a wrapper around C{os.makedirs} adding error handling not implemented + before Python 2.5. + + """ + try: + os.makedirs(path, mode) + except OSError, err: + # Ignore EEXIST. This is only handled in os.makedirs as included in + # Python 2.5 and above. + if err.errno != errno.EEXIST or not os.path.exists(path): + raise + + def ResetTempfileModule(): """Resets the random name generator of the tempfile module. @@ -1117,6 +1128,16 @@ def RemoveHostFromEtcHosts(hostname): RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName()) +def TimestampForFilename(): + """Returns the current time formatted for filenames. + + The format doesn't contain colons as some shells and applications them as + separators. + + """ + return time.strftime("%Y-%m-%d_%H_%M_%S") + + def CreateBackup(file_name): """Creates a backup of a file. @@ -1131,7 +1152,8 @@ def CreateBackup(file_name): raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" % file_name) - prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time())) + prefix = ("%s.backup-%s." % + (os.path.basename(file_name), TimestampForFilename())) dir_name = os.path.dirname(file_name) fsrc = open(file_name, 'rb') @@ -1139,6 +1161,7 @@ def CreateBackup(file_name): (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name) fdst = os.fdopen(fd, 'wb') try: + logging.debug("Backing up %s at %s", file_name, backup_name) shutil.copyfileobj(fsrc, fdst) finally: fdst.close() @@ -1477,6 +1500,103 @@ def any(seq, pred=bool): # pylint: disable-msg=W0622 return False +def SingleWaitForFdCondition(fdobj, event, timeout): + """Waits for a condition to occur on the socket. + + Immediately returns at the first interruption. + + @type fdobj: integer or object supporting a fileno() method + @param fdobj: entity to wait for events on + @type event: integer + @param event: ORed condition (see select module) + @type timeout: float or None + @param timeout: Timeout in seconds + @rtype: int or None + @return: None for timeout, otherwise occured conditions + + """ + check = (event | select.POLLPRI | + select.POLLNVAL | select.POLLHUP | select.POLLERR) + + if timeout is not None: + # Poller object expects milliseconds + timeout *= 1000 + + poller = select.poll() + poller.register(fdobj, event) + try: + # TODO: If the main thread receives a signal and we have no timeout, we + # could wait forever. This should check a global "quit" flag or something + # every so often. + io_events = poller.poll(timeout) + except select.error, err: + if err[0] != errno.EINTR: + raise + io_events = [] + if io_events and io_events[0][1] & check: + return io_events[0][1] + else: + return None + + +class FdConditionWaiterHelper(object): + """Retry helper for WaitForFdCondition. + + This class contains the retried and wait functions that make sure + WaitForFdCondition can continue waiting until the timeout is actually + expired. + + """ + + def __init__(self, timeout): + self.timeout = timeout + + def Poll(self, fdobj, event): + result = SingleWaitForFdCondition(fdobj, event, self.timeout) + if result is None: + raise RetryAgain() + else: + return result + + def UpdateTimeout(self, timeout): + self.timeout = timeout + + +def WaitForFdCondition(fdobj, event, timeout): + """Waits for a condition to occur on the socket. + + Retries until the timeout is expired, even if interrupted. + + @type fdobj: integer or object supporting a fileno() method + @param fdobj: entity to wait for events on + @type event: integer + @param event: ORed condition (see select module) + @type timeout: float or None + @param timeout: Timeout in seconds + @rtype: int or None + @return: None for timeout, otherwise occured conditions + + """ + if timeout is not None: + retrywaiter = FdConditionWaiterHelper(timeout) + try: + result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout, + args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout) + except RetryTimeout: + result = None + else: + result = None + while result is None: + result = SingleWaitForFdCondition(fdobj, event, timeout) + return result + + +def partition(seq, pred=bool): # # pylint: disable-msg=W0622 + "Partition a list in two, based on the given predicate" + return (list(itertools.ifilter(pred, seq)), + list(itertools.ifilterfalse(pred, seq))) + + def UniqueSequence(seq): """Returns a list with unique elements. @@ -2020,6 +2140,69 @@ def TailFile(fname, lines=20): return rows[-lines:] +def _ParseAsn1Generalizedtime(value): + """Parses an ASN1 GENERALIZEDTIME timestamp as used by pyOpenSSL. + + @type value: string + @param value: ASN1 GENERALIZEDTIME timestamp + + """ + m = re.match(r"^(\d+)([-+]\d\d)(\d\d)$", value) + if m: + # We have an offset + asn1time = m.group(1) + hours = int(m.group(2)) + minutes = int(m.group(3)) + utcoffset = (60 * hours) + minutes + else: + if not value.endswith("Z"): + raise ValueError("Missing timezone") + asn1time = value[:-1] + utcoffset = 0 + + parsed = time.strptime(asn1time, "%Y%m%d%H%M%S") + + tt = datetime.datetime(*(parsed[:7])) - datetime.timedelta(minutes=utcoffset) + + return calendar.timegm(tt.utctimetuple()) + + +def GetX509CertValidity(cert): + """Returns the validity period of the certificate. + + @type cert: OpenSSL.crypto.X509 + @param cert: X509 certificate object + + """ + # The get_notBefore and get_notAfter functions are only supported in + # pyOpenSSL 0.7 and above. + try: + get_notbefore_fn = cert.get_notBefore + except AttributeError: + not_before = None + else: + not_before_asn1 = get_notbefore_fn() + + if not_before_asn1 is None: + not_before = None + else: + not_before = _ParseAsn1Generalizedtime(not_before_asn1) + + try: + get_notafter_fn = cert.get_notAfter + except AttributeError: + not_after = None + else: + not_after_asn1 = get_notafter_fn() + + if not_after_asn1 is None: + not_after = None + else: + not_after = _ParseAsn1Generalizedtime(not_after_asn1) + + return (not_before, not_after) + + def SafeEncode(text): """Return a 'safe' version of a source string. @@ -2156,15 +2339,15 @@ def GetFilesystemStats(path): return (tsize, fsize) -def RunInSeparateProcess(fn): +def RunInSeparateProcess(fn, *args): """Runs a function in a separate process. Note: Only boolean return values are supported. @type fn: callable @param fn: Function to be called - @rtype: tuple of (int/None, int/None) - @return: Exit code and signal number + @rtype: bool + @return: Function's result """ pid = os.fork() @@ -2175,7 +2358,7 @@ def RunInSeparateProcess(fn): ResetTempfileModule() # Call function - result = int(bool(fn())) + result = int(bool(fn(*args))) assert result in (0, 1) except: # pylint: disable-msg=W0702 logging.exception("Error while calling function in separate process") @@ -2426,6 +2609,9 @@ def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep, return fn(*args) except RetryAgain: pass + except RetryTimeout: + raise errors.ProgrammerError("Nested retry loop detected that didn't" + " handle RetryTimeout") remaining_time = end_time - _time_fn() @@ -2446,17 +2632,31 @@ class FileLock(object): """Utility class for file locks. """ - def __init__(self, filename): + def __init__(self, fd, filename): """Constructor for FileLock. - This will open the file denoted by the I{filename} argument. - + @type fd: file + @param fd: File object @type filename: str - @param filename: path to the file to be locked + @param filename: Path of the file opened at I{fd} """ + self.fd = fd self.filename = filename - self.fd = open(self.filename, "w") + + @classmethod + def Open(cls, filename): + """Creates and opens a file to be used as a file-based lock. + + @type filename: string + @param filename: path to the file to be locked + + """ + # Using "os.open" is necessary to allow both opening existing file + # read/write and creating if not existing. Vanilla "open" will truncate an + # existing file -or- allow creating if not existing. + return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"), + filename) def __del__(self): self.Close() @@ -2486,33 +2686,31 @@ class FileLock(object): assert self.fd, "Lock was closed" assert timeout is None or timeout >= 0, \ "If specified, timeout must be positive" + assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set" - if timeout is not None: + # When a timeout is used, LOCK_NB must always be set + if not (timeout is None and blocking): flag |= fcntl.LOCK_NB - timeout_end = time.time() + timeout - # Blocking doesn't have effect with timeout - elif not blocking: - flag |= fcntl.LOCK_NB - timeout_end = None + if timeout is None: + self._Lock(self.fd, flag, timeout) + else: + try: + Retry(self._Lock, (0.1, 1.2, 1.0), timeout, + args=(self.fd, flag, timeout)) + except RetryTimeout: + raise errors.LockError(errmsg) - # TODO: Convert to utils.Retry + @staticmethod + def _Lock(fd, flag, timeout): + try: + fcntl.flock(fd, flag) + except IOError, err: + if timeout is not None and err.errno == errno.EAGAIN: + raise RetryAgain() - retry = True - while retry: - try: - fcntl.flock(self.fd, flag) - retry = False - except IOError, err: - if err.errno in (errno.EAGAIN, ): - if timeout_end is not None and time.time() < timeout_end: - # Wait before trying again - time.sleep(max(0.1, min(1.0, timeout))) - else: - raise errors.LockError(errmsg) - else: - logging.exception("fcntl.flock failed") - raise + logging.exception("fcntl.flock failed") + raise def Exclusive(self, blocking=False, timeout=None): """Locks the file in exclusive mode.