Synnefo » snf-ganeti » ganeti-local

	lib/utils/algo.py \

	lib/utils/retry.py

	test/ganeti.utils.retry_unittest.py \

from ganeti.utils.retry import * # pylint: disable-msg=W0401

#

#

# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful, but

# WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

# General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

# 02110-1301, USA.

"""Utility functions for retrying function calls with a timeout.

"""

import time

from ganeti import errors

#: Special delay to specify whole remaining timeout

RETRY_REMAINING_TIME = object()

class RetryTimeout(Exception):

  """Retry loop timed out.

  Any arguments which was passed by the retried function to RetryAgain will be

  preserved in RetryTimeout, if it is raised. If such argument was an exception

  the RaiseInner helper method will reraise it.

  """

  def RaiseInner(self):

    if self.args and isinstance(self.args[0], Exception):

      raise self.args[0]

    else:

      raise RetryTimeout(*self.args)

class RetryAgain(Exception):

  """Retry again.

  Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as

  arguments to RetryTimeout. If an exception is passed, the RaiseInner() method

  of the RetryTimeout() method can be used to reraise it.

  """

class _RetryDelayCalculator(object):

  """Calculator for increasing delays.

  """

  __slots__ = [

    "_factor",

    "_limit",

    "_next",

    "_start",

    ]

  def __init__(self, start, factor, limit):

    """Initializes this class.

    @type start: float

    @param start: Initial delay

    @type factor: float

    @param factor: Factor for delay increase

    @type limit: float or None

    @param limit: Upper limit for delay or None for no limit

    """

    assert start > 0.0

    assert factor >= 1.0

    assert limit is None or limit >= 0.0

    self._start = start

    self._factor = factor

    self._limit = limit

    self._next = start

  def __call__(self):

    """Returns current delay and calculates the next one.

    """

    current = self._next

    # Update for next run

    if self._limit is None or self._next < self._limit:

      self._next = min(self._limit, self._next * self._factor)

    return current

def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,

          _time_fn=time.time):

  """Call a function repeatedly until it succeeds.

  The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}

  anymore. Between calls a delay, specified by C{delay}, is inserted. After a

  total of C{timeout} seconds, this function throws L{RetryTimeout}.

  C{delay} can be one of the following:

    - callable returning the delay length as a float

    - Tuple of (start, factor, limit)

    - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is

      useful when overriding L{wait_fn} to wait for an external event)

    - A static delay as a number (int or float)

  @type fn: callable

  @param fn: Function to be called

  @param delay: Either a callable (returning the delay), a tuple of (start,

                factor, limit) (see L{_RetryDelayCalculator}),

                L{RETRY_REMAINING_TIME} or a number (int or float)

  @type timeout: float

  @param timeout: Total timeout

  @type wait_fn: callable

  @param wait_fn: Waiting function

  @return: Return value of function

  """

  assert callable(fn)

  assert callable(wait_fn)

  assert callable(_time_fn)

  if args is None:

    args = []

  end_time = _time_fn() + timeout

  if callable(delay):

    # External function to calculate delay

    calc_delay = delay

  elif isinstance(delay, (tuple, list)):

    # Increasing delay with optional upper boundary

    (start, factor, limit) = delay

    calc_delay = _RetryDelayCalculator(start, factor, limit)

  elif delay is RETRY_REMAINING_TIME:

    # Always use the remaining time

    calc_delay = None

  else:

    # Static delay

    calc_delay = lambda: delay

  assert calc_delay is None or callable(calc_delay)

  while True:

    retry_args = []

    try:

      # pylint: disable-msg=W0142

      return fn(*args)

    except RetryAgain, err:

      retry_args = err.args

    except RetryTimeout:

      raise errors.ProgrammerError("Nested retry loop detected that didn't"

                                   " handle RetryTimeout")

    remaining_time = end_time - _time_fn()

    if remaining_time < 0.0:

      # pylint: disable-msg=W0142

      raise RetryTimeout(*retry_args)

    assert remaining_time >= 0.0

    if calc_delay is None:

      wait_fn(remaining_time)

    else:

      current_delay = calc_delay()

      if current_delay > 0.0:

        wait_fn(current_delay)

#!/usr/bin/python

#

# Copyright (C) 2011 Google Inc.

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful, but

# WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

# General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

# 02110-1301, USA.

"""Script for testing ganeti.utils.retry"""

import unittest

from ganeti import constants

from ganeti import errors

from ganeti import utils

import testutils

class TestRetry(testutils.GanetiTestCase):

  def setUp(self):

    testutils.GanetiTestCase.setUp(self)

    self.retries = 0

  @staticmethod

  def _RaiseRetryAgain():

    raise utils.RetryAgain()

  @staticmethod

  def _RaiseRetryAgainWithArg(args):

    raise utils.RetryAgain(*args)

  def _WrongNestedLoop(self):

    return utils.Retry(self._RaiseRetryAgain, 0.01, 0.02)

  def _RetryAndSucceed(self, retries):

    if self.retries < retries:

      self.retries += 1

      raise utils.RetryAgain()

    else:

      return True

  def testRaiseTimeout(self):

    self.failUnlessRaises(utils.RetryTimeout, utils.Retry,

                          self._RaiseRetryAgain, 0.01, 0.02)

    self.failUnlessRaises(utils.RetryTimeout, utils.Retry,

                          self._RetryAndSucceed, 0.01, 0, args=[1])

    self.failUnlessEqual(self.retries, 1)

  def testComplete(self):

    self.failUnlessEqual(utils.Retry(lambda: True, 0, 1), True)

    self.failUnlessEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2]),

                         True)

    self.failUnlessEqual(self.retries, 2)

  def testNestedLoop(self):

    try:

      self.failUnlessRaises(errors.ProgrammerError, utils.Retry,

                            self._WrongNestedLoop, 0, 1)

    except utils.RetryTimeout:

      self.fail("Didn't detect inner loop's exception")

  def testTimeoutArgument(self):

    retry_arg="my_important_debugging_message"

    try:

      utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[retry_arg]])

    except utils.RetryTimeout, err:

      self.failUnlessEqual(err.args, (retry_arg, ))

    else:

      self.fail("Expected timeout didn't happen")

  def testRaiseInnerWithExc(self):

    retry_arg="my_important_debugging_message"

    try:

      try:

        utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,

                    args=[[errors.GenericError(retry_arg, retry_arg)]])

      except utils.RetryTimeout, err:

        err.RaiseInner()

      else:

        self.fail("Expected timeout didn't happen")

    except errors.GenericError, err:

      self.failUnlessEqual(err.args, (retry_arg, retry_arg))

    else:

      self.fail("Expected GenericError didn't happen")

  def testRaiseInnerWithMsg(self):

    retry_arg="my_important_debugging_message"

    try:

      try:

        utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,

                    args=[[retry_arg, retry_arg]])

      except utils.RetryTimeout, err:

        err.RaiseInner()

      else:

        self.fail("Expected timeout didn't happen")

    except utils.RetryTimeout, err:

      self.failUnlessEqual(err.args, (retry_arg, retry_arg))

    else:

      self.fail("Expected RetryTimeout didn't happen")

if __name__ == "__main__":

  testutils.GanetiTestProgram()