Statistics
| Branch: | Tag: | Revision:

root / lib / utils / retry.py @ adec726e

History | View | Annotate | Download (6.2 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21
"""Utility functions for retrying function calls with a timeout.
22

23
"""
24

    
25

    
26
import time
27

    
28
from ganeti import errors
29

    
30

    
31
#: Special delay to specify whole remaining timeout
32
RETRY_REMAINING_TIME = object()
33

    
34

    
35
class RetryTimeout(Exception):
36
  """Retry loop timed out.
37

38
  Any arguments which was passed by the retried function to RetryAgain will be
39
  preserved in RetryTimeout, if it is raised. If such argument was an exception
40
  the RaiseInner helper method will reraise it.
41

42
  """
43
  def RaiseInner(self):
44
    if self.args and isinstance(self.args[0], Exception):
45
      raise self.args[0]
46
    else:
47
      raise RetryTimeout(*self.args)
48

    
49

    
50
class RetryAgain(Exception):
51
  """Retry again.
52

53
  Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as
54
  arguments to RetryTimeout. If an exception is passed, the RaiseInner() method
55
  of the RetryTimeout() method can be used to reraise it.
56

57
  """
58

    
59

    
60
class _RetryDelayCalculator(object):
61
  """Calculator for increasing delays.
62

63
  """
64
  __slots__ = [
65
    "_factor",
66
    "_limit",
67
    "_next",
68
    "_start",
69
    ]
70

    
71
  def __init__(self, start, factor, limit):
72
    """Initializes this class.
73

74
    @type start: float
75
    @param start: Initial delay
76
    @type factor: float
77
    @param factor: Factor for delay increase
78
    @type limit: float or None
79
    @param limit: Upper limit for delay or None for no limit
80

81
    """
82
    assert start > 0.0
83
    assert factor >= 1.0
84
    assert limit is None or limit >= 0.0
85

    
86
    self._start = start
87
    self._factor = factor
88
    self._limit = limit
89

    
90
    self._next = start
91

    
92
  def __call__(self):
93
    """Returns current delay and calculates the next one.
94

95
    """
96
    current = self._next
97

    
98
    # Update for next run
99
    if self._limit is None or self._next < self._limit:
100
      self._next = min(self._limit, self._next * self._factor)
101

    
102
    return current
103

    
104

    
105
def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
106
          _time_fn=time.time):
107
  """Call a function repeatedly until it succeeds.
108

109
  The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
110
  anymore. Between calls a delay, specified by C{delay}, is inserted. After a
111
  total of C{timeout} seconds, this function throws L{RetryTimeout}.
112

113
  C{delay} can be one of the following:
114
    - callable returning the delay length as a float
115
    - Tuple of (start, factor, limit)
116
    - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
117
      useful when overriding L{wait_fn} to wait for an external event)
118
    - A static delay as a number (int or float)
119

120
  @type fn: callable
121
  @param fn: Function to be called
122
  @param delay: Either a callable (returning the delay), a tuple of (start,
123
                factor, limit) (see L{_RetryDelayCalculator}),
124
                L{RETRY_REMAINING_TIME} or a number (int or float)
125
  @type timeout: float
126
  @param timeout: Total timeout
127
  @type wait_fn: callable
128
  @param wait_fn: Waiting function
129
  @return: Return value of function
130

131
  """
132
  assert callable(fn)
133
  assert callable(wait_fn)
134
  assert callable(_time_fn)
135

    
136
  if args is None:
137
    args = []
138

    
139
  end_time = _time_fn() + timeout
140

    
141
  if callable(delay):
142
    # External function to calculate delay
143
    calc_delay = delay
144

    
145
  elif isinstance(delay, (tuple, list)):
146
    # Increasing delay with optional upper boundary
147
    (start, factor, limit) = delay
148
    calc_delay = _RetryDelayCalculator(start, factor, limit)
149

    
150
  elif delay is RETRY_REMAINING_TIME:
151
    # Always use the remaining time
152
    calc_delay = None
153

    
154
  else:
155
    # Static delay
156
    calc_delay = lambda: delay
157

    
158
  assert calc_delay is None or callable(calc_delay)
159

    
160
  while True:
161
    retry_args = []
162
    try:
163
      # pylint: disable=W0142
164
      return fn(*args)
165
    except RetryAgain, err:
166
      retry_args = err.args
167
    except RetryTimeout:
168
      raise errors.ProgrammerError("Nested retry loop detected that didn't"
169
                                   " handle RetryTimeout")
170

    
171
    remaining_time = end_time - _time_fn()
172

    
173
    if remaining_time <= 0.0:
174
      # pylint: disable=W0142
175
      raise RetryTimeout(*retry_args)
176

    
177
    assert remaining_time > 0.0
178

    
179
    if calc_delay is None:
180
      wait_fn(remaining_time)
181
    else:
182
      current_delay = calc_delay()
183
      if current_delay > 0.0:
184
        wait_fn(current_delay)
185

    
186

    
187
def SimpleRetry(expected, fn, delay, timeout, args=None, wait_fn=time.sleep,
188
                _time_fn=time.time):
189
  """A wrapper over L{Retry} implementing a simpler interface.
190

191
  All the parameters are the same as for L{Retry}, except it has one
192
  extra argument: expected, which can be either a value (will be
193
  compared with the result of the function, or a callable (which will
194
  get the result passed and has to return a boolean). If the test is
195
  false, we will retry until either the timeout has passed or the
196
  tests succeeds. In both cases, the last result from calling the
197
  function will be returned.
198

199
  Note that this function is not expected to raise any retry-related
200
  exceptions, always simply returning values. As such, the function is
201
  designed to allow easy wrapping of code that doesn't use retry at
202
  all (e.g. "if fn(args)" replaced with "if SimpleRetry(True, fn,
203
  ...)".
204

205
  @see: L{Retry}
206

207
  """
208
  rdict = {}
209

    
210
  def helper(*innerargs):
211
    # pylint: disable=W0142
212
    result = rdict["result"] = fn(*innerargs)
213
    if not ((callable(expected) and expected(result)) or result == expected):
214
      raise RetryAgain()
215
    return result
216

    
217
  try:
218
    result = Retry(helper, delay, timeout, args=args,
219
                   wait_fn=wait_fn, _time_fn=_time_fn)
220
  except RetryTimeout:
221
    assert "result" in rdict
222
    result = rdict["result"]
223
  return result