root / lib / utils / retry.py @ 1cc324f0
History | View | Annotate | Download (6.2 kB)
1 |
#
|
---|---|
2 |
#
|
3 |
|
4 |
# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
|
5 |
#
|
6 |
# This program is free software; you can redistribute it and/or modify
|
7 |
# it under the terms of the GNU General Public License as published by
|
8 |
# the Free Software Foundation; either version 2 of the License, or
|
9 |
# (at your option) any later version.
|
10 |
#
|
11 |
# This program is distributed in the hope that it will be useful, but
|
12 |
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
13 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14 |
# General Public License for more details.
|
15 |
#
|
16 |
# You should have received a copy of the GNU General Public License
|
17 |
# along with this program; if not, write to the Free Software
|
18 |
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
19 |
# 02110-1301, USA.
|
20 |
|
21 |
"""Utility functions for retrying function calls with a timeout.
|
22 |
|
23 |
"""
|
24 |
|
25 |
|
26 |
import time |
27 |
|
28 |
from ganeti import errors |
29 |
|
30 |
|
31 |
#: Special delay to specify whole remaining timeout
|
32 |
RETRY_REMAINING_TIME = object()
|
33 |
|
34 |
|
35 |
class RetryTimeout(Exception): |
36 |
"""Retry loop timed out.
|
37 |
|
38 |
Any arguments which was passed by the retried function to RetryAgain will be
|
39 |
preserved in RetryTimeout, if it is raised. If such argument was an exception
|
40 |
the RaiseInner helper method will reraise it.
|
41 |
|
42 |
"""
|
43 |
def RaiseInner(self): |
44 |
if self.args and isinstance(self.args[0], Exception): |
45 |
raise self.args[0] |
46 |
else:
|
47 |
raise RetryTimeout(*self.args) |
48 |
|
49 |
|
50 |
class RetryAgain(Exception): |
51 |
"""Retry again.
|
52 |
|
53 |
Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as
|
54 |
arguments to RetryTimeout. If an exception is passed, the RaiseInner() method
|
55 |
of the RetryTimeout() method can be used to reraise it.
|
56 |
|
57 |
"""
|
58 |
|
59 |
|
60 |
class _RetryDelayCalculator(object): |
61 |
"""Calculator for increasing delays.
|
62 |
|
63 |
"""
|
64 |
__slots__ = [ |
65 |
"_factor",
|
66 |
"_limit",
|
67 |
"_next",
|
68 |
"_start",
|
69 |
] |
70 |
|
71 |
def __init__(self, start, factor, limit): |
72 |
"""Initializes this class.
|
73 |
|
74 |
@type start: float
|
75 |
@param start: Initial delay
|
76 |
@type factor: float
|
77 |
@param factor: Factor for delay increase
|
78 |
@type limit: float or None
|
79 |
@param limit: Upper limit for delay or None for no limit
|
80 |
|
81 |
"""
|
82 |
assert start > 0.0 |
83 |
assert factor >= 1.0 |
84 |
assert limit is None or limit >= 0.0 |
85 |
|
86 |
self._start = start
|
87 |
self._factor = factor
|
88 |
self._limit = limit
|
89 |
|
90 |
self._next = start
|
91 |
|
92 |
def __call__(self): |
93 |
"""Returns current delay and calculates the next one.
|
94 |
|
95 |
"""
|
96 |
current = self._next
|
97 |
|
98 |
# Update for next run
|
99 |
if self._limit is None or self._next < self._limit: |
100 |
self._next = min(self._limit, self._next * self._factor) |
101 |
|
102 |
return current
|
103 |
|
104 |
|
105 |
def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep, |
106 |
_time_fn=time.time): |
107 |
"""Call a function repeatedly until it succeeds.
|
108 |
|
109 |
The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
|
110 |
anymore. Between calls a delay, specified by C{delay}, is inserted. After a
|
111 |
total of C{timeout} seconds, this function throws L{RetryTimeout}.
|
112 |
|
113 |
C{delay} can be one of the following:
|
114 |
- callable returning the delay length as a float
|
115 |
- Tuple of (start, factor, limit)
|
116 |
- L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
|
117 |
useful when overriding L{wait_fn} to wait for an external event)
|
118 |
- A static delay as a number (int or float)
|
119 |
|
120 |
@type fn: callable
|
121 |
@param fn: Function to be called
|
122 |
@param delay: Either a callable (returning the delay), a tuple of (start,
|
123 |
factor, limit) (see L{_RetryDelayCalculator}),
|
124 |
L{RETRY_REMAINING_TIME} or a number (int or float)
|
125 |
@type timeout: float
|
126 |
@param timeout: Total timeout
|
127 |
@type wait_fn: callable
|
128 |
@param wait_fn: Waiting function
|
129 |
@return: Return value of function
|
130 |
|
131 |
"""
|
132 |
assert callable(fn) |
133 |
assert callable(wait_fn) |
134 |
assert callable(_time_fn) |
135 |
|
136 |
if args is None: |
137 |
args = [] |
138 |
|
139 |
end_time = _time_fn() + timeout |
140 |
|
141 |
if callable(delay): |
142 |
# External function to calculate delay
|
143 |
calc_delay = delay |
144 |
|
145 |
elif isinstance(delay, (tuple, list)): |
146 |
# Increasing delay with optional upper boundary
|
147 |
(start, factor, limit) = delay |
148 |
calc_delay = _RetryDelayCalculator(start, factor, limit) |
149 |
|
150 |
elif delay is RETRY_REMAINING_TIME: |
151 |
# Always use the remaining time
|
152 |
calc_delay = None
|
153 |
|
154 |
else:
|
155 |
# Static delay
|
156 |
calc_delay = lambda: delay
|
157 |
|
158 |
assert calc_delay is None or callable(calc_delay) |
159 |
|
160 |
while True: |
161 |
retry_args = [] |
162 |
try:
|
163 |
# pylint: disable=W0142
|
164 |
return fn(*args)
|
165 |
except RetryAgain, err:
|
166 |
retry_args = err.args |
167 |
except RetryTimeout:
|
168 |
raise errors.ProgrammerError("Nested retry loop detected that didn't" |
169 |
" handle RetryTimeout")
|
170 |
|
171 |
remaining_time = end_time - _time_fn() |
172 |
|
173 |
if remaining_time < 0.0: |
174 |
# pylint: disable=W0142
|
175 |
raise RetryTimeout(*retry_args)
|
176 |
|
177 |
assert remaining_time >= 0.0 |
178 |
|
179 |
if calc_delay is None: |
180 |
wait_fn(remaining_time) |
181 |
else:
|
182 |
current_delay = calc_delay() |
183 |
if current_delay > 0.0: |
184 |
wait_fn(current_delay) |
185 |
|
186 |
|
187 |
def SimpleRetry(expected, fn, delay, timeout, args=None, wait_fn=time.sleep, |
188 |
_time_fn=time.time): |
189 |
"""A wrapper over L{Retry} implementing a simpler interface.
|
190 |
|
191 |
All the parameters are the same as for L{Retry}, except it has one
|
192 |
extra argument: expected, which can be either a value (will be
|
193 |
compared with the result of the function, or a callable (which will
|
194 |
get the result passed and has to return a boolean). If the test is
|
195 |
false, we will retry until either the timeout has passed or the
|
196 |
tests succeeds. In both cases, the last result from calling the
|
197 |
function will be returned.
|
198 |
|
199 |
Note that this function is not expected to raise any retry-related
|
200 |
exceptions, always simply returning values. As such, the function is
|
201 |
designed to allow easy wrapping of code that doesn't use retry at
|
202 |
all (e.g. "if fn(args)" replaced with "if SimpleRetry(True, fn,
|
203 |
...)".
|
204 |
|
205 |
@see: L{Retry}
|
206 |
|
207 |
"""
|
208 |
rdict = {} |
209 |
|
210 |
def helper(*innerargs): |
211 |
# pylint: disable=W0142
|
212 |
result = rdict["result"] = fn(*innerargs)
|
213 |
if not ((callable(expected) and expected(result)) or result == expected): |
214 |
raise RetryAgain()
|
215 |
return result
|
216 |
|
217 |
try:
|
218 |
result = Retry(helper, delay, timeout, args=args, |
219 |
wait_fn=wait_fn, _time_fn=_time_fn) |
220 |
except RetryTimeout:
|
221 |
assert "result" in rdict |
222 |
result = rdict["result"]
|
223 |
return result
|