root / lib / utils / retry.py @ 18397489
History | View | Annotate | Download (6.2 kB)
1 | 79d22269 | Michael Hanselmann | #
|
---|---|---|---|
2 | 79d22269 | Michael Hanselmann | #
|
3 | 79d22269 | Michael Hanselmann | |
4 | 79d22269 | Michael Hanselmann | # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
|
5 | 79d22269 | Michael Hanselmann | #
|
6 | 79d22269 | Michael Hanselmann | # This program is free software; you can redistribute it and/or modify
|
7 | 79d22269 | Michael Hanselmann | # it under the terms of the GNU General Public License as published by
|
8 | 79d22269 | Michael Hanselmann | # the Free Software Foundation; either version 2 of the License, or
|
9 | 79d22269 | Michael Hanselmann | # (at your option) any later version.
|
10 | 79d22269 | Michael Hanselmann | #
|
11 | 79d22269 | Michael Hanselmann | # This program is distributed in the hope that it will be useful, but
|
12 | 79d22269 | Michael Hanselmann | # WITHOUT ANY WARRANTY; without even the implied warranty of
|
13 | 79d22269 | Michael Hanselmann | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14 | 79d22269 | Michael Hanselmann | # General Public License for more details.
|
15 | 79d22269 | Michael Hanselmann | #
|
16 | 79d22269 | Michael Hanselmann | # You should have received a copy of the GNU General Public License
|
17 | 79d22269 | Michael Hanselmann | # along with this program; if not, write to the Free Software
|
18 | 79d22269 | Michael Hanselmann | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
19 | 79d22269 | Michael Hanselmann | # 02110-1301, USA.
|
20 | 79d22269 | Michael Hanselmann | |
21 | 79d22269 | Michael Hanselmann | """Utility functions for retrying function calls with a timeout.
|
22 | 79d22269 | Michael Hanselmann |
|
23 | 79d22269 | Michael Hanselmann | """
|
24 | 79d22269 | Michael Hanselmann | |
25 | 79d22269 | Michael Hanselmann | |
26 | 79d22269 | Michael Hanselmann | import time |
27 | 79d22269 | Michael Hanselmann | |
28 | 79d22269 | Michael Hanselmann | from ganeti import errors |
29 | 79d22269 | Michael Hanselmann | |
30 | 79d22269 | Michael Hanselmann | |
31 | 79d22269 | Michael Hanselmann | #: Special delay to specify whole remaining timeout
|
32 | 79d22269 | Michael Hanselmann | RETRY_REMAINING_TIME = object()
|
33 | 79d22269 | Michael Hanselmann | |
34 | 79d22269 | Michael Hanselmann | |
35 | 79d22269 | Michael Hanselmann | class RetryTimeout(Exception): |
36 | 79d22269 | Michael Hanselmann | """Retry loop timed out.
|
37 | 79d22269 | Michael Hanselmann |
|
38 | 79d22269 | Michael Hanselmann | Any arguments which was passed by the retried function to RetryAgain will be
|
39 | 79d22269 | Michael Hanselmann | preserved in RetryTimeout, if it is raised. If such argument was an exception
|
40 | 79d22269 | Michael Hanselmann | the RaiseInner helper method will reraise it.
|
41 | 79d22269 | Michael Hanselmann |
|
42 | 79d22269 | Michael Hanselmann | """
|
43 | 79d22269 | Michael Hanselmann | def RaiseInner(self): |
44 | 79d22269 | Michael Hanselmann | if self.args and isinstance(self.args[0], Exception): |
45 | 79d22269 | Michael Hanselmann | raise self.args[0] |
46 | 79d22269 | Michael Hanselmann | else:
|
47 | 79d22269 | Michael Hanselmann | raise RetryTimeout(*self.args) |
48 | 79d22269 | Michael Hanselmann | |
49 | 79d22269 | Michael Hanselmann | |
50 | 79d22269 | Michael Hanselmann | class RetryAgain(Exception): |
51 | 79d22269 | Michael Hanselmann | """Retry again.
|
52 | 79d22269 | Michael Hanselmann |
|
53 | 79d22269 | Michael Hanselmann | Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as
|
54 | 79d22269 | Michael Hanselmann | arguments to RetryTimeout. If an exception is passed, the RaiseInner() method
|
55 | 79d22269 | Michael Hanselmann | of the RetryTimeout() method can be used to reraise it.
|
56 | 79d22269 | Michael Hanselmann |
|
57 | 79d22269 | Michael Hanselmann | """
|
58 | 79d22269 | Michael Hanselmann | |
59 | 79d22269 | Michael Hanselmann | |
60 | 79d22269 | Michael Hanselmann | class _RetryDelayCalculator(object): |
61 | 79d22269 | Michael Hanselmann | """Calculator for increasing delays.
|
62 | 79d22269 | Michael Hanselmann |
|
63 | 79d22269 | Michael Hanselmann | """
|
64 | 79d22269 | Michael Hanselmann | __slots__ = [ |
65 | 79d22269 | Michael Hanselmann | "_factor",
|
66 | 79d22269 | Michael Hanselmann | "_limit",
|
67 | 79d22269 | Michael Hanselmann | "_next",
|
68 | 79d22269 | Michael Hanselmann | "_start",
|
69 | 79d22269 | Michael Hanselmann | ] |
70 | 79d22269 | Michael Hanselmann | |
71 | 79d22269 | Michael Hanselmann | def __init__(self, start, factor, limit): |
72 | 79d22269 | Michael Hanselmann | """Initializes this class.
|
73 | 79d22269 | Michael Hanselmann |
|
74 | 79d22269 | Michael Hanselmann | @type start: float
|
75 | 79d22269 | Michael Hanselmann | @param start: Initial delay
|
76 | 79d22269 | Michael Hanselmann | @type factor: float
|
77 | 79d22269 | Michael Hanselmann | @param factor: Factor for delay increase
|
78 | 79d22269 | Michael Hanselmann | @type limit: float or None
|
79 | 79d22269 | Michael Hanselmann | @param limit: Upper limit for delay or None for no limit
|
80 | 79d22269 | Michael Hanselmann |
|
81 | 79d22269 | Michael Hanselmann | """
|
82 | 79d22269 | Michael Hanselmann | assert start > 0.0 |
83 | 79d22269 | Michael Hanselmann | assert factor >= 1.0 |
84 | 79d22269 | Michael Hanselmann | assert limit is None or limit >= 0.0 |
85 | 79d22269 | Michael Hanselmann | |
86 | 79d22269 | Michael Hanselmann | self._start = start
|
87 | 79d22269 | Michael Hanselmann | self._factor = factor
|
88 | 79d22269 | Michael Hanselmann | self._limit = limit
|
89 | 79d22269 | Michael Hanselmann | |
90 | 79d22269 | Michael Hanselmann | self._next = start
|
91 | 79d22269 | Michael Hanselmann | |
92 | 79d22269 | Michael Hanselmann | def __call__(self): |
93 | 79d22269 | Michael Hanselmann | """Returns current delay and calculates the next one.
|
94 | 79d22269 | Michael Hanselmann |
|
95 | 79d22269 | Michael Hanselmann | """
|
96 | 79d22269 | Michael Hanselmann | current = self._next
|
97 | 79d22269 | Michael Hanselmann | |
98 | 79d22269 | Michael Hanselmann | # Update for next run
|
99 | 79d22269 | Michael Hanselmann | if self._limit is None or self._next < self._limit: |
100 | 79d22269 | Michael Hanselmann | self._next = min(self._limit, self._next * self._factor) |
101 | 79d22269 | Michael Hanselmann | |
102 | 79d22269 | Michael Hanselmann | return current
|
103 | 79d22269 | Michael Hanselmann | |
104 | 79d22269 | Michael Hanselmann | |
105 | 79d22269 | Michael Hanselmann | def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep, |
106 | 79d22269 | Michael Hanselmann | _time_fn=time.time): |
107 | 79d22269 | Michael Hanselmann | """Call a function repeatedly until it succeeds.
|
108 | 79d22269 | Michael Hanselmann |
|
109 | 79d22269 | Michael Hanselmann | The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
|
110 | 79d22269 | Michael Hanselmann | anymore. Between calls a delay, specified by C{delay}, is inserted. After a
|
111 | 79d22269 | Michael Hanselmann | total of C{timeout} seconds, this function throws L{RetryTimeout}.
|
112 | 79d22269 | Michael Hanselmann |
|
113 | 79d22269 | Michael Hanselmann | C{delay} can be one of the following:
|
114 | 79d22269 | Michael Hanselmann | - callable returning the delay length as a float
|
115 | 79d22269 | Michael Hanselmann | - Tuple of (start, factor, limit)
|
116 | 79d22269 | Michael Hanselmann | - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
|
117 | 79d22269 | Michael Hanselmann | useful when overriding L{wait_fn} to wait for an external event)
|
118 | 79d22269 | Michael Hanselmann | - A static delay as a number (int or float)
|
119 | 79d22269 | Michael Hanselmann |
|
120 | 79d22269 | Michael Hanselmann | @type fn: callable
|
121 | 79d22269 | Michael Hanselmann | @param fn: Function to be called
|
122 | 79d22269 | Michael Hanselmann | @param delay: Either a callable (returning the delay), a tuple of (start,
|
123 | 79d22269 | Michael Hanselmann | factor, limit) (see L{_RetryDelayCalculator}),
|
124 | 79d22269 | Michael Hanselmann | L{RETRY_REMAINING_TIME} or a number (int or float)
|
125 | 79d22269 | Michael Hanselmann | @type timeout: float
|
126 | 79d22269 | Michael Hanselmann | @param timeout: Total timeout
|
127 | 79d22269 | Michael Hanselmann | @type wait_fn: callable
|
128 | 79d22269 | Michael Hanselmann | @param wait_fn: Waiting function
|
129 | 79d22269 | Michael Hanselmann | @return: Return value of function
|
130 | 79d22269 | Michael Hanselmann |
|
131 | 79d22269 | Michael Hanselmann | """
|
132 | 79d22269 | Michael Hanselmann | assert callable(fn) |
133 | 79d22269 | Michael Hanselmann | assert callable(wait_fn) |
134 | 79d22269 | Michael Hanselmann | assert callable(_time_fn) |
135 | 79d22269 | Michael Hanselmann | |
136 | 79d22269 | Michael Hanselmann | if args is None: |
137 | 79d22269 | Michael Hanselmann | args = [] |
138 | 79d22269 | Michael Hanselmann | |
139 | 79d22269 | Michael Hanselmann | end_time = _time_fn() + timeout |
140 | 79d22269 | Michael Hanselmann | |
141 | 79d22269 | Michael Hanselmann | if callable(delay): |
142 | 79d22269 | Michael Hanselmann | # External function to calculate delay
|
143 | 79d22269 | Michael Hanselmann | calc_delay = delay |
144 | 79d22269 | Michael Hanselmann | |
145 | 79d22269 | Michael Hanselmann | elif isinstance(delay, (tuple, list)): |
146 | 79d22269 | Michael Hanselmann | # Increasing delay with optional upper boundary
|
147 | 79d22269 | Michael Hanselmann | (start, factor, limit) = delay |
148 | 79d22269 | Michael Hanselmann | calc_delay = _RetryDelayCalculator(start, factor, limit) |
149 | 79d22269 | Michael Hanselmann | |
150 | 79d22269 | Michael Hanselmann | elif delay is RETRY_REMAINING_TIME: |
151 | 79d22269 | Michael Hanselmann | # Always use the remaining time
|
152 | 79d22269 | Michael Hanselmann | calc_delay = None
|
153 | 79d22269 | Michael Hanselmann | |
154 | 79d22269 | Michael Hanselmann | else:
|
155 | 79d22269 | Michael Hanselmann | # Static delay
|
156 | 79d22269 | Michael Hanselmann | calc_delay = lambda: delay
|
157 | 79d22269 | Michael Hanselmann | |
158 | 79d22269 | Michael Hanselmann | assert calc_delay is None or callable(calc_delay) |
159 | 79d22269 | Michael Hanselmann | |
160 | 79d22269 | Michael Hanselmann | while True: |
161 | 79d22269 | Michael Hanselmann | retry_args = [] |
162 | 79d22269 | Michael Hanselmann | try:
|
163 | b459a848 | Andrea Spadaccini | # pylint: disable=W0142
|
164 | 79d22269 | Michael Hanselmann | return fn(*args)
|
165 | 79d22269 | Michael Hanselmann | except RetryAgain, err:
|
166 | 79d22269 | Michael Hanselmann | retry_args = err.args |
167 | 79d22269 | Michael Hanselmann | except RetryTimeout:
|
168 | 79d22269 | Michael Hanselmann | raise errors.ProgrammerError("Nested retry loop detected that didn't" |
169 | 79d22269 | Michael Hanselmann | " handle RetryTimeout")
|
170 | 79d22269 | Michael Hanselmann | |
171 | 79d22269 | Michael Hanselmann | remaining_time = end_time - _time_fn() |
172 | 79d22269 | Michael Hanselmann | |
173 | 79d22269 | Michael Hanselmann | if remaining_time < 0.0: |
174 | b459a848 | Andrea Spadaccini | # pylint: disable=W0142
|
175 | 79d22269 | Michael Hanselmann | raise RetryTimeout(*retry_args)
|
176 | 79d22269 | Michael Hanselmann | |
177 | 79d22269 | Michael Hanselmann | assert remaining_time >= 0.0 |
178 | 79d22269 | Michael Hanselmann | |
179 | 79d22269 | Michael Hanselmann | if calc_delay is None: |
180 | 79d22269 | Michael Hanselmann | wait_fn(remaining_time) |
181 | 79d22269 | Michael Hanselmann | else:
|
182 | 79d22269 | Michael Hanselmann | current_delay = calc_delay() |
183 | 79d22269 | Michael Hanselmann | if current_delay > 0.0: |
184 | 79d22269 | Michael Hanselmann | wait_fn(current_delay) |
185 | c7d3a832 | Iustin Pop | |
186 | c7d3a832 | Iustin Pop | |
187 | c7d3a832 | Iustin Pop | def SimpleRetry(expected, fn, delay, timeout, args=None, wait_fn=time.sleep, |
188 | c7d3a832 | Iustin Pop | _time_fn=time.time): |
189 | c7d3a832 | Iustin Pop | """A wrapper over L{Retry} implementing a simpler interface.
|
190 | c7d3a832 | Iustin Pop |
|
191 | c7d3a832 | Iustin Pop | All the parameters are the same as for L{Retry}, except it has one
|
192 | c7d3a832 | Iustin Pop | extra argument: expected, which can be either a value (will be
|
193 | c7d3a832 | Iustin Pop | compared with the result of the function, or a callable (which will
|
194 | c7d3a832 | Iustin Pop | get the result passed and has to return a boolean). If the test is
|
195 | c7d3a832 | Iustin Pop | false, we will retry until either the timeout has passed or the
|
196 | c7d3a832 | Iustin Pop | tests succeeds. In both cases, the last result from calling the
|
197 | c7d3a832 | Iustin Pop | function will be returned.
|
198 | c7d3a832 | Iustin Pop |
|
199 | c7d3a832 | Iustin Pop | Note that this function is not expected to raise any retry-related
|
200 | c7d3a832 | Iustin Pop | exceptions, always simply returning values. As such, the function is
|
201 | c7d3a832 | Iustin Pop | designed to allow easy wrapping of code that doesn't use retry at
|
202 | c7d3a832 | Iustin Pop | all (e.g. "if fn(args)" replaced with "if SimpleRetry(True, fn,
|
203 | c7d3a832 | Iustin Pop | ...)".
|
204 | c7d3a832 | Iustin Pop |
|
205 | c7d3a832 | Iustin Pop | @see: L{Retry}
|
206 | c7d3a832 | Iustin Pop |
|
207 | c7d3a832 | Iustin Pop | """
|
208 | c7d3a832 | Iustin Pop | rdict = {} |
209 | e687ec01 | Michael Hanselmann | |
210 | c7d3a832 | Iustin Pop | def helper(*innerargs): |
211 | b459a848 | Andrea Spadaccini | # pylint: disable=W0142
|
212 | c7d3a832 | Iustin Pop | result = rdict["result"] = fn(*innerargs)
|
213 | c7d3a832 | Iustin Pop | if not ((callable(expected) and expected(result)) or result == expected): |
214 | c7d3a832 | Iustin Pop | raise RetryAgain()
|
215 | c7d3a832 | Iustin Pop | return result
|
216 | c7d3a832 | Iustin Pop | |
217 | c7d3a832 | Iustin Pop | try:
|
218 | c7d3a832 | Iustin Pop | result = Retry(helper, delay, timeout, args=args, |
219 | c7d3a832 | Iustin Pop | wait_fn=wait_fn, _time_fn=_time_fn) |
220 | c7d3a832 | Iustin Pop | except RetryTimeout:
|
221 | c7d3a832 | Iustin Pop | assert "result" in rdict |
222 | c7d3a832 | Iustin Pop | result = rdict["result"]
|
223 | c7d3a832 | Iustin Pop | return result |