root / lib / cache.py @ 95a74ef3
History | View | Annotate | Download (5.4 kB)
1 | 13699e58 | René Nussbaumer | #
|
---|---|---|---|
2 | 13699e58 | René Nussbaumer | #
|
3 | 13699e58 | René Nussbaumer | |
4 | 13699e58 | René Nussbaumer | # Copyright (C) 2011 Google Inc.
|
5 | 13699e58 | René Nussbaumer | #
|
6 | 13699e58 | René Nussbaumer | # This program is free software; you can redistribute it and/or modify
|
7 | 13699e58 | René Nussbaumer | # it under the terms of the GNU General Public License as published by
|
8 | 13699e58 | René Nussbaumer | # the Free Software Foundation; either version 2 of the License, or
|
9 | 13699e58 | René Nussbaumer | # (at your option) any later version.
|
10 | 13699e58 | René Nussbaumer | #
|
11 | 13699e58 | René Nussbaumer | # This program is distributed in the hope that it will be useful, but
|
12 | 13699e58 | René Nussbaumer | # WITHOUT ANY WARRANTY; without even the implied warranty of
|
13 | 13699e58 | René Nussbaumer | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14 | 13699e58 | René Nussbaumer | # General Public License for more details.
|
15 | 13699e58 | René Nussbaumer | #
|
16 | 13699e58 | René Nussbaumer | # You should have received a copy of the GNU General Public License
|
17 | 13699e58 | René Nussbaumer | # along with this program; if not, write to the Free Software
|
18 | 13699e58 | René Nussbaumer | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
19 | 13699e58 | René Nussbaumer | # 02110-1301, USA.
|
20 | 13699e58 | René Nussbaumer | |
21 | 13699e58 | René Nussbaumer | |
22 | 13699e58 | René Nussbaumer | """This module implements caching."""
|
23 | 13699e58 | René Nussbaumer | |
24 | 13699e58 | René Nussbaumer | |
25 | 13699e58 | René Nussbaumer | import time |
26 | 13699e58 | René Nussbaumer | |
27 | 13699e58 | René Nussbaumer | from ganeti import locking |
28 | 13699e58 | René Nussbaumer | from ganeti import serializer |
29 | 13699e58 | René Nussbaumer | |
30 | 13699e58 | René Nussbaumer | |
31 | 13699e58 | René Nussbaumer | TIMESTAMP = "timestamp"
|
32 | 13699e58 | René Nussbaumer | TTL = "ttl"
|
33 | 13699e58 | René Nussbaumer | VALUE = "value"
|
34 | 13699e58 | René Nussbaumer | |
35 | 13699e58 | René Nussbaumer | |
36 | 13699e58 | René Nussbaumer | class CacheBase: |
37 | 13699e58 | René Nussbaumer | """This is the base class for all caches.
|
38 | 13699e58 | René Nussbaumer |
|
39 | 13699e58 | René Nussbaumer | """
|
40 | 13699e58 | René Nussbaumer | def __init__(self): |
41 | 13699e58 | René Nussbaumer | """Base init method.
|
42 | 13699e58 | René Nussbaumer |
|
43 | 13699e58 | René Nussbaumer | """
|
44 | 13699e58 | René Nussbaumer | |
45 | 13699e58 | René Nussbaumer | def Store(self, key, value, ttl=0): |
46 | 13699e58 | René Nussbaumer | """Stores key with value in the cache.
|
47 | 13699e58 | René Nussbaumer |
|
48 | 13699e58 | René Nussbaumer | @param key: The key to associate this cached value
|
49 | 13699e58 | René Nussbaumer | @param value: The value to cache
|
50 | 13699e58 | René Nussbaumer | @param ttl: TTL in seconds after when this entry is considered outdated
|
51 | 13699e58 | René Nussbaumer | @returns: L{True} on success, L{False} on failure
|
52 | 13699e58 | René Nussbaumer |
|
53 | 13699e58 | René Nussbaumer | """
|
54 | 13699e58 | René Nussbaumer | raise NotImplementedError |
55 | 13699e58 | René Nussbaumer | |
56 | 13699e58 | René Nussbaumer | def GetMulti(self, keys): |
57 | 13699e58 | René Nussbaumer | """Retrieve multiple values from the cache.
|
58 | 13699e58 | René Nussbaumer |
|
59 | 13699e58 | René Nussbaumer | @param keys: The keys to retrieve
|
60 | 13699e58 | René Nussbaumer | @returns: The list of values
|
61 | 13699e58 | René Nussbaumer |
|
62 | 13699e58 | René Nussbaumer | """
|
63 | 13699e58 | René Nussbaumer | raise NotImplementedError |
64 | 13699e58 | René Nussbaumer | |
65 | 13699e58 | René Nussbaumer | def Get(self, key): |
66 | 13699e58 | René Nussbaumer | """Retrieve the value from the cache.
|
67 | 13699e58 | René Nussbaumer |
|
68 | 13699e58 | René Nussbaumer | @param key: The key to retrieve
|
69 | 13699e58 | René Nussbaumer | @returns: The value or L{None} if not found
|
70 | 13699e58 | René Nussbaumer |
|
71 | 13699e58 | René Nussbaumer | """
|
72 | 13699e58 | René Nussbaumer | raise NotImplementedError |
73 | 13699e58 | René Nussbaumer | |
74 | 13699e58 | René Nussbaumer | def Invalidate(self, keys): |
75 | 13699e58 | René Nussbaumer | """Invalidate given keys.
|
76 | 13699e58 | René Nussbaumer |
|
77 | 13699e58 | René Nussbaumer | @param keys: The list of keys to invalidate
|
78 | 13699e58 | René Nussbaumer | @returns: L{True} on success, L{False} otherwise
|
79 | 13699e58 | René Nussbaumer |
|
80 | 13699e58 | René Nussbaumer | """
|
81 | 13699e58 | René Nussbaumer | raise NotImplementedError |
82 | 13699e58 | René Nussbaumer | |
83 | 13699e58 | René Nussbaumer | def Flush(self): |
84 | 13699e58 | René Nussbaumer | """Invalidates all of the keys and flushes the cache.
|
85 | 13699e58 | René Nussbaumer |
|
86 | 13699e58 | René Nussbaumer | """
|
87 | 13699e58 | René Nussbaumer | raise NotImplementedError |
88 | 13699e58 | René Nussbaumer | |
89 | 13699e58 | René Nussbaumer | def ResetState(self): |
90 | 13699e58 | René Nussbaumer | """Used to reset the state of the cache.
|
91 | 13699e58 | René Nussbaumer |
|
92 | 13699e58 | René Nussbaumer | This can be used to reinstantiate connection or any other state refresh
|
93 | 13699e58 | René Nussbaumer |
|
94 | 13699e58 | René Nussbaumer | """
|
95 | 13699e58 | René Nussbaumer | |
96 | 13699e58 | René Nussbaumer | def Cleanup(self): |
97 | 13699e58 | René Nussbaumer | """Cleanup the cache from expired entries.
|
98 | 13699e58 | René Nussbaumer |
|
99 | 13699e58 | René Nussbaumer | """
|
100 | 13699e58 | René Nussbaumer | |
101 | 13699e58 | René Nussbaumer | |
102 | 13699e58 | René Nussbaumer | class SimpleCache(CacheBase): |
103 | 13699e58 | René Nussbaumer | """Implements a very simple, dict base cache.
|
104 | 13699e58 | René Nussbaumer |
|
105 | 13699e58 | René Nussbaumer | """
|
106 | 13699e58 | René Nussbaumer | CLEANUP_ROUND = 1800
|
107 | 13699e58 | René Nussbaumer | _LOCK = "lock"
|
108 | 13699e58 | René Nussbaumer | |
109 | 13699e58 | René Nussbaumer | def __init__(self, _time_fn=time.time): |
110 | 13699e58 | René Nussbaumer | """Initialize this class.
|
111 | 13699e58 | René Nussbaumer |
|
112 | 13699e58 | René Nussbaumer | @param _time_fn: Function used to return time (unittest only)
|
113 | 13699e58 | René Nussbaumer |
|
114 | 13699e58 | René Nussbaumer | """
|
115 | 13699e58 | René Nussbaumer | CacheBase.__init__(self)
|
116 | 13699e58 | René Nussbaumer | |
117 | 13699e58 | René Nussbaumer | self._time_fn = _time_fn
|
118 | 13699e58 | René Nussbaumer | |
119 | 13699e58 | René Nussbaumer | self.cache = {}
|
120 | 13699e58 | René Nussbaumer | self.lock = locking.SharedLock("SimpleCache") |
121 | 13699e58 | René Nussbaumer | self.last_cleanup = self._time_fn() |
122 | 13699e58 | René Nussbaumer | |
123 | 13699e58 | René Nussbaumer | def _UnlockedCleanup(self): |
124 | 13699e58 | René Nussbaumer | """Does cleanup of the cache.
|
125 | 13699e58 | René Nussbaumer |
|
126 | 13699e58 | René Nussbaumer | """
|
127 | 13699e58 | René Nussbaumer | check_time = self._time_fn()
|
128 | 13699e58 | René Nussbaumer | if (self.last_cleanup + self.CLEANUP_ROUND) <= check_time: |
129 | 13699e58 | René Nussbaumer | keys = [] |
130 | 13699e58 | René Nussbaumer | for key, value in self.cache.items(): |
131 | 13699e58 | René Nussbaumer | if not value[TTL]: |
132 | 13699e58 | René Nussbaumer | continue
|
133 | 13699e58 | René Nussbaumer | |
134 | 13699e58 | René Nussbaumer | expired = value[TIMESTAMP] + value[TTL] |
135 | 13699e58 | René Nussbaumer | if expired < check_time:
|
136 | 13699e58 | René Nussbaumer | keys.append(key) |
137 | 13699e58 | René Nussbaumer | self._UnlockedInvalidate(keys)
|
138 | 13699e58 | René Nussbaumer | self.last_cleanup = check_time
|
139 | 13699e58 | René Nussbaumer | |
140 | 13699e58 | René Nussbaumer | @locking.ssynchronized(_LOCK)
|
141 | 13699e58 | René Nussbaumer | def Cleanup(self): |
142 | 13699e58 | René Nussbaumer | """Cleanup our cache.
|
143 | 13699e58 | René Nussbaumer |
|
144 | 13699e58 | René Nussbaumer | """
|
145 | 13699e58 | René Nussbaumer | self._UnlockedCleanup()
|
146 | 13699e58 | René Nussbaumer | |
147 | 13699e58 | René Nussbaumer | @locking.ssynchronized(_LOCK)
|
148 | 13699e58 | René Nussbaumer | def Store(self, key, value, ttl=0): |
149 | 13699e58 | René Nussbaumer | """Stores a value at key in the cache.
|
150 | 13699e58 | René Nussbaumer |
|
151 | 13699e58 | René Nussbaumer | See L{CacheBase.Store} for parameter description
|
152 | 13699e58 | René Nussbaumer |
|
153 | 13699e58 | René Nussbaumer | """
|
154 | 13699e58 | René Nussbaumer | assert ttl >= 0 |
155 | 13699e58 | René Nussbaumer | self._UnlockedCleanup()
|
156 | 13699e58 | René Nussbaumer | val = serializer.Dump(value) |
157 | 13699e58 | René Nussbaumer | cache_val = { |
158 | 13699e58 | René Nussbaumer | TIMESTAMP: self._time_fn(),
|
159 | 13699e58 | René Nussbaumer | TTL: ttl, |
160 | 13699e58 | René Nussbaumer | VALUE: val |
161 | 13699e58 | René Nussbaumer | } |
162 | 13699e58 | René Nussbaumer | self.cache[key] = cache_val
|
163 | 13699e58 | René Nussbaumer | return True |
164 | 13699e58 | René Nussbaumer | |
165 | 13699e58 | René Nussbaumer | @locking.ssynchronized(_LOCK, shared=1) |
166 | 13699e58 | René Nussbaumer | def GetMulti(self, keys): |
167 | 13699e58 | René Nussbaumer | """Retrieve the values of keys from cache.
|
168 | 13699e58 | René Nussbaumer |
|
169 | 13699e58 | René Nussbaumer | See L{CacheBase.GetMulti} for parameter description
|
170 | 13699e58 | René Nussbaumer |
|
171 | 13699e58 | René Nussbaumer | """
|
172 | 13699e58 | René Nussbaumer | return [self._ExtractValue(key) for key in keys] |
173 | 13699e58 | René Nussbaumer | |
174 | 13699e58 | René Nussbaumer | @locking.ssynchronized(_LOCK, shared=1) |
175 | 13699e58 | René Nussbaumer | def Get(self, key): |
176 | 13699e58 | René Nussbaumer | """Retrieve the value of key from cache.
|
177 | 13699e58 | René Nussbaumer |
|
178 | 13699e58 | René Nussbaumer | See L{CacheBase.Get} for parameter description
|
179 | 13699e58 | René Nussbaumer |
|
180 | 13699e58 | René Nussbaumer | """
|
181 | 13699e58 | René Nussbaumer | return self._ExtractValue(key) |
182 | 13699e58 | René Nussbaumer | |
183 | 13699e58 | René Nussbaumer | @locking.ssynchronized(_LOCK)
|
184 | 13699e58 | René Nussbaumer | def Invalidate(self, keys): |
185 | 13699e58 | René Nussbaumer | """Invalidates value for keys in cache.
|
186 | 13699e58 | René Nussbaumer |
|
187 | 13699e58 | René Nussbaumer | See L{CacheBase.Invalidate} for parameter description
|
188 | 13699e58 | René Nussbaumer |
|
189 | 13699e58 | René Nussbaumer | """
|
190 | 13699e58 | René Nussbaumer | return self._UnlockedInvalidate(keys) |
191 | 13699e58 | René Nussbaumer | |
192 | 13699e58 | René Nussbaumer | @locking.ssynchronized(_LOCK)
|
193 | 13699e58 | René Nussbaumer | def Flush(self): |
194 | 13699e58 | René Nussbaumer | """Invalidates all keys and values in cache.
|
195 | 13699e58 | René Nussbaumer |
|
196 | 13699e58 | René Nussbaumer | See L{CacheBase.Flush} for parameter description
|
197 | 13699e58 | René Nussbaumer |
|
198 | 13699e58 | René Nussbaumer | """
|
199 | 13699e58 | René Nussbaumer | self.cache.clear()
|
200 | 13699e58 | René Nussbaumer | self.last_cleanup = self._time_fn() |
201 | 13699e58 | René Nussbaumer | |
202 | 13699e58 | René Nussbaumer | def _UnlockedInvalidate(self, keys): |
203 | 13699e58 | René Nussbaumer | """Invalidate keys in cache.
|
204 | 13699e58 | René Nussbaumer |
|
205 | 13699e58 | René Nussbaumer | This is the unlocked version, see L{Invalidate} for parameter description
|
206 | 13699e58 | René Nussbaumer |
|
207 | 13699e58 | René Nussbaumer | """
|
208 | 13699e58 | René Nussbaumer | for key in keys: |
209 | 13699e58 | René Nussbaumer | self.cache.pop(key, None) |
210 | 13699e58 | René Nussbaumer | |
211 | 13699e58 | René Nussbaumer | return True |
212 | 13699e58 | René Nussbaumer | |
213 | 13699e58 | René Nussbaumer | def _ExtractValue(self, key): |
214 | 13699e58 | René Nussbaumer | """Extracts just the value for a key.
|
215 | 13699e58 | René Nussbaumer |
|
216 | 13699e58 | René Nussbaumer | This method is taking care if the value did not expire ans returns it
|
217 | 13699e58 | René Nussbaumer |
|
218 | 13699e58 | René Nussbaumer | @param key: The key to look for
|
219 | 13699e58 | René Nussbaumer | @returns: The value if key is not expired, L{None} otherwise
|
220 | 13699e58 | René Nussbaumer |
|
221 | 13699e58 | René Nussbaumer | """
|
222 | 13699e58 | René Nussbaumer | try:
|
223 | 13699e58 | René Nussbaumer | cache_val = self.cache[key]
|
224 | 13699e58 | René Nussbaumer | except KeyError: |
225 | 13699e58 | René Nussbaumer | return None |
226 | 13699e58 | René Nussbaumer | else:
|
227 | 13699e58 | René Nussbaumer | if cache_val[TTL] == 0: |
228 | 13699e58 | René Nussbaumer | return serializer.Load(cache_val[VALUE])
|
229 | 13699e58 | René Nussbaumer | else:
|
230 | 13699e58 | René Nussbaumer | expired = cache_val[TIMESTAMP] + cache_val[TTL] |
231 | 13699e58 | René Nussbaumer | |
232 | 13699e58 | René Nussbaumer | if self._time_fn() <= expired: |
233 | 13699e58 | René Nussbaumer | return serializer.Load(cache_val[VALUE])
|
234 | 13699e58 | René Nussbaumer | else:
|
235 | 13699e58 | René Nussbaumer | return None |