root / snf-common / synnefo / lib / pool / tests.py @ 5f6ad491
History | View | Annotate | Download (9.1 kB)
1 | 45e32a00 | Vangelis Koukis | #!/usr/bin/env python
|
---|---|---|---|
2 | 45e32a00 | Vangelis Koukis | #
|
3 | 45e32a00 | Vangelis Koukis | # -*- coding: utf-8 -*-
|
4 | 45e32a00 | Vangelis Koukis | #
|
5 | 45e32a00 | Vangelis Koukis | # Copyright 2011 GRNET S.A. All rights reserved.
|
6 | 45e32a00 | Vangelis Koukis | #
|
7 | 45e32a00 | Vangelis Koukis | # Redistribution and use in source and binary forms, with or
|
8 | 45e32a00 | Vangelis Koukis | # without modification, are permitted provided that the following
|
9 | 45e32a00 | Vangelis Koukis | # conditions are met:
|
10 | 45e32a00 | Vangelis Koukis | #
|
11 | 45e32a00 | Vangelis Koukis | # 1. Redistributions of source code must retain the above
|
12 | 45e32a00 | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
13 | 45e32a00 | Vangelis Koukis | # disclaimer.
|
14 | 45e32a00 | Vangelis Koukis | #
|
15 | 45e32a00 | Vangelis Koukis | # 2. Redistributions in binary form must reproduce the above
|
16 | 45e32a00 | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
17 | 45e32a00 | Vangelis Koukis | # disclaimer in the documentation and/or other materials
|
18 | 45e32a00 | Vangelis Koukis | # provided with the distribution.
|
19 | 45e32a00 | Vangelis Koukis | #
|
20 | 45e32a00 | Vangelis Koukis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
21 | 45e32a00 | Vangelis Koukis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
22 | 45e32a00 | Vangelis Koukis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
23 | 45e32a00 | Vangelis Koukis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
24 | 45e32a00 | Vangelis Koukis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
25 | 45e32a00 | Vangelis Koukis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
26 | 45e32a00 | Vangelis Koukis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
27 | 45e32a00 | Vangelis Koukis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
28 | 45e32a00 | Vangelis Koukis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
29 | 45e32a00 | Vangelis Koukis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
30 | 45e32a00 | Vangelis Koukis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
31 | 45e32a00 | Vangelis Koukis | # POSSIBILITY OF SUCH DAMAGE.
|
32 | 45e32a00 | Vangelis Koukis | #
|
33 | 45e32a00 | Vangelis Koukis | # The views and conclusions contained in the software and
|
34 | 45e32a00 | Vangelis Koukis | # documentation are those of the authors and should not be
|
35 | 45e32a00 | Vangelis Koukis | # interpreted as representing official policies, either expressed
|
36 | 45e32a00 | Vangelis Koukis | # or implied, of GRNET S.A.
|
37 | 45e32a00 | Vangelis Koukis | #
|
38 | 45e32a00 | Vangelis Koukis | #
|
39 | 45e32a00 | Vangelis Koukis | |
40 | 45e32a00 | Vangelis Koukis | """Unit Tests for the pool classes in synnefo.lib.pool
|
41 | 45e32a00 | Vangelis Koukis |
|
42 | 45e32a00 | Vangelis Koukis | Provides unit tests for the code implementing pool
|
43 | 45e32a00 | Vangelis Koukis | classes in the synnefo.lib.pool module.
|
44 | 45e32a00 | Vangelis Koukis |
|
45 | 45e32a00 | Vangelis Koukis | """
|
46 | 45e32a00 | Vangelis Koukis | |
47 | de67123e | Vangelis Koukis | # Support running under a gevent-monkey-patched environment
|
48 | de67123e | Vangelis Koukis | # if the "monkey" argument is specified in the command line.
|
49 | de67123e | Vangelis Koukis | import sys |
50 | de67123e | Vangelis Koukis | if "monkey" in sys.argv: |
51 | de67123e | Vangelis Koukis | from gevent import monkey |
52 | de67123e | Vangelis Koukis | monkey.patch_all() |
53 | de67123e | Vangelis Koukis | sys.argv.pop(sys.argv.index("monkey"))
|
54 | 45e32a00 | Vangelis Koukis | |
55 | 45e32a00 | Vangelis Koukis | import sys |
56 | 45e32a00 | Vangelis Koukis | import time |
57 | 45e32a00 | Vangelis Koukis | import threading |
58 | a8935947 | Georgios D. Tsoukalas | from collections import defaultdict |
59 | 45e32a00 | Vangelis Koukis | |
60 | a8935947 | Georgios D. Tsoukalas | from synnefo.lib.pool import ObjectPool, PoolLimitError, PoolVerificationError |
61 | 45e32a00 | Vangelis Koukis | |
62 | 45e32a00 | Vangelis Koukis | # Use backported unittest functionality if Python < 2.7
|
63 | 45e32a00 | Vangelis Koukis | try:
|
64 | 45e32a00 | Vangelis Koukis | import unittest2 as unittest |
65 | 45e32a00 | Vangelis Koukis | except ImportError: |
66 | 45e32a00 | Vangelis Koukis | if sys.version_info < (2, 7): |
67 | 45e32a00 | Vangelis Koukis | raise Exception("The unittest2 package is required for Python < 2.7") |
68 | 45e32a00 | Vangelis Koukis | import unittest |
69 | 45e32a00 | Vangelis Koukis | |
70 | 45e32a00 | Vangelis Koukis | |
71 | a8935947 | Georgios D. Tsoukalas | from threading import Lock |
72 | a8935947 | Georgios D. Tsoukalas | |
73 | a8935947 | Georgios D. Tsoukalas | mutex = Lock() |
74 | a8935947 | Georgios D. Tsoukalas | |
75 | 45e32a00 | Vangelis Koukis | class NumbersPool(ObjectPool): |
76 | 45e32a00 | Vangelis Koukis | max = 0
|
77 | 45e32a00 | Vangelis Koukis | |
78 | a8935947 | Georgios D. Tsoukalas | def _pool_create_safe(self): |
79 | a8935947 | Georgios D. Tsoukalas | with mutex:
|
80 | a8935947 | Georgios D. Tsoukalas | n = self.max
|
81 | a8935947 | Georgios D. Tsoukalas | self.max += 1 |
82 | a8935947 | Georgios D. Tsoukalas | return n
|
83 | a8935947 | Georgios D. Tsoukalas | |
84 | a8935947 | Georgios D. Tsoukalas | def _pool_create_unsafe(self): |
85 | 45e32a00 | Vangelis Koukis | n = self.max
|
86 | 45e32a00 | Vangelis Koukis | self.max += 1 |
87 | 45e32a00 | Vangelis Koukis | return n
|
88 | 45e32a00 | Vangelis Koukis | |
89 | a8935947 | Georgios D. Tsoukalas | # set this to _pool_create_unsafe to check
|
90 | a8935947 | Georgios D. Tsoukalas | # the thread-safety test
|
91 | a8935947 | Georgios D. Tsoukalas | #_pool_create = _pool_create_unsafe
|
92 | a8935947 | Georgios D. Tsoukalas | _pool_create = _pool_create_safe |
93 | a8935947 | Georgios D. Tsoukalas | |
94 | a8935947 | Georgios D. Tsoukalas | def _pool_verify(self, obj): |
95 | a8935947 | Georgios D. Tsoukalas | return True |
96 | a8935947 | Georgios D. Tsoukalas | |
97 | 09cdd926 | Vangelis Koukis | def _pool_cleanup(self, obj): |
98 | 3447b13d | Vangelis Koukis | n = int(obj)
|
99 | 3447b13d | Vangelis Koukis | if n < 0: |
100 | 3447b13d | Vangelis Koukis | return True |
101 | a8935947 | Georgios D. Tsoukalas | return False |
102 | 45e32a00 | Vangelis Koukis | |
103 | 45e32a00 | Vangelis Koukis | |
104 | 45e32a00 | Vangelis Koukis | class ObjectPoolTestCase(unittest.TestCase): |
105 | 45e32a00 | Vangelis Koukis | def test_create_pool_requires_size(self): |
106 | 45e32a00 | Vangelis Koukis | """Test __init__() requires valid size argument"""
|
107 | 45e32a00 | Vangelis Koukis | self.assertRaises(ValueError, ObjectPool) |
108 | 3447b13d | Vangelis Koukis | self.assertRaises(ValueError, ObjectPool, size="size10") |
109 | 3447b13d | Vangelis Koukis | self.assertRaises(ValueError, ObjectPool, size=0) |
110 | 3447b13d | Vangelis Koukis | self.assertRaises(ValueError, ObjectPool, size=-1) |
111 | 45e32a00 | Vangelis Koukis | |
112 | 45e32a00 | Vangelis Koukis | def test_create_pool(self): |
113 | 45e32a00 | Vangelis Koukis | """Test pool creation works"""
|
114 | 45e32a00 | Vangelis Koukis | pool = ObjectPool(100)
|
115 | 45e32a00 | Vangelis Koukis | self.assertEqual(pool.size, 100) |
116 | 45e32a00 | Vangelis Koukis | |
117 | 45e32a00 | Vangelis Koukis | def test_get_not_implemented(self): |
118 | 09cdd926 | Vangelis Koukis | """Test pool_get() method not implemented in abstract class"""
|
119 | 45e32a00 | Vangelis Koukis | pool = ObjectPool(100)
|
120 | a8935947 | Georgios D. Tsoukalas | self.assertRaises(NotImplementedError, pool._pool_create) |
121 | a8935947 | Georgios D. Tsoukalas | self.assertRaises(NotImplementedError, pool._pool_verify, None) |
122 | 45e32a00 | Vangelis Koukis | |
123 | 45e32a00 | Vangelis Koukis | def test_put_not_implemented(self): |
124 | 09cdd926 | Vangelis Koukis | """Test pool_put() method not implemented in abstract class"""
|
125 | 45e32a00 | Vangelis Koukis | pool = ObjectPool(100)
|
126 | a8935947 | Georgios D. Tsoukalas | self.assertRaises(NotImplementedError, pool._pool_cleanup, None) |
127 | 45e32a00 | Vangelis Koukis | |
128 | 45e32a00 | Vangelis Koukis | |
129 | 45e32a00 | Vangelis Koukis | class NumbersPoolTestCase(unittest.TestCase): |
130 | de67123e | Vangelis Koukis | N = 1500
|
131 | 45e32a00 | Vangelis Koukis | SEC = 0.5
|
132 | 45e32a00 | Vangelis Koukis | maxDiff = None
|
133 | 45e32a00 | Vangelis Koukis | |
134 | 45e32a00 | Vangelis Koukis | def setUp(self): |
135 | 45e32a00 | Vangelis Koukis | self.numbers = NumbersPool(self.N) |
136 | 45e32a00 | Vangelis Koukis | |
137 | 45e32a00 | Vangelis Koukis | def test_initially_empty(self): |
138 | 45e32a00 | Vangelis Koukis | """Test pool is empty upon creation"""
|
139 | 45e32a00 | Vangelis Koukis | self.assertEqual(self.numbers._set, set([])) |
140 | 45e32a00 | Vangelis Koukis | |
141 | 45e32a00 | Vangelis Koukis | def test_seq_allocate_all(self): |
142 | 45e32a00 | Vangelis Koukis | """Test allocation and deallocation of all pool objects"""
|
143 | 45e32a00 | Vangelis Koukis | n = [] |
144 | 45e32a00 | Vangelis Koukis | for _ in xrange(0, self.N): |
145 | 09cdd926 | Vangelis Koukis | n.append(self.numbers.pool_get())
|
146 | 45e32a00 | Vangelis Koukis | self.assertEqual(n, range(0, self.N)) |
147 | 45e32a00 | Vangelis Koukis | for i in n: |
148 | 09cdd926 | Vangelis Koukis | self.numbers.pool_put(i)
|
149 | 45e32a00 | Vangelis Koukis | self.assertEqual(self.numbers._set, set(n)) |
150 | 45e32a00 | Vangelis Koukis | |
151 | 45e32a00 | Vangelis Koukis | def test_parallel_allocate_all(self): |
152 | 45e32a00 | Vangelis Koukis | """Allocate all pool objects in parallel"""
|
153 | 45e32a00 | Vangelis Koukis | def allocate_one(pool, results, index): |
154 | 09cdd926 | Vangelis Koukis | n = pool.pool_get() |
155 | 45e32a00 | Vangelis Koukis | results[index] = n |
156 | 45e32a00 | Vangelis Koukis | |
157 | 45e32a00 | Vangelis Koukis | results = [None] * self.N |
158 | 45e32a00 | Vangelis Koukis | threads = [threading.Thread(target=allocate_one, |
159 | a8935947 | Georgios D. Tsoukalas | args=(self.numbers, results, i))
|
160 | 45e32a00 | Vangelis Koukis | for i in xrange(0, self.N)] |
161 | 45e32a00 | Vangelis Koukis | |
162 | 45e32a00 | Vangelis Koukis | for t in threads: |
163 | 45e32a00 | Vangelis Koukis | t.start() |
164 | 45e32a00 | Vangelis Koukis | for t in threads: |
165 | 45e32a00 | Vangelis Koukis | t.join() |
166 | 45e32a00 | Vangelis Koukis | |
167 | 09cdd926 | Vangelis Koukis | # This nonblocking pool_get() should fail
|
168 | a8935947 | Georgios D. Tsoukalas | self.assertRaises(PoolLimitError, self.numbers.pool_get, |
169 | 3447b13d | Vangelis Koukis | blocking=False)
|
170 | 45e32a00 | Vangelis Koukis | self.assertEqual(sorted(results), range(0, self.N)) |
171 | 45e32a00 | Vangelis Koukis | |
172 | 3447b13d | Vangelis Koukis | def test_allocate_no_create(self): |
173 | 3447b13d | Vangelis Koukis | """Allocate objects from the pool without creating them"""
|
174 | 3447b13d | Vangelis Koukis | for i in xrange(0, self.N): |
175 | 3447b13d | Vangelis Koukis | self.assertIsNone(self.numbers.pool_get(create=False)) |
176 | 3447b13d | Vangelis Koukis | |
177 | 3447b13d | Vangelis Koukis | # This nonblocking pool_get() should fail
|
178 | a8935947 | Georgios D. Tsoukalas | self.assertRaises(PoolLimitError, self.numbers.pool_get, |
179 | 3447b13d | Vangelis Koukis | blocking=False)
|
180 | 3447b13d | Vangelis Koukis | |
181 | 3447b13d | Vangelis Koukis | def test_pool_cleanup_returns_failure(self): |
182 | 3447b13d | Vangelis Koukis | """Put a broken object, test a new one is retrieved eventually"""
|
183 | 3447b13d | Vangelis Koukis | n = [] |
184 | 3447b13d | Vangelis Koukis | for _ in xrange(0, self.N): |
185 | 3447b13d | Vangelis Koukis | n.append(self.numbers.pool_get())
|
186 | 3447b13d | Vangelis Koukis | self.assertEqual(n, range(0, self.N)) |
187 | 3447b13d | Vangelis Koukis | |
188 | 3447b13d | Vangelis Koukis | del n[-1:] |
189 | 3447b13d | Vangelis Koukis | self.numbers.pool_put(-1) # This is a broken object |
190 | 3447b13d | Vangelis Koukis | self.assertFalse(self.numbers._set) |
191 | 3447b13d | Vangelis Koukis | self.assertEqual(self.numbers.pool_get(), self.N) |
192 | 3447b13d | Vangelis Koukis | |
193 | 45e32a00 | Vangelis Koukis | def test_parallel_get_blocks(self): |
194 | 45e32a00 | Vangelis Koukis | """Test threads block if no object left in the pool"""
|
195 | 45e32a00 | Vangelis Koukis | def allocate_one_and_sleep(pool, sec, result, index): |
196 | 09cdd926 | Vangelis Koukis | n = pool.pool_get() |
197 | 45e32a00 | Vangelis Koukis | time.sleep(sec) |
198 | 45e32a00 | Vangelis Koukis | result[index] = n |
199 | 09cdd926 | Vangelis Koukis | pool.pool_put(n) |
200 | 45e32a00 | Vangelis Koukis | |
201 | a8935947 | Georgios D. Tsoukalas | nr_threads = 2 * self.N + 1 |
202 | a8935947 | Georgios D. Tsoukalas | results = [None] * nr_threads
|
203 | 45e32a00 | Vangelis Koukis | threads = [threading.Thread(target=allocate_one_and_sleep, |
204 | a8935947 | Georgios D. Tsoukalas | args=(self.numbers, self.SEC, results, i)) |
205 | a8935947 | Georgios D. Tsoukalas | for i in xrange(nr_threads)] |
206 | 45e32a00 | Vangelis Koukis | |
207 | 45e32a00 | Vangelis Koukis | # This should take 3 * SEC seconds
|
208 | 45e32a00 | Vangelis Koukis | start = time.time() |
209 | 45e32a00 | Vangelis Koukis | for t in threads: |
210 | 45e32a00 | Vangelis Koukis | t.start() |
211 | 45e32a00 | Vangelis Koukis | for t in threads: |
212 | 45e32a00 | Vangelis Koukis | t.join() |
213 | 45e32a00 | Vangelis Koukis | diff = time.time() - start |
214 | 45e32a00 | Vangelis Koukis | self.assertTrue(diff > 3 * self.SEC) |
215 | 45e32a00 | Vangelis Koukis | self.assertLess((diff - 3 * self.SEC) / 3 * self.SEC, .5) |
216 | 45e32a00 | Vangelis Koukis | |
217 | a8935947 | Georgios D. Tsoukalas | freq = defaultdict(int)
|
218 | 45e32a00 | Vangelis Koukis | for r in results: |
219 | a8935947 | Georgios D. Tsoukalas | freq[r] += 1
|
220 | a8935947 | Georgios D. Tsoukalas | |
221 | a8935947 | Georgios D. Tsoukalas | # The maximum number used must be exactly the pool size.
|
222 | a8935947 | Georgios D. Tsoukalas | self.assertEqual(max(results), self.N - 1) |
223 | a8935947 | Georgios D. Tsoukalas | # At least one number must have been used three times
|
224 | 45e32a00 | Vangelis Koukis | triples = [r for r in freq if freq[r] == 3] |
225 | a8935947 | Georgios D. Tsoukalas | self.assertGreater(len(triples), 0) |
226 | a8935947 | Georgios D. Tsoukalas | # The sum of all frequencies must equal to the number of threads.
|
227 | a8935947 | Georgios D. Tsoukalas | self.assertEqual(sum(freq.values()), nr_threads) |
228 | a8935947 | Georgios D. Tsoukalas | |
229 | a8935947 | Georgios D. Tsoukalas | def test_verify_create(self): |
230 | a8935947 | Georgios D. Tsoukalas | numbers = self.numbers
|
231 | a8935947 | Georgios D. Tsoukalas | nums = [numbers.pool_get() for _ in xrange(self.N)] |
232 | a8935947 | Georgios D. Tsoukalas | for num in nums: |
233 | a8935947 | Georgios D. Tsoukalas | numbers.pool_put(num) |
234 | a8935947 | Georgios D. Tsoukalas | |
235 | a8935947 | Georgios D. Tsoukalas | def verify(num): |
236 | a8935947 | Georgios D. Tsoukalas | if num in nums: |
237 | a8935947 | Georgios D. Tsoukalas | return False |
238 | a8935947 | Georgios D. Tsoukalas | return True |
239 | a8935947 | Georgios D. Tsoukalas | |
240 | a8935947 | Georgios D. Tsoukalas | self.numbers._pool_verify = verify
|
241 | a8935947 | Georgios D. Tsoukalas | self.assertEqual(numbers.pool_get(), self.N) |
242 | a8935947 | Georgios D. Tsoukalas | |
243 | a8935947 | Georgios D. Tsoukalas | def test_verify_error(self): |
244 | a8935947 | Georgios D. Tsoukalas | numbers = self.numbers
|
245 | a8935947 | Georgios D. Tsoukalas | nums = [numbers.pool_get() for _ in xrange(self.N)] |
246 | a8935947 | Georgios D. Tsoukalas | for num in nums: |
247 | a8935947 | Georgios D. Tsoukalas | numbers.pool_put(num) |
248 | a8935947 | Georgios D. Tsoukalas | |
249 | a8935947 | Georgios D. Tsoukalas | def false(*args): |
250 | a8935947 | Georgios D. Tsoukalas | return False |
251 | a8935947 | Georgios D. Tsoukalas | |
252 | a8935947 | Georgios D. Tsoukalas | self.numbers._pool_verify = false
|
253 | a8935947 | Georgios D. Tsoukalas | self.assertRaises(PoolVerificationError, numbers.pool_get)
|
254 | a8935947 | Georgios D. Tsoukalas | |
255 | a8935947 | Georgios D. Tsoukalas | class ThreadSafetyTest(unittest.TestCase): |
256 | a8935947 | Georgios D. Tsoukalas | |
257 | a8935947 | Georgios D. Tsoukalas | pool_class = NumbersPool |
258 | a8935947 | Georgios D. Tsoukalas | |
259 | a8935947 | Georgios D. Tsoukalas | def setUp(self): |
260 | a8935947 | Georgios D. Tsoukalas | size = 3000
|
261 | a8935947 | Georgios D. Tsoukalas | self.size = size
|
262 | a8935947 | Georgios D. Tsoukalas | self.pool = self.pool_class(size) |
263 | a8935947 | Georgios D. Tsoukalas | |
264 | a8935947 | Georgios D. Tsoukalas | def test_parallel_sleeping_create(self): |
265 | a8935947 | Georgios D. Tsoukalas | def create(pool, results, i): |
266 | a8935947 | Georgios D. Tsoukalas | time.sleep(1)
|
267 | a8935947 | Georgios D. Tsoukalas | results[i] = pool._pool_create() |
268 | a8935947 | Georgios D. Tsoukalas | |
269 | a8935947 | Georgios D. Tsoukalas | pool = self.pool
|
270 | a8935947 | Georgios D. Tsoukalas | N = self.size
|
271 | a8935947 | Georgios D. Tsoukalas | results = [None] * N
|
272 | a8935947 | Georgios D. Tsoukalas | threads = [threading.Thread(target=create, args=(pool, results, i)) |
273 | a8935947 | Georgios D. Tsoukalas | for i in xrange(N)] |
274 | a8935947 | Georgios D. Tsoukalas | for t in threads: |
275 | a8935947 | Georgios D. Tsoukalas | t.start() |
276 | a8935947 | Georgios D. Tsoukalas | for t in threads: |
277 | a8935947 | Georgios D. Tsoukalas | t.join() |
278 | a8935947 | Georgios D. Tsoukalas | |
279 | a8935947 | Georgios D. Tsoukalas | freq = defaultdict(int)
|
280 | a8935947 | Georgios D. Tsoukalas | for r in results: |
281 | a8935947 | Georgios D. Tsoukalas | freq[r] += 1
|
282 | a8935947 | Georgios D. Tsoukalas | |
283 | a8935947 | Georgios D. Tsoukalas | mults = [(n, c) for n, c in freq.items() if c > 1] |
284 | a8935947 | Georgios D. Tsoukalas | if mults:
|
285 | a8935947 | Georgios D. Tsoukalas | #print mults
|
286 | a8935947 | Georgios D. Tsoukalas | raise AssertionError("_pool_create() is not thread safe") |
287 | 45e32a00 | Vangelis Koukis | |
288 | 45e32a00 | Vangelis Koukis | |
289 | 45e32a00 | Vangelis Koukis | if __name__ == '__main__': |
290 | 45e32a00 | Vangelis Koukis | unittest.main() |