Revision e02728f9
b/kamaki/cli/__init__.py | ||
---|---|---|
34 | 34 |
|
35 | 35 |
from __future__ import print_function |
36 | 36 |
|
37 |
import gevent.monkey |
|
38 |
#Monkey-patch everything for gevent early on |
|
39 |
gevent.monkey.patch_all() |
|
40 |
|
|
41 | 37 |
import logging |
42 | 38 |
|
43 | 39 |
from inspect import getargspec |
b/kamaki/clients/connection/kamakicon.py | ||
---|---|---|
36 | 36 |
from synnefo.lib.pool.http import get_http_connection |
37 | 37 |
from kamaki.clients.connection import HTTPConnection, HTTPResponse,\ |
38 | 38 |
HTTPConnectionError |
39 |
from gevent.dns import DNSError |
|
40 | 39 |
from socket import gaierror |
41 | 40 |
|
42 | 41 |
from json import loads |
... | ... | |
141 | 140 |
body=data) |
142 | 141 |
except Exception as err: |
143 | 142 |
conn.close() |
144 |
if isinstance(err, DNSError) or isinstance(err, gaierror):
|
|
143 |
if isinstance(err, gaierror): |
|
145 | 144 |
raise HTTPConnectionError('Cannot connect to %s' % self.url, |
146 | 145 |
status=701, |
147 | 146 |
details='%s: %s' % (type(err), unicode(err))) |
b/kamaki/clients/connection/tests.py | ||
---|---|---|
31 | 31 |
# interpreted as representing official policies, either expressed |
32 | 32 |
# or implied, of GRNET S.A. |
33 | 33 |
|
34 |
import gevent |
|
35 |
import gevent.monkey |
|
36 |
# Monkey-patch everything for gevent early on |
|
37 |
gevent.monkey.patch_all() |
|
38 |
import gevent.pool |
|
39 |
|
|
40 | 34 |
import unittest |
41 | 35 |
import sys |
42 | 36 |
from StringIO import StringIO |
... | ... | |
68 | 62 |
def tearDown(self): |
69 | 63 |
pass |
70 | 64 |
|
65 |
""" |
|
71 | 66 |
def _get_async_content(self, con, **kwargs): |
72 | 67 |
class SilentGreenlet(gevent.Greenlet): |
73 | 68 |
def _report_error(self, exc_info): |
... | ... | |
84 | 79 |
g = SilentGreenlet(self._get_content_len, con, **kwargs) |
85 | 80 |
self.async_pool.start(g) |
86 | 81 |
return g |
82 |
""" |
|
87 | 83 |
|
88 | 84 |
def _get_content_len(self, con, **kwargs): |
89 | 85 |
r = con.perform_request('GET', **kwargs) |
... | ... | |
113 | 109 |
self.assertNotEqual(r2, r4) |
114 | 110 |
#print('1:%s 2:%s 3:%s 4:%s 5:%s'%(r1, r2, r3, r4, r5)) |
115 | 111 |
|
116 |
gevent.joinall([h1, h2, h3, h4, h5]) |
|
117 |
|
|
118 | 112 |
if __name__ == '__main__': |
119 | 113 |
suiteFew = unittest.TestSuite() |
120 | 114 |
suiteFew.addTest(unittest.makeSuite(testKamakiCon)) |
b/kamaki/clients/pithos.py | ||
---|---|---|
31 | 31 |
# interpreted as representing official policies, either expressed |
32 | 32 |
# or implied, of GRNET S.A. |
33 | 33 |
|
34 |
import gevent |
|
35 |
#import gevent.monkey |
|
36 |
# Monkey-patch everything for gevent early on |
|
37 |
#gevent.monkey.patch_all() |
|
38 |
import gevent.pool |
|
34 |
from threading import Thread |
|
39 | 35 |
|
40 | 36 |
from os import fstat |
41 | 37 |
from hashlib import new as newhashlib |
42 |
from time import time, sleep |
|
43 |
import sys |
|
38 |
from time import time |
|
44 | 39 |
|
45 | 40 |
from binascii import hexlify |
46 | 41 |
|
... | ... | |
69 | 64 |
return (start, end) |
70 | 65 |
|
71 | 66 |
|
67 |
class SilentEvent(Thread): |
|
68 |
""" Thread-run method(*args, **kwargs) |
|
69 |
put exception in exception_bucket |
|
70 |
""" |
|
71 |
def __init__(self, method, *args, **kwargs): |
|
72 |
super(self.__class__, self).__init__() |
|
73 |
self.method = method |
|
74 |
self.args = args |
|
75 |
self.kwargs = kwargs |
|
76 |
|
|
77 |
@property |
|
78 |
def exception(self): |
|
79 |
return getattr(self, '_exception', False) |
|
80 |
|
|
81 |
@property |
|
82 |
def value(self): |
|
83 |
return getattr(self, '_value', None) |
|
84 |
|
|
85 |
def run(self): |
|
86 |
try: |
|
87 |
self._value = self.method(*(self.args), **(self.kwargs)) |
|
88 |
except Exception as e: |
|
89 |
print('______\n%s\n_______' % e) |
|
90 |
self._exception = e |
|
91 |
|
|
92 |
|
|
72 | 93 |
class PithosClient(PithosRestAPI): |
73 | 94 |
"""GRNet Pithos API client""" |
74 | 95 |
|
96 |
_thread_exceptions = [] |
|
97 |
|
|
75 | 98 |
def __init__(self, base_url, token, account=None, container=None): |
76 | 99 |
super(PithosClient, self).__init__(base_url, token, account, container) |
77 | 100 |
self.async_pool = None |
... | ... | |
117 | 140 |
|
118 | 141 |
# upload_* auxiliary methods |
119 | 142 |
def put_block_async(self, data, hash): |
120 |
class SilentGreenlet(gevent.Greenlet): |
|
121 |
def _report_error(self, exc_info): |
|
122 |
try: |
|
123 |
sys.stderr = StringIO() |
|
124 |
gevent.Greenlet._report_error(self, exc_info) |
|
125 |
finally: |
|
126 |
if hasattr(sys, '_stderr'): |
|
127 |
sys.stderr = sys._stderr |
|
128 |
POOL_SIZE = self.POOL_SIZE if hasattr(self, 'POOL_SIZE') else 5 |
|
129 |
if self.async_pool is None: |
|
130 |
self.async_pool = gevent.pool.Pool(size=POOL_SIZE) |
|
131 |
g = SilentGreenlet(self.put_block, data, hash) |
|
132 |
self.async_pool.start(g) |
|
133 |
return g |
|
143 |
event = SilentEvent(method=self.put_block, data=data, hash=hash) |
|
144 |
event.start() |
|
145 |
return event |
|
134 | 146 |
|
135 | 147 |
def put_block(self, data, hash): |
136 | 148 |
r = self.container_post(update=True, |
... | ... | |
233 | 245 |
data = fileobj.read(bytes) |
234 | 246 |
r = self.put_block_async(data, hash) |
235 | 247 |
flying.append(r) |
236 |
for r in flying: |
|
237 |
if r.ready(): |
|
238 |
if r.exception: |
|
239 |
raise r.exception |
|
248 |
unfinished = [] |
|
249 |
for thread in flying: |
|
250 |
if thread.isAlive() or thread.exception: |
|
251 |
unfinished.append(thread) |
|
252 |
else: |
|
240 | 253 |
if upload_cb: |
241 | 254 |
upload_gen.next() |
242 |
flying = [r for r in flying if not r.ready()]
|
|
255 |
flying = unfinished
|
|
243 | 256 |
while upload_cb: |
244 | 257 |
try: |
245 | 258 |
upload_gen.next() |
246 | 259 |
except StopIteration: |
247 | 260 |
break |
248 |
gevent.joinall(flying) |
|
261 |
|
|
262 |
for thread in flying: |
|
263 |
thread.join() |
|
249 | 264 |
|
250 | 265 |
failures = [r for r in flying if r.exception] |
251 | 266 |
if len(failures): |
252 |
details = ', '.join(['(%s).%s' % (i, r.exception)\ |
|
267 |
details = ', '.join([' (%s).%s' % (i, r.exception)\
|
|
253 | 268 |
for i, r in enumerate(failures)]) |
254 | 269 |
raise ClientError(message="Block uploading failed", |
255 | 270 |
status=505, |
... | ... | |
340 | 355 |
dst.flush() |
341 | 356 |
|
342 | 357 |
def _get_block_async(self, obj, **restargs): |
343 |
class SilentGreenlet(gevent.Greenlet): |
|
344 |
def _report_error(self, exc_info): |
|
345 |
try: |
|
346 |
sys.stderr = StringIO() |
|
347 |
gevent.Greenlet._report_error(self, exc_info) |
|
348 |
finally: |
|
349 |
if hasattr(sys, '_stderr'): |
|
350 |
sys.stderr = sys._stderr |
|
351 |
if not hasattr(self, 'POOL_SIZE'): |
|
352 |
self.POOL_SIZE = 5 |
|
353 |
if self.async_pool is None: |
|
354 |
self.async_pool = gevent.pool.Pool(size=self.POOL_SIZE) |
|
355 |
g = SilentGreenlet(self.object_get, obj, |
|
358 |
event = SilentEvent(self.object_get, |
|
359 |
obj, |
|
356 | 360 |
success=(200, 206), |
357 | 361 |
**restargs) |
358 |
self.async_pool.start(g)
|
|
359 |
return g
|
|
362 |
event.start()
|
|
363 |
return event
|
|
360 | 364 |
|
361 | 365 |
def _hash_from_file(self, fp, start, size, blockhash): |
362 | 366 |
fp.seek(start) |
... | ... | |
365 | 369 |
h.update(block.strip('\x00')) |
366 | 370 |
return hexlify(h.digest()) |
367 | 371 |
|
368 |
def _greenlet2file(self,
|
|
369 |
flying_greenlets,
|
|
372 |
def _thread2file(self,
|
|
373 |
flying, |
|
370 | 374 |
local_file, |
371 | 375 |
offset=0, |
372 | 376 |
**restargs): |
... | ... | |
375 | 379 |
- e.g. if the range is 10-100, all |
376 | 380 |
blocks will be written to normal_position - 10""" |
377 | 381 |
finished = [] |
378 |
for start, g in flying_greenlets.items():
|
|
379 |
if g.ready():
|
|
382 |
for start, g in flying.items(): |
|
383 |
if not g.isAlive():
|
|
380 | 384 |
if g.exception: |
381 | 385 |
raise g.exception |
382 | 386 |
block = g.value.content |
383 | 387 |
local_file.seek(start - offset) |
384 | 388 |
local_file.write(block) |
385 | 389 |
self._cb_next() |
386 |
finished.append(flying_greenlets.pop(start))
|
|
390 |
finished.append(flying.pop(start)) |
|
387 | 391 |
local_file.flush() |
388 | 392 |
return finished |
389 | 393 |
|
... | ... | |
399 | 403 |
**restargs): |
400 | 404 |
|
401 | 405 |
file_size = fstat(local_file.fileno()).st_size if resume else 0 |
402 |
flying_greenlets = {}
|
|
403 |
finished_greenlets = []
|
|
406 |
flying = {} |
|
407 |
finished = [] |
|
404 | 408 |
offset = 0 |
405 | 409 |
if filerange is not None: |
406 | 410 |
rstart = int(filerange.split('-')[0]) |
... | ... | |
414 | 418 |
blockhash): |
415 | 419 |
self._cb_next() |
416 | 420 |
continue |
417 |
if len(flying_greenlets) >= self.POOL_SIZE:
|
|
418 |
finished_greenlets += self._greenlet2file(flying_greenlets,
|
|
421 |
if len(flying) >= self.POOL_SIZE: |
|
422 |
finished += self._thread2file(flying,
|
|
419 | 423 |
local_file, |
420 | 424 |
offset, |
421 | 425 |
**restargs) |
... | ... | |
426 | 430 |
self._cb_next() |
427 | 431 |
continue |
428 | 432 |
restargs['async_headers'] = {'Range': 'bytes=%s-%s' % (start, end)} |
429 |
flying_greenlets[start] = self._get_block_async(obj, **restargs) |
|
430 |
|
|
431 |
#check the greenlets |
|
432 |
while len(flying_greenlets) > 0: |
|
433 |
sleep(0.001) |
|
434 |
finished_greenlets += self._greenlet2file(flying_greenlets, |
|
435 |
local_file, |
|
436 |
offset, |
|
437 |
**restargs) |
|
433 |
flying[start] = self._get_block_async(obj, **restargs) |
|
438 | 434 |
|
439 |
gevent.joinall(finished_greenlets) |
|
435 |
for thread in flying.values(): |
|
436 |
thread.join() |
|
437 |
finished += self._thread2file(flying, local_file, offset, **restargs) |
|
440 | 438 |
|
441 | 439 |
def download_object(self, |
442 | 440 |
obj, |
b/kamaki/clients/tests.py | ||
---|---|---|
31 | 31 |
# interpreted as representing official policies, either expressed |
32 | 32 |
# or implied, of GRNET S.A. |
33 | 33 |
|
34 |
import gevent.monkey |
|
35 |
# Monkey-patch everything for gevent early on |
|
36 |
gevent.monkey.patch_all() |
|
37 |
|
|
38 | 34 |
from argparse import ArgumentParser |
39 | 35 |
import unittest |
40 | 36 |
import time |
b/setup.py | ||
---|---|---|
34 | 34 |
# or implied, of GRNET S.A. |
35 | 35 |
|
36 | 36 |
from setuptools import setup |
37 |
#from sys import version_info
|
|
37 |
from sys import version_info |
|
38 | 38 |
|
39 | 39 |
import kamaki |
40 | 40 |
|
41 |
#Suggested packages can be installed manually later, but it is not nessecary |
|
42 |
suggested = ['ansicolors==1.0.2', 'progress==1.0.1']
|
|
43 |
required = ['gevent>=0.13.6', 'snf-common>=0.10', 'argparse']
|
|
41 |
|
|
42 |
optional = ['ansicolors', 'progress']
|
|
43 |
required = ['snf-common>=0.10', 'argparse'] |
|
44 | 44 |
|
45 | 45 |
setup( |
46 | 46 |
name='kamaki', |
47 | 47 |
version=kamaki.__version__, |
48 |
description='A command-line tool for poking clouds',
|
|
48 |
description='A command-line tool for managing clouds',
|
|
49 | 49 |
long_description=open('README.rst').read(), |
50 | 50 |
url='http://code.grnet.gr/projects/kamaki', |
51 | 51 |
license='BSD', |
52 |
packages=['kamaki', |
|
53 |
'kamaki.cli', |
|
54 |
'kamaki.clients', |
|
55 |
'kamaki.clients.connection', |
|
56 |
'kamaki.cli.commands'], |
|
52 |
packages=['kamaki', 'kamaki.clients', 'kamaki.clients.connection', 'kamaki.cli', 'kamaki.cli.commands'], |
|
57 | 53 |
include_package_data=True, |
58 | 54 |
entry_points={ |
59 | 55 |
'console_scripts': ['kamaki = kamaki.cli:main'] |
Also available in: Unified diff