root / snf-pithos-tools / pithos / tools / lib / transfer.py @ db117cac
History | View | Annotate | Download (4.2 kB)
1 | 2e662088 | Antony Chazapis | # Copyright 2011-2012 GRNET S.A. All rights reserved.
|
---|---|---|---|
2 | f390685d | Sofia Papagiannaki | #
|
3 | f390685d | Sofia Papagiannaki | # Redistribution and use in source and binary forms, with or
|
4 | f390685d | Sofia Papagiannaki | # without modification, are permitted provided that the following
|
5 | f390685d | Sofia Papagiannaki | # conditions are met:
|
6 | f390685d | Sofia Papagiannaki | #
|
7 | f390685d | Sofia Papagiannaki | # 1. Redistributions of source code must retain the above
|
8 | f390685d | Sofia Papagiannaki | # copyright notice, this list of conditions and the following
|
9 | f390685d | Sofia Papagiannaki | # disclaimer.
|
10 | f390685d | Sofia Papagiannaki | #
|
11 | f390685d | Sofia Papagiannaki | # 2. Redistributions in binary form must reproduce the above
|
12 | f390685d | Sofia Papagiannaki | # copyright notice, this list of conditions and the following
|
13 | f390685d | Sofia Papagiannaki | # disclaimer in the documentation and/or other materials
|
14 | f390685d | Sofia Papagiannaki | # provided with the distribution.
|
15 | f390685d | Sofia Papagiannaki | #
|
16 | f390685d | Sofia Papagiannaki | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 | f390685d | Sofia Papagiannaki | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 | f390685d | Sofia Papagiannaki | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 | f390685d | Sofia Papagiannaki | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 | f390685d | Sofia Papagiannaki | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 | f390685d | Sofia Papagiannaki | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 | f390685d | Sofia Papagiannaki | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 | f390685d | Sofia Papagiannaki | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 | f390685d | Sofia Papagiannaki | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 | f390685d | Sofia Papagiannaki | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 | f390685d | Sofia Papagiannaki | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 | f390685d | Sofia Papagiannaki | # POSSIBILITY OF SUCH DAMAGE.
|
28 | f390685d | Sofia Papagiannaki | #
|
29 | f390685d | Sofia Papagiannaki | # The views and conclusions contained in the software and
|
30 | f390685d | Sofia Papagiannaki | # documentation are those of the authors and should not be
|
31 | f390685d | Sofia Papagiannaki | # interpreted as representing official policies, either expressed
|
32 | f390685d | Sofia Papagiannaki | # or implied, of GRNET S.A.
|
33 | f390685d | Sofia Papagiannaki | |
34 | 822cc7eb | Antony Chazapis | import os |
35 | 98137a34 | Sofia Papagiannaki | import types |
36 | 6b6b6c1e | Antony Chazapis | import json |
37 | 822cc7eb | Antony Chazapis | |
38 | f390685d | Sofia Papagiannaki | from hashmap import HashMap |
39 | f390685d | Sofia Papagiannaki | from binascii import hexlify, unhexlify |
40 | f390685d | Sofia Papagiannaki | from cStringIO import StringIO |
41 | 822cc7eb | Antony Chazapis | from client import Fault |
42 | f390685d | Sofia Papagiannaki | |
43 | 6f6cec5a | Sofia Papagiannaki | from progress.bar import IncrementalBar |
44 | f390685d | Sofia Papagiannaki | |
45 | 0f74b3ba | Giorgos Verigakis | def upload(client, path, container, prefix, name=None, mimetype=None): |
46 | 822cc7eb | Antony Chazapis | |
47 | 822cc7eb | Antony Chazapis | meta = client.retrieve_container_metadata(container) |
48 | 822cc7eb | Antony Chazapis | blocksize = int(meta['x-container-block-size']) |
49 | 822cc7eb | Antony Chazapis | blockhash = meta['x-container-block-hash']
|
50 | f390685d | Sofia Papagiannaki | |
51 | 0f74b3ba | Giorgos Verigakis | size = os.path.getsize(path) |
52 | 822cc7eb | Antony Chazapis | hashes = HashMap(blocksize, blockhash) |
53 | 0f74b3ba | Giorgos Verigakis | hashes.load(open(path))
|
54 | f390685d | Sofia Papagiannaki | map = {'bytes': size, 'hashes': [hexlify(x) for x in hashes]} |
55 | f390685d | Sofia Papagiannaki | |
56 | 0f74b3ba | Giorgos Verigakis | objectname = name if name else os.path.split(path)[-1] |
57 | 98137a34 | Sofia Papagiannaki | object = prefix + objectname |
58 | 2db16f05 | Sofia Papagiannaki | kwargs = {'mimetype':mimetype} if mimetype else {} |
59 | 2db16f05 | Sofia Papagiannaki | v = None
|
60 | f390685d | Sofia Papagiannaki | try:
|
61 | 2db16f05 | Sofia Papagiannaki | v = client.create_object_by_hashmap(container, object, map, **kwargs) |
62 | f390685d | Sofia Papagiannaki | except Fault, fault:
|
63 | f390685d | Sofia Papagiannaki | if fault.status != 409: |
64 | f390685d | Sofia Papagiannaki | raise
|
65 | f390685d | Sofia Papagiannaki | else:
|
66 | 2db16f05 | Sofia Papagiannaki | return v
|
67 | f390685d | Sofia Papagiannaki | |
68 | 98137a34 | Sofia Papagiannaki | if type(fault.data) == types.StringType: |
69 | 6b6b6c1e | Antony Chazapis | missing = json.loads(fault.data) |
70 | 98137a34 | Sofia Papagiannaki | elif type(fault.data) == types.ListType: |
71 | 98137a34 | Sofia Papagiannaki | missing = fault.data |
72 | 98137a34 | Sofia Papagiannaki | |
73 | f390685d | Sofia Papagiannaki | if '' in missing: |
74 | f390685d | Sofia Papagiannaki | del missing[missing.index(''):] |
75 | f390685d | Sofia Papagiannaki | |
76 | 6f6cec5a | Sofia Papagiannaki | bar = IncrementalBar('Uploading', max=len(missing)) |
77 | 6f6cec5a | Sofia Papagiannaki | bar.suffix = '%(percent).1f%% - %(eta)ds'
|
78 | 0f74b3ba | Giorgos Verigakis | with open(path) as fp: |
79 | f390685d | Sofia Papagiannaki | for hash in missing: |
80 | f546c15e | Antony Chazapis | offset = hashes.index(unhexlify(hash)) * blocksize
|
81 | f390685d | Sofia Papagiannaki | fp.seek(offset) |
82 | f546c15e | Antony Chazapis | block = fp.read(blocksize) |
83 | 6d897a63 | Antony Chazapis | client.update_container_data(container, StringIO(block)) |
84 | 6f6cec5a | Sofia Papagiannaki | bar.next() |
85 | 6f6cec5a | Sofia Papagiannaki | bar.finish() |
86 | f390685d | Sofia Papagiannaki | |
87 | 2db16f05 | Sofia Papagiannaki | return client.create_object_by_hashmap(container, object, map, **kwargs) |
88 | f546c15e | Antony Chazapis | |
89 | 0f74b3ba | Giorgos Verigakis | def download(client, container, object, path): |
90 | f546c15e | Antony Chazapis | |
91 | fd9f505c | Antony Chazapis | res = client.retrieve_object_hashmap(container, object)
|
92 | fd9f505c | Antony Chazapis | blocksize = int(res['block_size']) |
93 | fd9f505c | Antony Chazapis | blockhash = res['block_hash']
|
94 | fd9f505c | Antony Chazapis | bytes = res['bytes']
|
95 | fd9f505c | Antony Chazapis | map = res['hashes']
|
96 | f546c15e | Antony Chazapis | |
97 | 0f74b3ba | Giorgos Verigakis | if os.path.exists(path):
|
98 | fd9f505c | Antony Chazapis | h = HashMap(blocksize, blockhash) |
99 | fd9f505c | Antony Chazapis | h.load(open(path))
|
100 | fd9f505c | Antony Chazapis | hashes = [hexlify(x) for x in h] |
101 | f546c15e | Antony Chazapis | else:
|
102 | 0f74b3ba | Giorgos Verigakis | open(path, 'w').close() # Create an empty file |
103 | f546c15e | Antony Chazapis | hashes = [] |
104 | f546c15e | Antony Chazapis | |
105 | fd9f505c | Antony Chazapis | with open(path, 'a+') as fp: |
106 | cb446fb8 | Antony Chazapis | if bytes != 0: |
107 | cb446fb8 | Antony Chazapis | for i, h in enumerate(map): |
108 | cb446fb8 | Antony Chazapis | if i < len(hashes) and h == hashes[i]: |
109 | cb446fb8 | Antony Chazapis | continue
|
110 | cb446fb8 | Antony Chazapis | start = i * blocksize |
111 | cb446fb8 | Antony Chazapis | end = '' if i == len(map) - 1 else ((i + 1) * blocksize) - 1 |
112 | cb446fb8 | Antony Chazapis | data = client.retrieve_object(container, object, range='bytes=%s-%s' % (start, end)) |
113 | cb446fb8 | Antony Chazapis | if i != len(map) - 1: |
114 | cb446fb8 | Antony Chazapis | data += (blocksize - len(data)) * '\x00' |
115 | cb446fb8 | Antony Chazapis | fp.seek(start) |
116 | cb446fb8 | Antony Chazapis | fp.write(data) |
117 | cb446fb8 | Antony Chazapis | fp.truncate(bytes) |