Revision 58a6c894
b/pithos/api/functions.py | ||
---|---|---|
46 | 46 |
from pithos.api.util import (format_meta_key, printable_meta_dict, get_account_meta, |
47 | 47 |
put_account_meta, get_container_meta, put_container_meta, get_object_meta, put_object_meta, |
48 | 48 |
validate_modification_preconditions, validate_matching_preconditions, copy_or_move_object, |
49 |
get_content_length, get_range, get_content_range, raw_input_socket, socket_read_iterator,
|
|
50 |
ObjectWrapper, hashmap_hash, api_method) |
|
49 |
get_version, get_content_length, get_range, get_content_range, raw_input_socket,
|
|
50 |
socket_read_iterator, ObjectWrapper, hashmap_hash, api_method)
|
|
51 | 51 |
from pithos.backends import backend |
52 | 52 |
|
53 | 53 |
|
... | ... | |
168 | 168 |
limit = 10000 |
169 | 169 |
|
170 | 170 |
try: |
171 |
containers = backend.list_containers(request.user, marker, limit)
|
|
171 |
containers = [x[0] for x in backend.list_containers(request.user, marker, limit)]
|
|
172 | 172 |
except NameError: |
173 | 173 |
containers = [] |
174 | 174 |
|
... | ... | |
322 | 322 |
keys = [] |
323 | 323 |
|
324 | 324 |
try: |
325 |
objects = backend.list_objects(request.user, v_container, prefix, delimiter, marker, limit, virtual, keys)
|
|
325 |
objects = [x[0] for x in backend.list_objects(request.user, v_container, prefix, delimiter, marker, limit, virtual, keys)]
|
|
326 | 326 |
except NameError: |
327 | 327 |
raise ItemNotFound('Container does not exist') |
328 | 328 |
|
... | ... | |
361 | 361 |
# unauthorized (401), |
362 | 362 |
# badRequest (400) |
363 | 363 |
|
364 |
version = get_version(request) |
|
364 | 365 |
try: |
365 |
meta = backend.get_object_meta(request.user, v_container, v_object) |
|
366 |
meta = backend.get_object_meta(request.user, v_container, v_object, version)
|
|
366 | 367 |
except NameError: |
367 | 368 |
raise ItemNotFound('Object does not exist') |
369 |
except IndexError: |
|
370 |
raise ItemNotFound('Version does not exist') |
|
368 | 371 |
|
369 | 372 |
response = HttpResponse(status=204) |
370 | 373 |
put_object_meta(response, meta) |
... | ... | |
381 | 384 |
# badRequest (400), |
382 | 385 |
# notModified (304) |
383 | 386 |
|
387 |
version = get_version(request) |
|
384 | 388 |
try: |
385 |
meta = backend.get_object_meta(request.user, v_container, v_object) |
|
389 |
meta = backend.get_object_meta(request.user, v_container, v_object, version)
|
|
386 | 390 |
except NameError: |
387 | 391 |
raise ItemNotFound('Object does not exist') |
392 |
except IndexError: |
|
393 |
raise ItemNotFound('Version does not exist') |
|
388 | 394 |
|
389 | 395 |
# Evaluate conditions. |
390 | 396 |
validate_modification_preconditions(request, meta) |
... | ... | |
396 | 402 |
return response |
397 | 403 |
|
398 | 404 |
try: |
399 |
# TODO: Also check for IndexError. |
|
400 |
size, hashmap = backend.get_object_hashmap(request.user, v_container, v_object) |
|
405 |
size, hashmap = backend.get_object_hashmap(request.user, v_container, v_object, version) |
|
401 | 406 |
except NameError: |
402 | 407 |
raise ItemNotFound('Object does not exist') |
408 |
except IndexError: |
|
409 |
raise ItemNotFound('Version does not exist') |
|
403 | 410 |
|
404 | 411 |
# Reply with the hashmap. |
405 | 412 |
if request.serialization != 'text': |
... | ... | |
477 | 484 |
raise LengthRequired('Missing Content-Type header') |
478 | 485 |
|
479 | 486 |
md5 = hashlib.md5() |
480 |
if content_length == 0: |
|
481 |
try: |
|
482 |
backend.update_object_hashmap(request.user, v_container, v_object, 0, []) |
|
483 |
except NameError: |
|
484 |
raise ItemNotFound('Container does not exist') |
|
485 |
else: |
|
486 |
size = 0 |
|
487 |
hashmap = [] |
|
488 |
sock = raw_input_socket(request) |
|
489 |
for data in socket_read_iterator(sock, content_length, backend.block_size): |
|
490 |
# TODO: Raise 408 (Request Timeout) if this takes too long. |
|
491 |
# TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data. |
|
492 |
size += len(data) |
|
493 |
hashmap.append(backend.put_block(data)) |
|
494 |
md5.update(data) |
|
487 |
size = 0 |
|
488 |
hashmap = [] |
|
489 |
sock = raw_input_socket(request) |
|
490 |
for data in socket_read_iterator(sock, content_length, backend.block_size): |
|
491 |
# TODO: Raise 408 (Request Timeout) if this takes too long. |
|
492 |
# TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data. |
|
493 |
size += len(data) |
|
494 |
hashmap.append(backend.put_block(data)) |
|
495 |
md5.update(data) |
|
495 | 496 |
|
496 | 497 |
meta['hash'] = md5.hexdigest().lower() |
497 | 498 |
etag = request.META.get('HTTP_ETAG') |
... | ... | |
499 | 500 |
raise UnprocessableEntity('Object ETag does not match') |
500 | 501 |
|
501 | 502 |
try: |
503 |
# TODO: Update metadata with the hashmap. |
|
502 | 504 |
backend.update_object_hashmap(request.user, v_container, v_object, size, hashmap) |
503 | 505 |
except NameError: |
504 | 506 |
raise ItemNotFound('Container does not exist') |
... | ... | |
589 | 591 |
content_length = get_content_length(request) |
590 | 592 |
|
591 | 593 |
try: |
592 |
# TODO: Also check for IndexError. |
|
593 | 594 |
size, hashmap = backend.get_object_hashmap(request.user, v_container, v_object) |
594 | 595 |
except NameError: |
595 | 596 |
raise ItemNotFound('Object does not exist') |
... | ... | |
632 | 633 |
if offset > size: |
633 | 634 |
size = offset |
634 | 635 |
try: |
636 |
# TODO: Update metadata with the hashmap. |
|
635 | 637 |
backend.update_object_hashmap(request.user, v_container, v_object, size, hashmap) |
636 | 638 |
except NameError: |
637 | 639 |
raise ItemNotFound('Container does not exist') |
638 | 640 |
|
639 | 641 |
# Update ETag. |
640 |
# TODO: Move this to the backend. |
|
641 | 642 |
meta = {} |
642 | 643 |
meta['hash'] = hashmap_hash(hashmap) |
643 | 644 |
try: |
b/pithos/api/tests.py | ||
---|---|---|
82 | 82 |
'Last-Modified', |
83 | 83 |
'Date', |
84 | 84 |
'X-Object-Manifest', |
85 |
'Content-Range',)} |
|
85 |
'Content-Range', |
|
86 |
'X-Object-Version', |
|
87 |
'X-Object-Version-Timestamp',)} |
|
86 | 88 |
self.contentTypes = {'xml':'application/xml', |
87 | 89 |
'json':'application/json', |
88 | 90 |
'':'text/plain'} |
b/pithos/api/util.py | ||
---|---|---|
67 | 67 |
return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()]) |
68 | 68 |
|
69 | 69 |
def format_meta_key(k): |
70 |
"""Convert underscores to dashes and capitalize intra-dash strings""" |
|
70 |
"""Convert underscores to dashes and capitalize intra-dash strings."""
|
|
71 | 71 |
return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')]) |
72 | 72 |
|
73 | 73 |
def get_meta_prefix(request, prefix): |
74 |
"""Get all prefix-* request headers in a dict. Reformat keys with format_meta_key()""" |
|
74 |
"""Get all prefix-* request headers in a dict. Reformat keys with format_meta_key()."""
|
|
75 | 75 |
prefix = 'HTTP_' + prefix.upper().replace('-', '_') |
76 | 76 |
return dict([(format_meta_key(k[5:]), v) for k, v in request.META.iteritems() if k.startswith(prefix)]) |
77 | 77 |
|
78 | 78 |
def get_account_meta(request): |
79 |
"""Get metadata from an account request""" |
|
79 |
"""Get metadata from an account request."""
|
|
80 | 80 |
meta = get_meta_prefix(request, 'X-Account-Meta-') |
81 | 81 |
return meta |
82 | 82 |
|
83 | 83 |
def put_account_meta(response, meta): |
84 |
"""Put metadata in an account response""" |
|
84 |
"""Put metadata in an account response."""
|
|
85 | 85 |
response['X-Account-Container-Count'] = meta['count'] |
86 | 86 |
response['X-Account-Bytes-Used'] = meta['bytes'] |
87 | 87 |
if 'modified' in meta: |
... | ... | |
90 | 90 |
response[k.encode('utf-8')] = meta[k].encode('utf-8') |
91 | 91 |
|
92 | 92 |
def get_container_meta(request): |
93 |
"""Get metadata from a container request""" |
|
93 |
"""Get metadata from a container request."""
|
|
94 | 94 |
meta = get_meta_prefix(request, 'X-Container-Meta-') |
95 | 95 |
return meta |
96 | 96 |
|
97 | 97 |
def put_container_meta(response, meta): |
98 |
"""Put metadata in a container response""" |
|
98 |
"""Put metadata in a container response."""
|
|
99 | 99 |
response['X-Container-Object-Count'] = meta['count'] |
100 | 100 |
response['X-Container-Bytes-Used'] = meta['bytes'] |
101 |
if 'modified' in meta: |
|
102 |
response['Last-Modified'] = http_date(int(meta['modified'])) |
|
101 |
response['Last-Modified'] = http_date(int(meta['modified'])) |
|
103 | 102 |
for k in [x for x in meta.keys() if x.startswith('X-Container-Meta-')]: |
104 | 103 |
response[k.encode('utf-8')] = meta[k].encode('utf-8') |
105 | 104 |
response['X-Container-Object-Meta'] = [x[14:] for x in meta['object_meta'] if x.startswith('X-Object-Meta-')] |
... | ... | |
107 | 106 |
response['X-Container-Block-Hash'] = backend.hash_algorithm |
108 | 107 |
|
109 | 108 |
def get_object_meta(request): |
110 |
"""Get metadata from an object request""" |
|
109 |
"""Get metadata from an object request."""
|
|
111 | 110 |
meta = get_meta_prefix(request, 'X-Object-Meta-') |
112 | 111 |
if request.META.get('CONTENT_TYPE'): |
113 | 112 |
meta['Content-Type'] = request.META['CONTENT_TYPE'] |
... | ... | |
120 | 119 |
return meta |
121 | 120 |
|
122 | 121 |
def put_object_meta(response, meta): |
123 |
"""Put metadata in an object response""" |
|
122 |
"""Put metadata in an object response."""
|
|
124 | 123 |
response['ETag'] = meta['hash'] |
125 | 124 |
response['Content-Length'] = meta['bytes'] |
126 | 125 |
response['Content-Type'] = meta.get('Content-Type', 'application/octet-stream') |
127 | 126 |
response['Last-Modified'] = http_date(int(meta['modified'])) |
127 |
response['X-Object-Version'] = meta['version'] |
|
128 |
response['X-Object-Version-Timestamp'] = meta['version_timestamp'] |
|
128 | 129 |
for k in [x for x in meta.keys() if x.startswith('X-Object-Meta-')]: |
129 | 130 |
response[k.encode('utf-8')] = meta[k].encode('utf-8') |
130 | 131 |
for k in ('Content-Encoding', 'Content-Disposition', 'X-Object-Manifest'): |
... | ... | |
132 | 133 |
response[k] = meta[k] |
133 | 134 |
|
134 | 135 |
def validate_modification_preconditions(request, meta): |
135 |
"""Check that the modified timestamp conforms with the preconditions set""" |
|
136 |
"""Check that the modified timestamp conforms with the preconditions set."""
|
|
136 | 137 |
if 'modified' not in meta: |
137 | 138 |
return # TODO: Always return? |
138 | 139 |
|
... | ... | |
149 | 150 |
raise PreconditionFailed('Resource has been modified') |
150 | 151 |
|
151 | 152 |
def validate_matching_preconditions(request, meta): |
152 |
"""Check that the ETag conforms with the preconditions set""" |
|
153 |
"""Check that the ETag conforms with the preconditions set."""
|
|
153 | 154 |
if 'hash' not in meta: |
154 | 155 |
return # TODO: Always return? |
155 | 156 |
|
... | ... | |
164 | 165 |
raise NotModified('Resource Etag matches') |
165 | 166 |
|
166 | 167 |
def copy_or_move_object(request, src_path, dest_path, move=False): |
167 |
"""Copy or move an object""" |
|
168 |
"""Copy or move an object."""
|
|
168 | 169 |
if type(src_path) == str: |
169 | 170 |
parts = src_path.split('/') |
170 | 171 |
if len(parts) < 3 or parts[0] != '': |
... | ... | |
194 | 195 |
if k in src_meta: |
195 | 196 |
meta[k] = src_meta[k] |
196 | 197 |
|
198 |
# TODO: Copy or move with 'versioned' set. |
|
197 | 199 |
try: |
198 | 200 |
if move: |
199 | 201 |
backend.move_object(request.user, src_container, src_name, dest_container, dest_name, meta, replace_meta=True) |
... | ... | |
202 | 204 |
except NameError: |
203 | 205 |
raise ItemNotFound('Container or object does not exist') |
204 | 206 |
|
207 |
def get_version(request): |
|
208 |
version = request.GET.get('version') |
|
209 |
if version is not None: |
|
210 |
try: |
|
211 |
version = int(version) |
|
212 |
except ValueError: |
|
213 |
return None |
|
214 |
if version < 0: |
|
215 |
return None |
|
216 |
return version |
|
217 |
|
|
205 | 218 |
def get_content_length(request): |
206 | 219 |
content_length = request.META.get('CONTENT_LENGTH') |
207 | 220 |
if not content_length: |
... | ... | |
215 | 228 |
return content_length |
216 | 229 |
|
217 | 230 |
def get_range(request, size): |
218 |
"""Parse a Range header from the request |
|
231 |
"""Parse a Range header from the request.
|
|
219 | 232 |
|
220 | 233 |
Either returns None, when the header is not existent or should be ignored, |
221 | 234 |
or a list of (offset, length) tuples - should be further checked. |
... | ... | |
251 | 264 |
return ret |
252 | 265 |
|
253 | 266 |
def get_content_range(request): |
254 |
"""Parse a Content-Range header from the request |
|
267 |
"""Parse a Content-Range header from the request.
|
|
255 | 268 |
|
256 | 269 |
Either returns None, when the header is not existent or should be ignored, |
257 | 270 |
or an (offset, length, total) tuple - check as length, total may be None. |
... | ... | |
291 | 304 |
return (offset, length, total) |
292 | 305 |
|
293 | 306 |
def raw_input_socket(request): |
294 |
"""Return the socket for reading the rest of the request""" |
|
307 |
"""Return the socket for reading the rest of the request."""
|
|
295 | 308 |
server_software = request.META.get('SERVER_SOFTWARE') |
296 | 309 |
if not server_software: |
297 | 310 |
if 'wsgi.input' in request.environ: |
... | ... | |
306 | 319 |
MAX_UPLOAD_SIZE = 10 * (1024 * 1024) # 10MB |
307 | 320 |
|
308 | 321 |
def socket_read_iterator(sock, length=0, blocksize=4096): |
309 |
"""Return a maximum of blocksize data read from the socket in each iteration |
|
322 |
"""Return a maximum of blocksize data read from the socket in each iteration.
|
|
310 | 323 |
|
311 | 324 |
Read up to 'length'. If 'length' is negative, will attempt a chunked read. |
312 | 325 |
The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE. |
... | ... | |
356 | 369 |
yield data |
357 | 370 |
|
358 | 371 |
class ObjectWrapper(object): |
359 |
"""Return the object's data block-per-block in each iteration |
|
372 |
"""Return the object's data block-per-block in each iteration.
|
|
360 | 373 |
|
361 | 374 |
Read from the object using the offset and length provided in each entry of the range list. |
362 | 375 |
""" |
... | ... | |
430 | 443 |
return '\r\n'.join(out) |
431 | 444 |
|
432 | 445 |
def hashmap_hash(hashmap): |
433 |
""" Produce the root hash, treating the hashmap as a Merkle-like tree."""
|
|
446 |
"""Produce the root hash, treating the hashmap as a Merkle-like tree.""" |
|
434 | 447 |
|
435 | 448 |
def subhash(d): |
436 | 449 |
h = hashlib.new(backend.hash_algorithm) |
... | ... | |
472 | 485 |
return response |
473 | 486 |
|
474 | 487 |
def request_serialization(request, format_allowed=False): |
475 |
"""Return the serialization format requested |
|
488 |
"""Return the serialization format requested.
|
|
476 | 489 |
|
477 | 490 |
Valid formats are 'text' and 'json', 'xml' if 'format_allowed' is True. |
478 | 491 |
""" |
... | ... | |
495 | 508 |
return 'text' |
496 | 509 |
|
497 | 510 |
def api_method(http_method=None, format_allowed=False): |
498 |
"""Decorator function for views that implement an API method""" |
|
511 |
"""Decorator function for views that implement an API method."""
|
|
499 | 512 |
def decorator(func): |
500 | 513 |
@wraps(func) |
501 | 514 |
def wrapper(request, *args, **kwargs): |
b/pithos/backends/base.py | ||
---|---|---|
44 | 44 |
'block_size': Suggested is 4MB |
45 | 45 |
""" |
46 | 46 |
|
47 |
def get_account_meta(self, account): |
|
47 |
def delete_account(self, account): |
|
48 |
"""Delete the account with the given name. |
|
49 |
|
|
50 |
Raises: |
|
51 |
IndexError: Account is not empty |
|
52 |
""" |
|
53 |
return |
|
54 |
|
|
55 |
def get_account_meta(self, account, until=None): |
|
48 | 56 |
"""Return a dictionary with the account metadata. |
49 | 57 |
|
50 | 58 |
The keys returned are all user-defined, except: |
51 | 59 |
'name': The account name |
52 | 60 |
'count': The number of containers (or 0) |
53 | 61 |
'bytes': The total data size (or 0) |
54 |
'modified': Last modification timestamp |
|
62 |
'modified': Last modification timestamp (overall) |
|
63 |
'until_timestamp': Last modification until the timestamp provided |
|
55 | 64 |
""" |
56 | 65 |
return {} |
57 | 66 |
|
... | ... | |
64 | 73 |
""" |
65 | 74 |
return |
66 | 75 |
|
67 |
def put_container(self, account, name): |
|
76 |
def list_containers(self, account, marker=None, limit=10000, until=None): |
|
77 |
"""Return a list of container (name, version_id) tuples existing under an account. |
|
78 |
|
|
79 |
Parameters: |
|
80 |
'marker': Start list from the next item after 'marker' |
|
81 |
'limit': Number of containers to return |
|
82 |
""" |
|
83 |
return [] |
|
84 |
|
|
85 |
def put_container(self, account, container): |
|
68 | 86 |
"""Create a new container with the given name. |
69 | 87 |
|
70 | 88 |
Raises: |
... | ... | |
72 | 90 |
""" |
73 | 91 |
return |
74 | 92 |
|
75 |
def delete_container(self, account, name):
|
|
93 |
def delete_container(self, account, container):
|
|
76 | 94 |
"""Delete the container with the given name. |
77 | 95 |
|
78 | 96 |
Raises: |
... | ... | |
81 | 99 |
""" |
82 | 100 |
return |
83 | 101 |
|
84 |
def get_container_meta(self, account, name):
|
|
102 |
def get_container_meta(self, account, container, until=None):
|
|
85 | 103 |
"""Return a dictionary with the container metadata. |
86 | 104 |
|
87 | 105 |
The keys returned are all user-defined, except: |
88 | 106 |
'name': The container name |
89 | 107 |
'count': The number of objects |
90 | 108 |
'bytes': The total data size |
91 |
'created': Creation timestamp
|
|
92 |
'modified': Last modification timestamp
|
|
109 |
'modified': Last modification timestamp (overall)
|
|
110 |
'until_timestamp': Last modification until the timestamp provided
|
|
93 | 111 |
|
94 | 112 |
Raises: |
95 | 113 |
NameError: Container does not exist |
96 | 114 |
""" |
97 | 115 |
return {} |
98 | 116 |
|
99 |
def update_container_meta(self, account, name, meta, replace=False):
|
|
117 |
def update_container_meta(self, account, container, meta, replace=False):
|
|
100 | 118 |
"""Update the metadata associated with the container. |
101 | 119 |
|
102 | 120 |
Parameters: |
... | ... | |
108 | 126 |
""" |
109 | 127 |
return |
110 | 128 |
|
111 |
def list_containers(self, account, marker=None, limit=10000): |
|
112 |
"""Return a list of containers existing under an account. |
|
113 |
|
|
114 |
Parameters: |
|
115 |
'marker': Start list from the next item after 'marker' |
|
116 |
'limit': Number of containers to return |
|
117 |
""" |
|
118 |
return [] |
|
119 |
|
|
120 |
def list_objects(self, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[]): |
|
121 |
"""Return a list of objects existing under a container. |
|
129 |
def list_objects(self, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[], until=None): |
|
130 |
"""Return a list of object (name, version_id) tuples existing under a container. |
|
122 | 131 |
|
123 | 132 |
Parameters: |
124 | 133 |
'prefix': List objects starting with 'prefix' |
... | ... | |
137 | 146 |
""" |
138 | 147 |
return [] |
139 | 148 |
|
140 |
def list_object_meta(self, account, name):
|
|
149 |
def list_object_meta(self, account, container, until=None):
|
|
141 | 150 |
"""Return a list with all the container's object meta keys. |
142 | 151 |
|
143 | 152 |
Raises: |
... | ... | |
145 | 154 |
""" |
146 | 155 |
return [] |
147 | 156 |
|
148 |
def get_object_meta(self, account, container, name): |
|
157 |
def get_object_meta(self, account, container, name, version=None):
|
|
149 | 158 |
"""Return a dictionary with the object metadata. |
150 | 159 |
|
151 | 160 |
The keys returned are all user-defined, except: |
152 |
'name': The account name
|
|
161 |
'name': The object name
|
|
153 | 162 |
'bytes': The total data size |
154 |
'version': The latest version (zero if not versioned)
|
|
155 |
'created': Creation timestamp
|
|
156 |
'modified': Last modification timestamp
|
|
163 |
'modified': Last modification timestamp (overall)
|
|
164 |
'version': The version identifier
|
|
165 |
'version_timestamp': The version's modification timestamp
|
|
157 | 166 |
|
158 | 167 |
Raises: |
159 | 168 |
NameError: Container/object does not exist |
169 |
IndexError: Version does not exist |
|
160 | 170 |
""" |
161 | 171 |
return {} |
162 | 172 |
|
... | ... | |
164 | 174 |
"""Update the metadata associated with the object. |
165 | 175 |
|
166 | 176 |
Parameters: |
167 |
'meta': Dictionary with metadata to update.\ |
|
168 |
Use the 'versioned' key to control versioning |
|
177 |
'meta': Dictionary with metadata to update. |
|
169 | 178 |
'replace': Replace instead of update |
170 | 179 |
|
171 | 180 |
Raises: |
... | ... | |
186 | 195 |
"""Create/update an object with the specified size and partial hashes. |
187 | 196 |
|
188 | 197 |
Raises: |
189 |
NameError: Container/block does not exist
|
|
198 |
NameError: Container does not exist |
|
190 | 199 |
""" |
191 | 200 |
return |
192 | 201 |
|
193 |
def copy_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False): |
|
202 |
def copy_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False, src_version=None):
|
|
194 | 203 |
"""Copy an object's data and metadata. |
195 | 204 |
|
196 | 205 |
Parameters: |
197 | 206 |
'dest_meta': Dictionary with metadata to changes from source to destination |
198 | 207 |
'replace_meta': Replace metadata instead of update |
208 |
'src_version': Copy from the version provided. |
|
199 | 209 |
|
200 | 210 |
Raises: |
201 | 211 |
NameError: Container/object does not exist |
212 |
IndexError: Version does not exist |
|
202 | 213 |
""" |
203 | 214 |
return |
204 | 215 |
|
205 |
def move_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False): |
|
216 |
def move_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False, src_version=None):
|
|
206 | 217 |
"""Move an object's data and metadata. |
207 | 218 |
|
208 | 219 |
Parameters: |
209 | 220 |
'dest_meta': Dictionary with metadata to changes from source to destination |
210 | 221 |
'replace_meta': Replace metadata instead of update |
222 |
'src_version': Copy from the version provided. |
|
211 | 223 |
|
212 | 224 |
Raises: |
213 | 225 |
NameError: Container/object does not exist |
226 |
IndexError: Version does not exist |
|
214 | 227 |
""" |
215 | 228 |
return |
216 | 229 |
|
... | ... | |
222 | 235 |
""" |
223 | 236 |
return |
224 | 237 |
|
238 |
def list_versions(self, account, container, name): |
|
239 |
"""Return a list of version (version_id, version_modified) tuples for an object.""" |
|
240 |
return [] |
|
241 |
|
|
225 | 242 |
def get_block(self, hash): |
226 | 243 |
"""Return a block's data. |
227 | 244 |
|
b/pithos/backends/simple.py | ||
---|---|---|
52 | 52 |
Uses SQLite for storage. |
53 | 53 |
""" |
54 | 54 |
|
55 |
# TODO: Automatic/manual clean-up after a time interval. |
|
56 |
|
|
55 | 57 |
def __init__(self, db): |
56 | 58 |
self.hash_algorithm = 'sha1' |
57 | 59 |
self.block_size = 128 * 1024 # 128KB |
... | ... | |
61 | 63 |
os.makedirs(basepath) |
62 | 64 |
|
63 | 65 |
self.con = sqlite3.connect(db) |
64 |
sql = '''create table if not exists objects ( |
|
65 |
name text, tstamp text, primary key (name))''' |
|
66 |
sql = '''create table if not exists versions ( |
|
67 |
version_id integer primary key, |
|
68 |
name text, |
|
69 |
tstamp datetime default current_timestamp, |
|
70 |
size integer default 0, |
|
71 |
hide integer default 0)''' |
|
66 | 72 |
self.con.execute(sql) |
67 | 73 |
sql = '''create table if not exists metadata ( |
68 |
name text, key text, value text, primary key (name, key))''' |
|
69 |
self.con.execute(sql) |
|
70 |
sql = '''create table if not exists versions ( |
|
71 |
object_id int, version int, size int, primary key (object_id, version))''' |
|
74 |
version_id integer, key text, value text, primary key (version_id, key))''' |
|
72 | 75 |
self.con.execute(sql) |
73 | 76 |
sql = '''create table if not exists blocks ( |
74 | 77 |
block_id text, data blob, primary key (block_id))''' |
75 | 78 |
self.con.execute(sql) |
76 | 79 |
sql = '''create table if not exists hashmaps ( |
77 |
version_id int, pos int, block_id text, primary key (version_id, pos))'''
|
|
80 |
version_id integer, pos integer, block_id text, primary key (version_id, pos))'''
|
|
78 | 81 |
self.con.execute(sql) |
79 | 82 |
self.con.commit() |
80 | 83 |
|
81 |
def get_account_meta(self, account): |
|
84 |
def delete_account(self, account): |
|
85 |
"""Delete the account with the given name.""" |
|
86 |
|
|
87 |
logger.debug("delete_account: %s", account) |
|
88 |
count, bytes, tstamp = self._get_pathstats(account) |
|
89 |
if count > 0: |
|
90 |
raise IndexError('Account is not empty') |
|
91 |
self._del_path(account) # Point of no return. |
|
92 |
|
|
93 |
def get_account_meta(self, account, until=None): |
|
82 | 94 |
"""Return a dictionary with the account metadata.""" |
83 | 95 |
|
84 |
logger.debug("get_account_meta: %s", account) |
|
85 |
count, bytes = self._get_pathstats(account) |
|
96 |
logger.debug("get_account_meta: %s %s", account, until) |
|
97 |
try: |
|
98 |
version_id, mtime = self._get_accountinfo(account, until) |
|
99 |
except NameError: |
|
100 |
version_id = None |
|
101 |
count, bytes, tstamp = self._get_pathstats(account, until) |
|
102 |
if until is None: |
|
103 |
modified = tstamp |
|
104 |
else: |
|
105 |
modified = self._get_pathstats(account)[2] # Overall last modification |
|
86 | 106 |
|
87 | 107 |
# Proper count. |
88 |
sql = 'select count(name) from objects where name glob ? and not name glob ?' |
|
108 |
sql = 'select count(name) from (%s) where name glob ? and not name glob ?' |
|
109 |
sql = sql % self._sql_until(until) |
|
89 | 110 |
c = self.con.execute(sql, (account + '/*', account + '/*/*')) |
90 | 111 |
row = c.fetchone() |
91 | 112 |
count = row[0] |
92 | 113 |
|
93 |
meta = self._get_metadata(account) |
|
114 |
meta = self._get_metadata(account, version_id)
|
|
94 | 115 |
meta.update({'name': account, 'count': count, 'bytes': bytes}) |
116 |
if modified: |
|
117 |
meta.update({'modified': modified}) |
|
118 |
if until is not None: |
|
119 |
meta.update({'until_timestamp': tstamp}) |
|
95 | 120 |
return meta |
96 | 121 |
|
97 | 122 |
def update_account_meta(self, account, meta, replace=False): |
98 | 123 |
"""Update the metadata associated with the account.""" |
99 | 124 |
|
100 | 125 |
logger.debug("update_account_meta: %s %s %s", account, meta, replace) |
101 |
self._update_metadata(account, None, None, meta, replace) |
|
126 |
self._put_metadata(account, meta, replace) |
|
127 |
|
|
128 |
def list_containers(self, account, marker=None, limit=10000, until=None): |
|
129 |
"""Return a list of containers existing under an account.""" |
|
130 |
|
|
131 |
logger.debug("list_containers: %s %s %s %s", account, marker, limit, until) |
|
132 |
return self._list_objects(account, '', '/', marker, limit, False, [], until) |
|
102 | 133 |
|
103 |
def put_container(self, account, name):
|
|
134 |
def put_container(self, account, container):
|
|
104 | 135 |
"""Create a new container with the given name.""" |
105 | 136 |
|
106 |
logger.debug("put_container: %s %s", account, name)
|
|
137 |
logger.debug("put_container: %s %s", account, container)
|
|
107 | 138 |
try: |
108 |
path, link, tstamp = self._get_containerinfo(account, name)
|
|
139 |
path, version_id, mtime = self._get_containerinfo(account, container)
|
|
109 | 140 |
except NameError: |
110 |
path = os.path.join(account, name)
|
|
111 |
link = self._put_linkinfo(path)
|
|
141 |
path = os.path.join(account, container)
|
|
142 |
version_id = self._put_version(path)
|
|
112 | 143 |
else: |
113 | 144 |
raise NameError('Container already exists') |
114 |
self._update_metadata(account, name, None, None) |
|
115 | 145 |
|
116 |
def delete_container(self, account, name):
|
|
146 |
def delete_container(self, account, container):
|
|
117 | 147 |
"""Delete the container with the given name.""" |
118 | 148 |
|
119 |
logger.debug("delete_container: %s %s", account, name)
|
|
120 |
path, link, tstamp = self._get_containerinfo(account, name)
|
|
121 |
count, bytes = self._get_pathstats(path) |
|
149 |
logger.debug("delete_container: %s %s", account, container)
|
|
150 |
path, version_id, mtime = self._get_containerinfo(account, container)
|
|
151 |
count, bytes, tstamp = self._get_pathstats(path)
|
|
122 | 152 |
if count > 0: |
123 | 153 |
raise IndexError('Container is not empty') |
124 |
self._del_path(path) |
|
125 |
self._update_metadata(account, None, None, None)
|
|
154 |
self._del_path(path) # Point of no return.
|
|
155 |
self._copy_version(account, account, True, True) # New account version.
|
|
126 | 156 |
|
127 |
def get_container_meta(self, account, name):
|
|
157 |
def get_container_meta(self, account, container, until=None):
|
|
128 | 158 |
"""Return a dictionary with the container metadata.""" |
129 | 159 |
|
130 |
logger.debug("get_container_meta: %s %s", account, name) |
|
131 |
path, link, tstamp = self._get_containerinfo(account, name) |
|
132 |
count, bytes = self._get_pathstats(path) |
|
133 |
meta = self._get_metadata(path) |
|
134 |
meta.update({'name': name, 'count': count, 'bytes': bytes, 'created': tstamp}) |
|
160 |
logger.debug("get_container_meta: %s %s %s", account, container, until) |
|
161 |
|
|
162 |
path, version_id, mtime = self._get_containerinfo(account, container, until) |
|
163 |
count, bytes, tstamp = self._get_pathstats(path, until) |
|
164 |
if until is None: |
|
165 |
modified = tstamp |
|
166 |
else: |
|
167 |
modified = self._get_pathstats(account)[2] # Overall last modification |
|
168 |
|
|
169 |
meta = self._get_metadata(path, version_id) |
|
170 |
meta.update({'name': container, 'count': count, 'bytes': bytes, 'modified': modified}) |
|
171 |
if until is not None: |
|
172 |
meta.update({'until_timestamp': tstamp}) |
|
135 | 173 |
return meta |
136 | 174 |
|
137 |
def update_container_meta(self, account, name, meta, replace=False):
|
|
175 |
def update_container_meta(self, account, container, meta, replace=False):
|
|
138 | 176 |
"""Update the metadata associated with the container.""" |
139 | 177 |
|
140 |
logger.debug("update_container_meta: %s %s %s %s", account, name, meta, replace) |
|
141 |
path, link, tstamp = self._get_containerinfo(account, name) |
|
142 |
self._update_metadata(account, name, None, meta, replace) |
|
143 |
|
|
144 |
def list_containers(self, account, marker=None, limit=10000): |
|
145 |
"""Return a list of containers existing under an account.""" |
|
146 |
|
|
147 |
logger.debug("list_containers: %s %s %s", account, marker, limit) |
|
148 |
return self._list_objects(account, '', '/', marker, limit, False, []) |
|
178 |
logger.debug("update_container_meta: %s %s %s %s", account, container, meta, replace) |
|
179 |
path, version_id, mtime = self._get_containerinfo(account, container) |
|
180 |
self._put_metadata(path, meta, replace) |
|
149 | 181 |
|
150 |
def list_objects(self, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[]): |
|
182 |
def list_objects(self, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[], until=None):
|
|
151 | 183 |
"""Return a list of objects existing under a container.""" |
152 | 184 |
|
153 |
logger.debug("list_objects: %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit)
|
|
154 |
path, link, tstamp = self._get_containerinfo(account, container)
|
|
155 |
return self._list_objects(path, prefix, delimiter, marker, limit, virtual, keys) |
|
185 |
logger.debug("list_objects: %s %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit, until)
|
|
186 |
path, version_id, mtime = self._get_containerinfo(account, container, until)
|
|
187 |
return self._list_objects(path, prefix, delimiter, marker, limit, virtual, keys, until)
|
|
156 | 188 |
|
157 |
def list_object_meta(self, account, name):
|
|
189 |
def list_object_meta(self, account, container, until=None):
|
|
158 | 190 |
"""Return a list with all the container's object meta keys.""" |
159 | 191 |
|
160 |
logger.debug("list_object_meta: %s %s", account, name) |
|
161 |
path, link, tstamp = self._get_containerinfo(account, name) |
|
162 |
sql = 'select distinct key from metadata where name like ?' |
|
192 |
logger.debug("list_object_meta: %s %s %s", account, container, until) |
|
193 |
path, version_id, mtime = self._get_containerinfo(account, container, until) |
|
194 |
sql = '''select distinct m.key from (%s) o, metadata m |
|
195 |
where m.version_id = o.version_id and o.name like ?''' |
|
196 |
sql = sql % self._sql_until(until) |
|
163 | 197 |
c = self.con.execute(sql, (path + '/%',)) |
164 | 198 |
return [x[0] for x in c.fetchall()] |
165 | 199 |
|
166 |
def get_object_meta(self, account, container, name): |
|
200 |
def get_object_meta(self, account, container, name, version=None):
|
|
167 | 201 |
"""Return a dictionary with the object metadata.""" |
168 | 202 |
|
169 |
logger.debug("get_object_meta: %s %s %s", account, container, name) |
|
170 |
path, link, tstamp = self._get_containerinfo(account, container) |
|
171 |
path, link, tstamp, version, size = self._get_objectinfo(account, container, name) |
|
172 |
meta = self._get_metadata(path) |
|
173 |
meta.update({'name': name, 'bytes': size, 'version': version, 'created': tstamp}) |
|
203 |
logger.debug("get_object_meta: %s %s %s %s", account, container, name, version) |
|
204 |
path, version_id, mtime, size = self._get_objectinfo(account, container, name, version) |
|
205 |
if version is None: |
|
206 |
modified = mtime |
|
207 |
else: |
|
208 |
modified = self._get_version(path)[1] # Overall last modification |
|
209 |
|
|
210 |
meta = self._get_metadata(path, version_id) |
|
211 |
meta.update({'name': name, 'bytes': size, 'version': version_id, 'version_timestamp': mtime, 'modified': modified}) |
|
174 | 212 |
return meta |
175 | 213 |
|
176 | 214 |
def update_object_meta(self, account, container, name, meta, replace=False): |
177 | 215 |
"""Update the metadata associated with the object.""" |
178 | 216 |
|
179 | 217 |
logger.debug("update_object_meta: %s %s %s %s %s", account, container, name, meta, replace) |
180 |
path, link, tstamp = self._get_containerinfo(account, container) |
|
181 |
path, link, tstamp, version, size = self._get_objectinfo(account, container, name) |
|
182 |
if 'versioned' in meta: |
|
183 |
if meta['versioned']: |
|
184 |
if version == 0: |
|
185 |
sql = 'update versions set version = 1 where object_id = ?' |
|
186 |
self.con.execute(sql, (link,)) |
|
187 |
self.con.commit() |
|
188 |
else: |
|
189 |
if version > 0: |
|
190 |
self._del_uptoversion(link, version) |
|
191 |
sql = 'update versions set version = 0 where object_id = ?' |
|
192 |
self.con.execute(sql, (link,)) |
|
193 |
self.con.commit() |
|
194 |
del(meta['versioned']) |
|
195 |
self._update_metadata(account, container, name, meta, replace) |
|
218 |
path, version_id, mtime, size = self._get_objectinfo(account, container, name) |
|
219 |
self._put_metadata(path, meta, replace) |
|
196 | 220 |
|
197 | 221 |
def get_object_hashmap(self, account, container, name, version=None): |
198 | 222 |
"""Return the object's size and a list with partial hashes.""" |
199 | 223 |
|
200 | 224 |
logger.debug("get_object_hashmap: %s %s %s %s", account, container, name, version) |
201 |
path, link, tstamp = self._get_containerinfo(account, container) |
|
202 |
path, link, tstamp, version, size = self._get_objectinfo(account, container, name, version) |
|
203 |
|
|
204 |
sql = '''select block_id from hashmaps where version_id = |
|
205 |
(select rowid from versions where object_id = ? and version = ?) |
|
206 |
order by pos''' |
|
207 |
c = self.con.execute(sql, (link, version)) |
|
225 |
path, version_id, mtime, size = self._get_objectinfo(account, container, name, version) |
|
226 |
sql = 'select block_id from hashmaps where version_id = ? order by pos asc' |
|
227 |
c = self.con.execute(sql, (version_id,)) |
|
208 | 228 |
hashmap = [x[0] for x in c.fetchall()] |
209 | 229 |
return size, hashmap |
210 | 230 |
|
... | ... | |
212 | 232 |
"""Create/update an object with the specified size and partial hashes.""" |
213 | 233 |
|
214 | 234 |
logger.debug("update_object_hashmap: %s %s %s %s %s", account, container, name, size, hashmap) |
215 |
path, link, tstamp = self._get_containerinfo(account, container) |
|
216 |
try: |
|
217 |
path, link, tstamp, version, s = self._get_objectinfo(account, container, name) |
|
218 |
except NameError: |
|
219 |
version = 0 |
|
220 |
|
|
221 |
if version == 0: |
|
222 |
path = os.path.join(account, container, name) |
|
223 |
|
|
224 |
self._del_path(path, delmeta=False) |
|
225 |
link = self._put_linkinfo(path) |
|
226 |
else: |
|
227 |
version += 1 |
|
228 |
|
|
229 |
sql = 'insert or replace into versions (object_id, version, size) values (?, ?, ?)' |
|
230 |
version_id = self.con.execute(sql, (link, version, size)).lastrowid |
|
235 |
path = self._get_containerinfo(account, container)[0] |
|
236 |
path = os.path.join(path, name) |
|
237 |
src_version_id, dest_version_id = self._copy_version(path, path, True, False) |
|
238 |
sql = 'update versions set size = ? where version_id = ?' |
|
239 |
self.con.execute(sql, (size, dest_version_id)) |
|
240 |
# TODO: Check for block_id existence. |
|
231 | 241 |
for i in range(len(hashmap)): |
232 | 242 |
sql = 'insert or replace into hashmaps (version_id, pos, block_id) values (?, ?, ?)' |
233 |
self.con.execute(sql, (version_id, i, hashmap[i])) |
|
243 |
self.con.execute(sql, (dest_version_id, i, hashmap[i]))
|
|
234 | 244 |
self.con.commit() |
235 | 245 |
|
236 |
def copy_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False): |
|
246 |
def copy_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False, src_version=None):
|
|
237 | 247 |
"""Copy an object's data and metadata.""" |
238 | 248 |
|
239 |
logger.debug("copy_object: %s %s %s %s %s %s %s", account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta) |
|
240 |
size, hashmap = self.get_object_hashmap(account, src_container, src_name) |
|
241 |
self.update_object_hashmap(account, dest_container, dest_name, size, hashmap) |
|
242 |
if not replace_meta: |
|
243 |
meta = self._get_metadata(os.path.join(account, src_container, src_name)) |
|
244 |
meta.update(dest_meta) |
|
249 |
logger.debug("copy_object: %s %s %s %s %s %s %s %s", account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta, src_version) |
|
250 |
if src_version is None: |
|
251 |
src_path = self._get_objectinfo(account, src_container, src_name)[0] |
|
245 | 252 |
else: |
246 |
meta = dest_meta |
|
247 |
self._update_metadata(account, dest_container, dest_name, meta, replace_meta) |
|
253 |
src_path = os.path.join(account, src_container, src_name) |
|
254 |
dest_path = self._get_containerinfo(account, dest_container)[0] |
|
255 |
dest_path = os.path.join(dest_path, dest_name) |
|
256 |
src_version_id, dest_version_id = self._copy_version(src_path, dest_path, not replace_meta, True, src_version) |
|
257 |
for k, v in dest_meta.iteritems(): |
|
258 |
sql = 'insert or replace into metadata (version_id, key, value) values (?, ?, ?)' |
|
259 |
self.con.execute(sql, (dest_version_id, k, v)) |
|
260 |
self.con.commit() |
|
248 | 261 |
|
249 |
def move_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False): |
|
262 |
def move_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False, src_version=None):
|
|
250 | 263 |
"""Move an object's data and metadata.""" |
251 | 264 |
|
252 |
logger.debug("move_object: %s %s %s %s %s %s %s", account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta)
|
|
253 |
self.copy_object(account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta) |
|
265 |
logger.debug("move_object: %s %s %s %s, %s %s %s %s", account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta, src_version)
|
|
266 |
self.copy_object(account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta, src_version)
|
|
254 | 267 |
self.delete_object(account, src_container, src_name) |
255 | 268 |
|
256 | 269 |
def delete_object(self, account, container, name): |
257 | 270 |
"""Delete an object.""" |
258 | 271 |
|
259 | 272 |
logger.debug("delete_object: %s %s %s", account, container, name) |
260 |
path, link, tstamp = self._get_containerinfo(account, container) |
|
273 |
path, version_id, mtime, size = self._get_objectinfo(account, container, name) |
|
274 |
self._put_version(path, 0, 1) |
|
275 |
|
|
276 |
def list_versions(self, account, container, name): |
|
277 |
"""Return a list of version (version_id, version_modified) tuples for an object.""" |
|
278 |
|
|
279 |
# This will even show deleted versions. |
|
261 | 280 |
path = os.path.join(account, container, name) |
262 |
link, tstamp = self._get_linkinfo(path)
|
|
263 |
self._del_path(path)
|
|
264 |
self._update_metadata(account, container, None, None)
|
|
281 |
sql = '''select distinct version_id, strftime('%s', tstamp) from versions where name = ?'''
|
|
282 |
c = self.con.execute(sql, (path,))
|
|
283 |
return [(str(x[0]), int(x[1])) for x in c.fetchall()]
|
|
265 | 284 |
|
266 | 285 |
def get_block(self, hash): |
267 | 286 |
"""Return a block's data.""" |
... | ... | |
299 | 318 |
dest_data = src_data[:offset] + data + src_data[offset + len(data):] |
300 | 319 |
return self.put_block(dest_data) |
301 | 320 |
|
302 |
def _get_linkinfo(self, path): |
|
303 |
c = self.con.execute('select rowid, tstamp from objects where name = ?', (path,)) |
|
321 |
def _sql_until(self, until=None): |
|
322 |
"""Return the sql to get the latest versions until the timestamp given.""" |
|
323 |
if until is None: |
|
324 |
until = int(time.time()) |
|
325 |
sql = '''select version_id, name, strftime('%s', tstamp) as tstamp, size from versions v |
|
326 |
where version_id = (select max(version_id) from versions |
|
327 |
where v.name = name and tstamp <= datetime(%s, 'unixepoch')) |
|
328 |
and hide = 0''' |
|
329 |
return sql % ('%s', until) |
|
330 |
|
|
331 |
def _get_pathstats(self, path, until=None): |
|
332 |
"""Return count and sum of size of everything under path and latest timestamp.""" |
|
333 |
|
|
334 |
sql = 'select count(version_id), total(size), max(tstamp) from (%s) where name like ?' |
|
335 |
sql = sql % self._sql_until(until) |
|
336 |
c = self.con.execute(sql, (path + '/%',)) |
|
304 | 337 |
row = c.fetchone() |
305 |
if row: |
|
306 |
return str(row[0]), str(row[1]) |
|
338 |
tstamp = row[2] if row[2] is not None else 0 |
|
339 |
return int(row[0]), int(row[1]), int(tstamp) |
|
340 |
|
|
341 |
def _get_version(self, path, version=None): |
|
342 |
if version is None: |
|
343 |
sql = '''select version_id, strftime('%s', tstamp), size, hide from versions where name = ? |
|
344 |
order by version_id desc limit 1''' |
|
345 |
c = self.con.execute(sql, (path,)) |
|
346 |
row = c.fetchone() |
|
347 |
if not row or int(row[3]): |
|
348 |
raise NameError('Object does not exist') |
|
307 | 349 |
else: |
308 |
raise NameError('Object does not exist') |
|
309 |
|
|
310 |
def _put_linkinfo(self, path): |
|
311 |
sql = 'insert into objects (name, tstamp) values (?, ?)' |
|
312 |
id = self.con.execute(sql, (path, int(time.time()))).lastrowid |
|
350 |
sql = '''select version_id, strftime('%s', tstamp), size from versions where name = ? |
|
351 |
and version_id = ?''' |
|
352 |
c = self.con.execute(sql, (path, version)) |
|
353 |
row = c.fetchone() |
|
354 |
if not row: |
|
355 |
raise IndexError('Version does not exist') |
|
356 |
return str(row[0]), int(row[1]), int(row[2]) |
|
357 |
|
|
358 |
def _put_version(self, path, size=0, hide=0): |
|
359 |
sql = 'insert into versions (name, size, hide) values (?, ?, ?)' |
|
360 |
id = self.con.execute(sql, (path, size, hide)).lastrowid |
|
313 | 361 |
self.con.commit() |
314 | 362 |
return str(id) |
315 | 363 |
|
316 |
def _get_containerinfo(self, account, container): |
|
317 |
path = os.path.join(account, container) |
|
364 |
def _copy_version(self, src_path, dest_path, copy_meta=True, copy_data=True, src_version=None): |
|
365 |
if src_version is not None: |
|
366 |
src_version_id, mtime, size = self._get_version(src_path, src_version) |
|
367 |
else: |
|
368 |
# Latest or create from scratch. |
|
369 |
try: |
|
370 |
src_version_id, mtime, size = self._get_version(src_path) |
|
371 |
except NameError: |
|
372 |
src_version_id = None |
|
373 |
size = 0 |
|
374 |
if not copy_data: |
|
375 |
size = 0 |
|
376 |
dest_version_id = self._put_version(dest_path, size) |
|
377 |
if copy_meta and src_version_id is not None: |
|
378 |
sql = 'insert into metadata select %s, key, value from metadata where version_id = ?' |
|
379 |
sql = sql % dest_version_id |
|
380 |
self.con.execute(sql, (src_version_id,)) |
|
381 |
if copy_data and src_version_id is not None: |
|
382 |
sql = 'insert into hashmaps select %s, pos, block_id from hashmaps where version_id = ?' |
|
383 |
sql = sql % dest_version_id |
|
384 |
self.con.execute(sql, (src_version_id,)) |
|
385 |
self.con.commit() |
|
386 |
return src_version_id, dest_version_id |
|
387 |
|
|
388 |
def _get_versioninfo(self, account, container, name, until=None): |
|
389 |
"""Return path, latest version, associated timestamp and size until the timestamp given.""" |
|
390 |
|
|
391 |
p = (account, container, name) |
|
318 | 392 |
try: |
319 |
link, tstamp = self._get_linkinfo(path) |
|
320 |
except NameError: |
|
393 |
p = p[:p.index(None)] |
|
394 |
except ValueError: |
|
395 |
pass |
|
396 |
path = os.path.join(*p) |
|
397 |
sql = '''select version_id, tstamp, size from (%s) where name = ?''' |
|
398 |
sql = sql % self._sql_until(until) |
|
399 |
c = self.con.execute(sql, (path,)) |
|
400 |
row = c.fetchone() |
|
401 |
if row is None: |
|
402 |
raise NameError('Path does not exist') |
|
403 |
return path, str(row[0]), int(row[1]), int(row[2]) |
|
404 |
|
|
405 |
def _get_accountinfo(self, account, until=None): |
|
406 |
try: |
|
407 |
path, version_id, mtime, size = self._get_versioninfo(account, None, None, until) |
|
408 |
return version_id, mtime |
|
409 |
except: |
|
410 |
raise NameError('Account does not exist') |
|
411 |
|
|
412 |
def _get_containerinfo(self, account, container, until=None): |
|
413 |
try: |
|
414 |
path, version_id, mtime, size = self._get_versioninfo(account, container, None, until) |
|
415 |
return path, version_id, mtime |
|
416 |
except: |
|
321 | 417 |
raise NameError('Container does not exist') |
322 |
return path, link, tstamp |
|
323 | 418 |
|
324 | 419 |
def _get_objectinfo(self, account, container, name, version=None): |
325 | 420 |
path = os.path.join(account, container, name) |
326 |
link, tstamp = self._get_linkinfo(path) |
|
327 |
if not version: # If zero or None. |
|
328 |
sql = '''select version, size from versions v, |
|
329 |
(select object_id, max(version) as m from versions |
|
330 |
where object_id = ? group by object_id) as g |
|
331 |
where v.object_id = g.object_id and v.version = g.m''' |
|
332 |
c = self.con.execute(sql, (link,)) |
|
333 |
else: |
|
334 |
sql = 'select version, size from versions where object_id = ? and version = ?' |
|
335 |
c = self.con.execute(sql, (link, version)) |
|
336 |
row = c.fetchone() |
|
337 |
if not row: |
|
338 |
raise IndexError('Version does not exist') |
|
339 |
|
|
340 |
return path, link, tstamp, int(row[0]), int(row[1]) |
|
341 |
|
|
342 |
def _get_pathstats(self, path): |
|
343 |
"""Return count and sum of size of all objects under path.""" |
|
344 |
|
|
345 |
sql = '''select count(o), total(size) from ( |
|
346 |
select v.object_id as o, v.size from versions v, |
|
347 |
(select object_id, max(version) as m from versions where object_id in |
|
348 |
(select rowid from objects where name like ?) group by object_id) as g |
|
349 |
where v.object_id = g.object_id and v.version = g.m |
|
350 |
union |
|
351 |
select rowid as o, 0 as size from objects where name like ? |
|
352 |
and rowid not in (select object_id from versions))''' |
|
353 |
c = self.con.execute(sql, (path + '/%', path + '/%')) |
|
354 |
row = c.fetchone() |
|
355 |
return int(row[0]), int(row[1]) |
|
421 |
version_id, mtime, size = self._get_version(path, version) |
|
422 |
return path, version_id, mtime, size |
|
356 | 423 |
|
357 |
def _list_objects(self, path, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[]): |
|
424 |
def _get_metadata(self, path, version): |
|
425 |
sql = 'select key, value from metadata where version_id = ?' |
|
426 |
c = self.con.execute(sql, (version,)) |
|
427 |
return dict(c.fetchall()) |
|
428 |
|
|
429 |
def _put_metadata(self, path, meta, replace=False): |
|
430 |
"""Create a new version and store metadata.""" |
|
431 |
|
|
432 |
src_version_id, dest_version_id = self._copy_version(path, path, not replace, True) |
|
433 |
for k, v in meta.iteritems(): |
|
434 |
sql = 'insert or replace into metadata (version_id, key, value) values (?, ?, ?)' |
|
435 |
self.con.execute(sql, (dest_version_id, k, v)) |
|
436 |
self.con.commit() |
|
437 |
|
|
438 |
def _list_objects(self, path, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[], until=None): |
|
358 | 439 |
cont_prefix = path + '/' |
359 | 440 |
if keys and len(keys) > 0: |
360 |
sql = '''select distinct o.name from objects o, metadata m where o.name like ? and
|
|
361 |
m.name = o.name and m.key in (%s) order by o.name'''
|
|
362 |
sql = sql % ', '.join('?' * len(keys))
|
|
441 |
sql = '''select distinct o.name, o.version_id from (%s) o, metadata m where o.name like ? and
|
|
442 |
m.version_id = o.version_id and m.key in (%s) order by o.name'''
|
|
443 |
sql = sql % (self._sql_until(until), ', '.join('?' * len(keys)))
|
|
363 | 444 |
param = (cont_prefix + prefix + '%',) + tuple(keys) |
364 | 445 |
else: |
365 |
sql = 'select name from objects where name like ? order by name' |
|
446 |
sql = 'select name, version_id from (%s) where name like ? order by name' |
|
447 |
sql = sql % self._sql_until(until) |
|
366 | 448 |
param = (cont_prefix + prefix + '%',) |
367 | 449 |
c = self.con.execute(sql, param) |
368 |
objects = [x[0][len(cont_prefix):] for x in c.fetchall()]
|
|
450 |
objects = [(x[0][len(cont_prefix):], x[1]) for x in c.fetchall()]
|
|
369 | 451 |
if delimiter: |
370 | 452 |
pseudo_objects = [] |
371 | 453 |
for x in objects: |
372 |
pseudo_name = x |
|
454 |
pseudo_name = x[0]
|
|
373 | 455 |
i = pseudo_name.find(delimiter, len(prefix)) |
374 | 456 |
if not virtual: |
375 | 457 |
# If the delimiter is not found, or the name ends |
376 | 458 |
# with the delimiter's first occurence. |
377 | 459 |
if i == -1 or len(pseudo_name) == i + len(delimiter): |
378 |
pseudo_objects.append(pseudo_name)
|
|
460 |
pseudo_objects.append(x)
|
|
379 | 461 |
else: |
380 | 462 |
# If the delimiter is found, keep up to (and including) the delimiter. |
381 | 463 |
if i != -1: |
382 | 464 |
pseudo_name = pseudo_name[:i + len(delimiter)] |
383 |
if pseudo_name not in pseudo_objects:
|
|
384 |
pseudo_objects.append(pseudo_name)
|
|
465 |
if pseudo_name not in [y[0] for y in pseudo_objects]:
|
|
466 |
pseudo_objects.append((pseudo_name, x[1]))
|
|
385 | 467 |
objects = pseudo_objects |
386 | 468 |
|
387 | 469 |
start = 0 |
388 | 470 |
if marker: |
389 | 471 |
try: |
390 |
start = objects.index(marker) + 1
|
|
472 |
start = [x[0] for x in objects].index(marker) + 1
|
|
391 | 473 |
except ValueError: |
392 | 474 |
pass |
393 | 475 |
if not limit or limit > 10000: |
394 | 476 |
limit = 10000 |
395 | 477 |
return objects[start:start + limit] |
396 | 478 |
|
397 |
def _get_metadata(self, path): |
|
398 |
sql = 'select key, value from metadata where name = ?' |
|
399 |
c = self.con.execute(sql, (path,)) |
|
400 |
return dict(c.fetchall()) |
|
401 |
|
|
402 |
def _put_metadata(self, path, meta, replace=False): |
|
403 |
if replace: |
|
404 |
sql = 'delete from metadata where name = ?' |
|
405 |
self.con.execute(sql, (path,)) |
|
406 |
for k, v in meta.iteritems(): |
|
407 |
sql = 'insert or replace into metadata (name, key, value) values (?, ?, ?)' |
|
408 |
self.con.execute(sql, (path, k, v)) |
|
409 |
self.con.commit() |
|
410 |
|
|
411 |
def _update_metadata(self, account, container, name, meta, replace=False): |
|
412 |
"""Recursively update metadata and set modification time.""" |
|
413 |
|
|
414 |
modified = {'modified': int(time.time())} |
|
415 |
if not meta: |
|
416 |
meta = {} |
|
417 |
meta.update(modified) |
|
418 |
path = (account, container, name) |
|
419 |
for x in reversed(range(3)): |
|
420 |
if not path[x]: |
|
421 |
continue |
|
422 |
self._put_metadata(os.path.join(*path[:x+1]), meta, replace) |
|
423 |
break |
|
424 |
for y in reversed(range(x)): |
|
425 |
self._put_metadata(os.path.join(*path[:y+1]), modified) |
|
426 |
|
|
427 |
def _del_uptoversion(self, link, version): |
|
428 |
sql = '''delete from hashmaps where version_id |
|
429 |
(select rowid from versions where object_id = ? and version < ?)''' |
|
430 |
self.con.execute(sql, (link, version)) |
|
431 |
self.con.execute('delete from versions where object_id = ?', (link,)) |
|
432 |
self.con.commit() |
|
433 |
|
|
434 |
def _del_path(self, path, delmeta=True): |
|
479 |
def _del_path(self, path): |
|
435 | 480 |
sql = '''delete from hashmaps where version_id in |
436 |
(select rowid from versions where object_id in |
|
437 |
(select rowid from objects where name = ?))''' |
|
481 |
(select version_id from versions where name = ?)''' |
|
482 |
self.con.execute(sql, (path,)) |
|
483 |
sql = '''delete from metadata where version_id in |
|
484 |
(select version_id from versions where name = ?)''' |
|
438 | 485 |
self.con.execute(sql, (path,)) |
439 |
sql = '''delete from versions where object_id in |
|
440 |
(select rowid from objects where name = ?)''' |
|
486 |
sql = '''delete from versions where name = ?''' |
|
441 | 487 |
self.con.execute(sql, (path,)) |
442 |
self.con.execute('delete from objects where name = ?', (path,)) |
|
443 |
if delmeta: |
|
444 |
self.con.execute('delete from metadata where name = ?', (path,)) |
|
445 | 488 |
self.con.commit() |
b/pithos/backends/tests.py | ||
---|---|---|
46 | 46 |
self.account = 'account1' |
47 | 47 |
|
48 | 48 |
def tearDown(self): |
49 |
containers = self.b.list_containers(self.account)
|
|
49 |
containers = [x[0] for x in self.b.list_containers(self.account)]
|
|
50 | 50 |
for container in containers: |
51 | 51 |
try: |
52 | 52 |
self.b.delete_container(self.account, container) |
53 | 53 |
except IndexError: |
54 | 54 |
# container not empty |
55 |
for obj in self.b.list_objects(self.account, container):
|
|
55 |
for obj in [x[0] for x in self.b.list_objects(self.account, container)]:
|
|
56 | 56 |
self.b.delete_object(self.account, container, obj) |
57 | 57 |
self.b.delete_container(self.account, container) |
58 | 58 |
|
... | ... | |
60 | 60 |
l1 = ['images', 'movies', 'documents', 'backups'] |
61 | 61 |
for item in l1: |
62 | 62 |
self.b.put_container(self.account, item) |
63 |
l2 = self.b.list_containers(self.account)
|
|
63 |
l2 = [x[0] for x in self.b.list_containers(self.account)]
|
|
64 | 64 |
l1.sort() |
65 | 65 |
self.assertEquals(l1, l2) |
66 | 66 |
|
... | ... | |
68 | 68 |
l1 = ['apples', 'bananas', 'kiwis', 'oranges', 'pears'] |
69 | 69 |
for item in l1: |
70 | 70 |
self.b.put_container(self.account, item) |
71 |
l2 = self.b.list_containers(self.account, limit=2)
|
|
71 |
l2 = [x[0] for x in self.b.list_containers(self.account, limit=2)]
|
|
72 | 72 |
self.assertEquals(len(l2), 2) |
73 | 73 |
self.assertEquals(l1[:2], l2) |
74 | 74 |
|
75 |
l2 = self.b.list_containers(self.account, limit=2, marker='bananas')
|
|
75 |
l2 = [x[0] for x in self.b.list_containers(self.account, limit=2, marker='bananas')]
|
|
76 | 76 |
self.assertTrue(len(l2) <= 2) |
77 | 77 |
self.assertEquals(l1[2:4], l2) |
78 | 78 |
|
79 |
l2 = self.b.list_containers(self.account, limit=2, marker='oranges')
|
|
79 |
l2 = [x[0] for x in self.b.list_containers(self.account, limit=2, marker='oranges')]
|
|
80 | 80 |
self.assertTrue(len(l2) <= 2) |
81 | 81 |
self.assertEquals(l1[4:], l2) |
82 | 82 |
|
... | ... | |
133 | 133 |
self.account = 'account1' |
134 | 134 |
|
135 | 135 |
def tearDown(self): |
136 |
containers = self.b.list_containers(self.account)
|
|
136 |
containers = [x[0] for x in self.b.list_containers(self.account)]
|
|
137 | 137 |
for container in containers: |
138 | 138 |
try: |
139 | 139 |
self.b.delete_container(self.account, container) |
140 | 140 |
except IndexError: # container not empty |
141 |
for obj in self.b.list_objects(self.account, container):
|
|
141 |
for obj in [x[0] for x in self.b.list_objects(self.account, container)]:
|
|
142 | 142 |
self.b.delete_object(self.account, container, obj) |
143 | 143 |
self.b.delete_container(self.account, container) |
144 | 144 |
|
... | ... | |
147 | 147 |
|
148 | 148 |
def test_list_objects(self): |
149 | 149 |
self.b.put_container(self.account, 'container1') |
150 |
objects = self.b.list_objects(self.account, 'container1')
|
|
150 |
objects = [x[0] for x in self.b.list_objects(self.account, 'container1')]
|
|
151 | 151 |
self.assertEquals(len([]), len(objects)) |
152 | 152 |
l = [ |
153 | 153 |
{'name':'kate_beckinsale.jpg'}, |
... | ... | |
160 | 160 |
] |
161 | 161 |
for item in l: |
162 | 162 |
self.b.update_object_hashmap(self.account, 'container1', item['name'], 0, []) |
163 |
objects = self.b.list_objects(self.account, 'container1')
|
|
163 |
objects = [x[0] for x in self.b.list_objects(self.account, 'container1')]
|
|
164 | 164 |
self.assertEquals(len(l), len(objects)) |
165 | 165 |
|
166 | 166 |
def test_list_objects_with_limit_marker(self): |
... | ... | |
168 | 168 |
l = ['gala', 'grannysmith', 'honeycrisp', 'jonagold', 'reddelicious'] |
169 | 169 |
for item in l: |
170 | 170 |
self.b.update_object_hashmap(self.account, 'container1', item, 0, []) |
171 |
objects = self.b.list_objects(self.account, 'container1', limit=2)
|
|
171 |
objects = [x[0] for x in self.b.list_objects(self.account, 'container1', limit=2)]
|
|
172 | 172 |
self.assertEquals(l[:2], objects) |
173 | 173 |
|
174 |
objects = self.b.list_objects(self.account, 'container1', limit=2, marker='grannysmith')
|
|
174 |
objects = [x[0] for x in self.b.list_objects(self.account, 'container1', limit=2, marker='grannysmith')]
|
|
175 | 175 |
self.assertEquals(l[2:4], objects) |
176 | 176 |
|
177 |
objects = self.b.list_objects(self.account, 'container1', limit=2, marker='jonagold')
|
|
177 |
objects = [x[0] for x in self.b.list_objects(self.account, 'container1', limit=2, marker='jonagold')]
|
|
178 | 178 |
self.assertEquals(l[4:], objects) |
179 | 179 |
|
180 | 180 |
def test_list_pseudo_hierarchical_folders(self): |
... | ... | |
190 | 190 |
for item in l: |
191 | 191 |
self.b.update_object_hashmap(self.account, 'container1', item, 0, []) |
192 | 192 |
|
193 |
objects = self.b.list_objects(self.account, 'container1', prefix='photos/', delimiter='/')
|
|
193 |
objects = [x[0] for x in self.b.list_objects(self.account, 'container1', prefix='photos/', delimiter='/')]
|
|
194 | 194 |
self.assertEquals(['photos/animals/', 'photos/me.jpg', 'photos/plants/'], objects) |
195 | 195 |
|
196 |
objects = self.b.list_objects(self.account, 'container1', prefix='photos/animals/', delimiter='/')
|
|
196 |
objects = [x[0] for x in self.b.list_objects(self.account, 'container1', prefix='photos/animals/', delimiter='/')]
|
|
197 | 197 |
self.assertEquals(['photos/animals/cats/', 'photos/animals/dogs/'], objects) |
198 | 198 |
|
199 | 199 |
self.b.put_container(self.account, 'container2') |
200 | 200 |
l = ['photos/photo1', 'photos/photo2', 'movieobject', 'videos', 'videos/movieobj4'] |
201 | 201 |
for item in l: |
202 | 202 |
self.b.update_object_hashmap(self.account, 'container2', item, 0, []) |
203 |
objects = self.b.list_objects(self.account, 'container2', delimiter='/')
|
|
203 |
objects = [x[0] for x in self.b.list_objects(self.account, 'container2', delimiter='/')]
|
|
204 | 204 |
self.assertEquals(['movieobject', 'photos/', 'videos', 'videos/'], objects) |
205 | 205 |
|
206 | 206 |
def test_put_container(self): |
207 | 207 |
cname = 'container1' |
208 | 208 |
self.b.put_container(self.account, cname) |
209 |
self.assertTrue(cname in self.b.list_containers(self.account))
|
|
209 |
self.assertTrue(cname in [x[0] for x in self.b.list_containers(self.account)])
|
|
210 | 210 |
|
211 | 211 |
def test_put_container_twice(self): |
212 | 212 |
cname = 'container1' |
... | ... | |
217 | 217 |
cname = 'container1' |
218 | 218 |
self.b.put_container(self.account, cname) |
219 | 219 |
self.b.delete_container(self.account, cname) |
220 |
self.assertTrue(cname not in self.b.list_containers(self.account))
|
|
220 |
self.assertTrue(cname not in [x[0] for x in self.b.list_containers(self.account)])
|
|
221 | 221 |
|
222 | 222 |
def test_delete_non_exisitng_container(self): |
223 | 223 |
cname = 'container1' |
... | ... | |
248 | 248 |
self.account = 'account1' |
249 | 249 |
|
250 | 250 |
def tearDown(self): |
251 |
containers = self.b.list_containers(self.account)
|
|
251 |
containers = [x[0] for x in self.b.list_containers(self.account)]
|
|
252 | 252 |
for container in containers: |
253 | 253 |
try: |
254 | 254 |
self.b.delete_container(self.account, container) |
255 | 255 |
except IndexError: # container not empty |
256 |
for obj in self.b.list_objects(self.account, container):
|
|
256 |
for obj in [x[0] for x in self.b.list_objects(self.account, container)]:
|
|
257 | 257 |
self.b.delete_object(self.account, container, obj) |
258 | 258 |
self.b.delete_container(self.account, container) |
259 | 259 |
|
... | ... | |
328 | 328 |
self.b.put_container(self.account, dest_cname) |
329 | 329 |
self.b.update_object_meta(self.account, src_cname, src_obj, {'tag':'sfsfssf'}) |
330 | 330 |
self.b.copy_object(self.account, src_cname, src_obj, dest_cname, dest_obj) |
331 |
self.assertTrue(dest_obj in self.b.list_objects(self.account, |
|
332 |
dest_cname, |
|
333 |
prefix='photos/personal/', |
|
334 |
delimiter='/'))
|
|
331 |
self.assertTrue(dest_obj in [x[0] for x in self.b.list_objects(self.account,
|
|
332 |
dest_cname,
|
|
333 |
prefix='photos/personal/',
|
|
334 |
delimiter='/')])
|
|
335 | 335 |
# TODO: test metadata changes |
336 | 336 |
meta_tag = self.b.get_object_meta(self.account, dest_cname, dest_obj)['tag'] |
337 | 337 |
self.assertEquals(meta_tag, unicode('sfsfssf')) |
... | ... | |
347 | 347 |
self.b.put_container(self.account, cname) |
348 | 348 |
name = 'kate_beckinsale.jpg' |
349 | 349 |
self.b.update_object_hashmap(self.account, cname, name, 0, []) |
350 |
self.assertTrue(name in self.b.list_objects(self.account, cname))
|
|
350 |
self.assertTrue(name in [x[0] for x in self.b.list_objects(self.account, cname)])
|
|
351 | 351 |
|
352 | 352 |
self.b.delete_object(self.account, cname, name) |
353 |
self.assertTrue(name not in self.b.list_objects(self.account, cname))
|
|
353 |
self.assertTrue(name not in [x[0] for x in self.b.list_objects(self.account, cname)])
|
|
354 | 354 |
self.assertRaises(NameError, self.b.delete_object, self.account, cname, name) |
355 | 355 |
|
356 | 356 |
def test_get_non_existing_object_meta(self): |
Also available in: Unified diff