Revision 8cb45c13
| b/docs/source/devguide.rst | ||
|---|---|---|
| 25 | 25 |
========================= ================================ |
| 26 | 26 |
Revision Description |
| 27 | 27 |
========================= ================================ |
| 28 |
0.3 (June 10, 2011) Allow for publicly available objects via ``https://hostname/public``. |
|
| 28 |
0.3 (June 14, 2011) Large object support with ``X-Object-Manifest``. |
|
| 29 |
\ Allow for publicly available objects via ``https://hostname/public``. |
|
| 29 | 30 |
\ Support time-variant account/container listings. |
| 30 | 31 |
\ Add source version when duplicating with PUT/COPY/MOVE. |
| 31 | 32 |
\ Request version in object HEAD/GET requests (list versions with GET). |
| ... | ... | |
| 689 | 690 |
* Time-variant account/container listings via the ``until`` parameter. |
| 690 | 691 |
* Object versions - parameter ``version`` in HEAD/GET (list versions with GET), ``X-Object-Version-*`` meta in replies, ``X-Source-Version`` in PUT/COPY/MOVE. |
| 691 | 692 |
* Publicly accessible objects via ``https://hostname/public``. Control with ``X-Object-Public``. |
| 693 |
* Large object support with ``X-Object-Manifest``. |
|
| 692 | 694 |
|
| 693 | 695 |
Clarifications/suggestions: |
| 694 | 696 |
|
| ... | ... | |
| 700 | 702 |
* Container/object lists use a ``200`` return code if the reply is of type json/xml. The reply will include an empty json/xml. |
| 701 | 703 |
* In headers, dates are formatted according to RFC 1123. In extended information listings, dates are formatted according to ISO 8601. |
| 702 | 704 |
* The ``Last-Modified`` header value always reflects the actual latest change timestamp, regardless of time control parameters and version requests. Time precondition checks with ``If-Modified-Since`` and ``If-Unmodified-Since`` headers are applied to this value. |
| 703 |
* While ``X-Object-Manifest`` can be set and unset, large object support is not yet implemented (**TBD**).
|
|
| 705 |
* A ``HEAD`` or ``GET`` for an ``X-Object-Manifest`` object, will include modified ``Content-Length`` and ``ETag`` headers, according to the characteristics of the objects under the specified prefix. The ``Etag`` will be the MD5 hash of the corresponding ETags concatenated. In extended container listings there is no metadata processing.
|
|
| 704 | 706 |
|
| 705 | 707 |
The Pithos Client |
| 706 | 708 |
----------------- |
| b/pithos/api/functions.py | ||
|---|---|---|
| 44 | 44 |
LengthRequired, PreconditionFailed, RangeNotSatisfiable, UnprocessableEntity) |
| 45 | 45 |
from pithos.api.util import (format_meta_key, printable_meta_dict, get_account_meta, |
| 46 | 46 |
put_account_meta, get_container_meta, put_container_meta, get_object_meta, put_object_meta, |
| 47 |
validate_modification_preconditions, validate_matching_preconditions, split_container_object_string, |
|
| 48 |
copy_or_move_object, get_int_parameter, get_content_length, get_content_range, raw_input_socket, |
|
| 49 |
socket_read_iterator, object_data_response, hashmap_hash, api_method) |
|
| 47 |
update_manifest_meta, validate_modification_preconditions, validate_matching_preconditions, |
|
| 48 |
split_container_object_string, copy_or_move_object, get_int_parameter, get_content_length, |
|
| 49 |
get_content_range, raw_input_socket, socket_read_iterator, object_data_response, |
|
| 50 |
hashmap_hash, api_method) |
|
| 50 | 51 |
from pithos.backends import backend |
| 51 | 52 |
|
| 52 | 53 |
|
| ... | ... | |
| 374 | 375 |
except IndexError: |
| 375 | 376 |
raise ItemNotFound('Version does not exist')
|
| 376 | 377 |
|
| 378 |
update_manifest_meta(request, v_account, meta) |
|
| 379 |
|
|
| 377 | 380 |
response = HttpResponse(status=204) |
| 378 | 381 |
put_object_meta(response, meta) |
| 379 | 382 |
return response |
| ... | ... | |
| 397 | 400 |
except IndexError: |
| 398 | 401 |
raise ItemNotFound('Version does not exist')
|
| 399 | 402 |
|
| 403 |
update_manifest_meta(request, v_account, meta) |
|
| 404 |
|
|
| 400 | 405 |
# Evaluate conditions. |
| 401 | 406 |
validate_modification_preconditions(request, meta) |
| 402 | 407 |
try: |
| ... | ... | |
| 423 | 428 |
response['Content-Length'] = len(data) |
| 424 | 429 |
return response |
| 425 | 430 |
|
| 426 |
try: |
|
| 427 |
size, hashmap = backend.get_object_hashmap(request.user, v_account, v_container, v_object, version) |
|
| 428 |
except NameError: |
|
| 429 |
raise ItemNotFound('Object does not exist')
|
|
| 430 |
except IndexError: |
|
| 431 |
raise ItemNotFound('Version does not exist')
|
|
| 431 |
sizes = [] |
|
| 432 |
hashmaps = [] |
|
| 433 |
if 'X-Object-Manifest' in meta: |
|
| 434 |
try: |
|
| 435 |
src_container, src_name = split_container_object_string(meta['X-Object-Manifest']) |
|
| 436 |
objects = backend.list_objects(request.user, v_account, src_container, prefix=src_name, virtual=False) |
|
| 437 |
except ValueError: |
|
| 438 |
raise BadRequest('Invalid X-Object-Manifest header')
|
|
| 439 |
except NameError: |
|
| 440 |
raise ItemNotFound('Container does not exist')
|
|
| 441 |
|
|
| 442 |
try: |
|
| 443 |
for x in objects: |
|
| 444 |
s, h = backend.get_object_hashmap(request.user, v_account, src_container, x[0], x[1]) |
|
| 445 |
sizes.append(s) |
|
| 446 |
hashmaps.append(h) |
|
| 447 |
except NameError: |
|
| 448 |
raise ItemNotFound('Object does not exist')
|
|
| 449 |
except IndexError: |
|
| 450 |
raise ItemNotFound('Version does not exist')
|
|
| 451 |
else: |
|
| 452 |
try: |
|
| 453 |
s, h = backend.get_object_hashmap(request.user, v_account, v_container, v_object, version) |
|
| 454 |
sizes.append(s) |
|
| 455 |
hashmaps.append(h) |
|
| 456 |
except NameError: |
|
| 457 |
raise ItemNotFound('Object does not exist')
|
|
| 458 |
except IndexError: |
|
| 459 |
raise ItemNotFound('Version does not exist')
|
|
| 432 | 460 |
|
| 433 | 461 |
# Reply with the hashmap. |
| 434 | 462 |
if request.serialization != 'text': |
| 463 |
size = sum(sizes) |
|
| 464 |
hashmap = sum(hashmaps, []) |
|
| 435 | 465 |
d = {'block_size': backend.block_size, 'block_hash': backend.hash_algorithm, 'bytes': size, 'hashes': hashmap}
|
| 436 | 466 |
if request.serialization == 'xml': |
| 437 | 467 |
d['object'] = v_object |
| ... | ... | |
| 444 | 474 |
response['Content-Length'] = len(data) |
| 445 | 475 |
return response |
| 446 | 476 |
|
| 447 |
return object_data_response(request, size, hashmap, meta)
|
|
| 477 |
return object_data_response(request, sizes, hashmaps, meta)
|
|
| 448 | 478 |
|
| 449 | 479 |
@api_method('PUT')
|
| 450 | 480 |
def object_write(request, v_account, v_container, v_object): |
| b/pithos/api/util.py | ||
|---|---|---|
| 153 | 153 |
if k in meta: |
| 154 | 154 |
response[k] = meta[k] |
| 155 | 155 |
|
| 156 |
def update_manifest_meta(request, v_account, meta): |
|
| 157 |
"""Update metadata if the object has an X-Object-Manifest.""" |
|
| 158 |
|
|
| 159 |
if 'X-Object-Manifest' in meta: |
|
| 160 |
hash = '' |
|
| 161 |
bytes = 0 |
|
| 162 |
try: |
|
| 163 |
src_container, src_name = split_container_object_string(meta['X-Object-Manifest']) |
|
| 164 |
objects = backend.list_objects(request.user, v_account, src_container, prefix=src_name, virtual=False) |
|
| 165 |
for x in objects: |
|
| 166 |
src_meta = backend.get_object_meta(request.user, v_account, src_container, x[0], x[1]) |
|
| 167 |
hash += src_meta['hash'] |
|
| 168 |
bytes += src_meta['bytes'] |
|
| 169 |
except: |
|
| 170 |
# Ignore errors. |
|
| 171 |
return |
|
| 172 |
meta['bytes'] = bytes |
|
| 173 |
md5 = hashlib.md5() |
|
| 174 |
md5.update(hash) |
|
| 175 |
meta['hash'] = md5.hexdigest().lower() |
|
| 176 |
|
|
| 156 | 177 |
def validate_modification_preconditions(request, meta): |
| 157 | 178 |
"""Check that the modified timestamp conforms with the preconditions set.""" |
| 158 | 179 |
|
| ... | ... | |
| 188 | 209 |
raise NotModified('Resource Etag matches')
|
| 189 | 210 |
|
| 190 | 211 |
def split_container_object_string(s): |
| 191 |
parts = s.split('/')
|
|
| 192 |
if len(parts) < 3 or parts[0] != '':
|
|
| 212 |
pos = s.find('/')
|
|
| 213 |
if pos == -1:
|
|
| 193 | 214 |
raise ValueError |
| 194 |
return parts[1], '/'.join(parts[2:])
|
|
| 215 |
return s[:pos], s[(pos + 1):]
|
|
| 195 | 216 |
|
| 196 | 217 |
def copy_or_move_object(request, v_account, src_container, src_name, dest_container, dest_name, move=False): |
| 197 | 218 |
"""Copy or move an object.""" |
| ... | ... | |
| 391 | 412 |
Read from the object using the offset and length provided in each entry of the range list. |
| 392 | 413 |
""" |
| 393 | 414 |
|
| 394 |
def __init__(self, ranges, size, hashmap, boundary):
|
|
| 415 |
def __init__(self, ranges, sizes, hashmaps, boundary):
|
|
| 395 | 416 |
self.ranges = ranges |
| 396 |
self.size = size
|
|
| 397 |
self.hashmap = hashmap
|
|
| 417 |
self.sizes = sizes
|
|
| 418 |
self.hashmaps = hashmaps
|
|
| 398 | 419 |
self.boundary = boundary |
| 420 |
self.size = sum(self.sizes) |
|
| 399 | 421 |
|
| 400 |
self.block_index = -1 |
|
| 422 |
self.file_index = 0 |
|
| 423 |
self.block_index = 0 |
|
| 424 |
self.block_hash = -1 |
|
| 401 | 425 |
self.block = '' |
| 402 | 426 |
|
| 403 | 427 |
self.range_index = -1 |
| ... | ... | |
| 408 | 432 |
|
| 409 | 433 |
def part_iterator(self): |
| 410 | 434 |
if self.length > 0: |
| 411 |
# Get the block for the current offset. |
|
| 412 |
bi = int(self.offset / backend.block_size) |
|
| 413 |
if self.block_index != bi: |
|
| 435 |
# Get the file for the current offset. |
|
| 436 |
file_size = self.sizes[self.file_index] |
|
| 437 |
while self.offset >= file_size: |
|
| 438 |
self.offset -= file_size |
|
| 439 |
self.file_index += 1 |
|
| 440 |
file_size = self.sizes[self.file_index] |
|
| 441 |
|
|
| 442 |
# Get the block for the current position. |
|
| 443 |
self.block_index = int(self.offset / backend.block_size) |
|
| 444 |
if self.block_hash != self.hashmaps[self.file_index][self.block_index]: |
|
| 445 |
self.block_hash = self.hashmaps[self.file_index][self.block_index] |
|
| 414 | 446 |
try: |
| 415 |
self.block = backend.get_block(self.hashmap[bi])
|
|
| 447 |
self.block = backend.get_block(self.block_hash)
|
|
| 416 | 448 |
except NameError: |
| 417 | 449 |
raise ItemNotFound('Block does not exist')
|
| 418 |
self.block_index = bi |
|
| 450 |
|
|
| 419 | 451 |
# Get the data from the block. |
| 420 | 452 |
bo = self.offset % backend.block_size |
| 421 |
bl = min(self.length, backend.block_size - bo)
|
|
| 453 |
bl = min(self.length, len(self.block) - bo)
|
|
| 422 | 454 |
data = self.block[bo:bo + bl] |
| 423 | 455 |
self.offset += bl |
| 424 | 456 |
self.length -= bl |
| ... | ... | |
| 441 | 473 |
if self.range_index < len(self.ranges): |
| 442 | 474 |
# Part header. |
| 443 | 475 |
self.offset, self.length = self.ranges[self.range_index] |
| 476 |
self.file_index = 0 |
|
| 444 | 477 |
if self.range_index > 0: |
| 445 | 478 |
out.append('')
|
| 446 | 479 |
out.append('--' + self.boundary)
|
| ... | ... | |
| 456 | 489 |
out.append('')
|
| 457 | 490 |
return '\r\n'.join(out) |
| 458 | 491 |
|
| 459 |
def object_data_response(request, size, hashmap, meta, public=False):
|
|
| 492 |
def object_data_response(request, sizes, hashmaps, meta, public=False):
|
|
| 460 | 493 |
"""Get the HttpResponse object for replying with the object's data.""" |
| 461 | 494 |
|
| 462 | 495 |
# Range handling. |
| 496 |
size = sum(sizes) |
|
| 463 | 497 |
ranges = get_range(request, size) |
| 464 | 498 |
if ranges is None: |
| 465 | 499 |
ranges = [(0, size)] |
| ... | ... | |
| 477 | 511 |
boundary = uuid.uuid4().hex |
| 478 | 512 |
else: |
| 479 | 513 |
boundary = '' |
| 480 |
wrapper = ObjectWrapper(ranges, size, hashmap, boundary)
|
|
| 514 |
wrapper = ObjectWrapper(ranges, sizes, hashmaps, boundary)
|
|
| 481 | 515 |
response = HttpResponse(wrapper, status=ret) |
| 482 | 516 |
put_object_meta(response, meta, public) |
| 483 | 517 |
if ret == 206: |
| b/pithos/public/functions.py | ||
|---|---|---|
| 36 | 36 |
from django.http import HttpResponse |
| 37 | 37 |
|
| 38 | 38 |
from pithos.api.faults import (Fault, BadRequest, ItemNotFound) |
| 39 |
from pithos.api.util import (put_object_meta, validate_modification_preconditions, |
|
| 40 |
validate_matching_preconditions, object_data_response, api_method) |
|
| 39 |
from pithos.api.util import (put_object_meta, update_manifest_meta, |
|
| 40 |
validate_modification_preconditions, validate_matching_preconditions, |
|
| 41 |
object_data_response, api_method) |
|
| 41 | 42 |
from pithos.backends import backend |
| 42 | 43 |
|
| 43 | 44 |
|
| ... | ... | |
| 69 | 70 |
|
| 70 | 71 |
if 'X-Object-Public' not in meta: |
| 71 | 72 |
raise ItemNotFound('Object does not exist')
|
| 73 |
update_manifest_meta(request, v_account, meta) |
|
| 72 | 74 |
|
| 73 | 75 |
response = HttpResponse(status=204) |
| 74 | 76 |
put_object_meta(response, meta, True) |
| ... | ... | |
| 92 | 94 |
|
| 93 | 95 |
if 'X-Object-Public' not in meta: |
| 94 | 96 |
raise ItemNotFound('Object does not exist')
|
| 97 |
update_manifest_meta(request, v_account, meta) |
|
| 95 | 98 |
|
| 96 | 99 |
# Evaluate conditions. |
| 97 | 100 |
validate_modification_preconditions(request, meta) |
| ... | ... | |
| 102 | 105 |
response['ETag'] = meta['hash'] |
| 103 | 106 |
return response |
| 104 | 107 |
|
| 105 |
try: |
|
| 106 |
size, hashmap = backend.get_object_hashmap(request.user, v_account, v_container, v_object) |
|
| 107 |
except NameError: |
|
| 108 |
raise ItemNotFound('Object does not exist')
|
|
| 108 |
sizes = [] |
|
| 109 |
hashmaps = [] |
|
| 110 |
if 'X-Object-Manifest' in meta: |
|
| 111 |
try: |
|
| 112 |
src_container, src_name = split_container_object_string(meta['X-Object-Manifest']) |
|
| 113 |
objects = backend.list_objects(request.user, v_account, src_container, prefix=src_name, virtual=False) |
|
| 114 |
except ValueError: |
|
| 115 |
raise ItemNotFound('Object does not exist')
|
|
| 116 |
except NameError: |
|
| 117 |
raise ItemNotFound('Object does not exist')
|
|
| 118 |
|
|
| 119 |
try: |
|
| 120 |
for x in objects: |
|
| 121 |
s, h = backend.get_object_hashmap(request.user, v_account, src_container, x[0], x[1]) |
|
| 122 |
sizes.append(s) |
|
| 123 |
hashmaps.append(h) |
|
| 124 |
except NameError: |
|
| 125 |
raise ItemNotFound('Object does not exist')
|
|
| 126 |
else: |
|
| 127 |
try: |
|
| 128 |
s, h = backend.get_object_hashmap(request.user, v_account, v_container, v_object, version) |
|
| 129 |
sizes.append(s) |
|
| 130 |
hashmaps.append(h) |
|
| 131 |
except NameError: |
|
| 132 |
raise ItemNotFound('Object does not exist')
|
|
| 109 | 133 |
|
| 110 |
return object_data_response(request, size, hashmap, meta, True)
|
|
| 134 |
return object_data_response(request, sizes, hashmaps, meta, True)
|
|
| 111 | 135 |
|
| 112 | 136 |
@api_method() |
| 113 | 137 |
def method_not_allowed(request): |
Also available in: Unified diff