Revision 78348987
| b/contrib/snf-pithos-tools/pithos/tools/lib/client.py | ||
|---|---|---|
| 78 | 78 |
self.debug = debug |
| 79 | 79 |
self.token = token |
| 80 | 80 |
|
| 81 |
def _req(self, method, path, body=None, headers={}, format='text', params={}):
|
|
| 81 |
def _req(self, method, path, body=None, headers=None, format='text', |
|
| 82 |
params=None): |
|
| 83 |
headers = headers or {}
|
|
| 84 |
params = params or {}
|
|
| 85 |
|
|
| 82 | 86 |
p = urlparse(self.url) |
| 83 | 87 |
if p.scheme == 'http': |
| 84 | 88 |
conn = HTTPConnection(p.netloc) |
| ... | ... | |
| 109 | 113 |
return _handle_response(resp, self.verbose, self.debug) |
| 110 | 114 |
|
| 111 | 115 |
def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None, |
| 112 |
blocksize=1024, params={}):
|
|
| 116 |
blocksize=1024, params=None):
|
|
| 113 | 117 |
"""perfomrs a chunked request""" |
| 118 |
params = params or {}
|
|
| 114 | 119 |
p = urlparse(self.url) |
| 115 | 120 |
if p.scheme == 'http': |
| 116 | 121 |
conn = HTTPConnection(p.netloc) |
| ... | ... | |
| 154 | 159 |
resp = conn.getresponse() |
| 155 | 160 |
return _handle_response(resp, self.verbose, self.debug) |
| 156 | 161 |
|
| 157 |
def delete(self, path, format='text', params={}):
|
|
| 162 |
def delete(self, path, format='text', params=None): |
|
| 163 |
params = params or {}
|
|
| 158 | 164 |
return self._req('DELETE', path, format=format, params=params)
|
| 159 | 165 |
|
| 160 |
def get(self, path, format='text', headers={}, params={}):
|
|
| 166 |
def get(self, path, format='text', headers=None, params=None): |
|
| 167 |
headers = headers or {}
|
|
| 168 |
params = params or {}
|
|
| 161 | 169 |
return self._req('GET', path, headers=headers, format=format,
|
| 162 | 170 |
params=params) |
| 163 | 171 |
|
| 164 |
def head(self, path, format='text', params={}):
|
|
| 172 |
def head(self, path, format='text', params=None): |
|
| 173 |
params = params or {}
|
|
| 165 | 174 |
return self._req('HEAD', path, format=format, params=params)
|
| 166 | 175 |
|
| 167 |
def post(self, path, body=None, format='text', headers=None, params={}):
|
|
| 176 |
def post(self, path, body=None, format='text', headers=None, params=None): |
|
| 177 |
params = params or {}
|
|
| 168 | 178 |
return self._req('POST', path, body, headers=headers, format=format,
|
| 169 | 179 |
params=params) |
| 170 | 180 |
|
| 171 |
def put(self, path, body=None, format='text', headers=None, params={}):
|
|
| 181 |
def put(self, path, body=None, format='text', headers=None, params=None): |
|
| 182 |
params = params or {}
|
|
| 172 | 183 |
return self._req('PUT', path, body, headers=headers, format=format,
|
| 173 | 184 |
params=params) |
| 174 | 185 |
|
| 175 |
def _list(self, path, format='text', params={}, **headers):
|
|
| 186 |
def _list(self, path, format='text', params=None, **headers): |
|
| 187 |
params = params or {}
|
|
| 176 | 188 |
status, headers, data = self.get(path, format=format, headers=headers, |
| 177 | 189 |
params=params) |
| 178 | 190 |
if format == 'json': |
| ... | ... | |
| 183 | 195 |
data = data.split('\n')[:-1] if data else ''
|
| 184 | 196 |
return data |
| 185 | 197 |
|
| 186 |
def _get_metadata(self, path, prefix=None, params={}):
|
|
| 198 |
def _get_metadata(self, path, prefix=None, params=None): |
|
| 199 |
params = params or {}
|
|
| 187 | 200 |
status, headers, data = self.head(path, params=params) |
| 188 | 201 |
prefixlen = len(prefix) if prefix else 0 |
| 189 | 202 |
meta = {}
|
| ... | ... | |
| 235 | 248 |
headers[k] = v |
| 236 | 249 |
return self.post(path, headers=headers) |
| 237 | 250 |
|
| 238 |
def _delete_metadata(self, path, entity, meta=[]):
|
|
| 251 |
def _delete_metadata(self, path, entity, meta=None):
|
|
| 239 | 252 |
"""delete previously set metadata""" |
| 253 |
meta = meta or [] |
|
| 240 | 254 |
ex_meta = self.retrieve_account_metadata(restricted=True) |
| 241 | 255 |
headers = {}
|
| 242 | 256 |
prefix = 'x-%s-meta-' % entity |
| ... | ... | |
| 248 | 262 |
# Storage Account Services |
| 249 | 263 |
|
| 250 | 264 |
def list_containers(self, format='text', limit=None, |
| 251 |
marker=None, params={}, account=None, **headers):
|
|
| 265 |
marker=None, params=None, account=None, **headers):
|
|
| 252 | 266 |
"""lists containers""" |
| 267 |
params = params or {}
|
|
| 253 | 268 |
account = account or self.account |
| 254 | 269 |
path = '/%s' % account |
| 255 | 270 |
params.update({'limit': limit, 'marker': marker})
|
| ... | ... | |
| 268 | 283 |
path = '/%s' % account |
| 269 | 284 |
return self._update_metadata(path, 'account', **meta) |
| 270 | 285 |
|
| 271 |
def delete_account_metadata(self, meta=[], account=None):
|
|
| 286 |
def delete_account_metadata(self, meta=None, account=None):
|
|
| 272 | 287 |
"""deletes the account metadata""" |
| 288 |
meta = meta or [] |
|
| 273 | 289 |
account = account or self.account |
| 274 | 290 |
path = '/%s' % account |
| 275 | 291 |
return self._delete_metadata(path, 'account', meta) |
| ... | ... | |
| 287 | 303 |
|
| 288 | 304 |
def list_objects(self, container, format='text', |
| 289 | 305 |
limit=None, marker=None, prefix=None, delimiter=None, |
| 290 |
path=None, include_trashed=False, params={}, account=None,
|
|
| 306 |
path=None, include_trashed=False, params=None, account=None,
|
|
| 291 | 307 |
**headers): |
| 292 | 308 |
"""returns a list with the container objects""" |
| 309 |
params = params or {}
|
|
| 293 | 310 |
account = account or self.account |
| 294 | 311 |
params.update({'limit': limit, 'marker': marker, 'prefix': prefix,
|
| 295 | 312 |
'delimiter': delimiter, 'path': path}) |
| ... | ... | |
| 300 | 317 |
l = self._filter_trashed(l) |
| 301 | 318 |
return l |
| 302 | 319 |
|
| 303 |
def create_container(self, container, account=None, meta={}, **headers):
|
|
| 320 |
def create_container(self, container, account=None, meta=None, **headers):
|
|
| 304 | 321 |
"""creates a container""" |
| 322 |
meta = meta or {}
|
|
| 305 | 323 |
account = account or self.account |
| 306 | 324 |
if not headers: |
| 307 | 325 |
headers = {}
|
| ... | ... | |
| 315 | 333 |
raise Fault(data, int(status)) |
| 316 | 334 |
return True |
| 317 | 335 |
|
| 318 |
def delete_container(self, container, params={}, account=None):
|
|
| 336 |
def delete_container(self, container, params=None, account=None):
|
|
| 319 | 337 |
"""deletes a container""" |
| 338 |
params = params or {}
|
|
| 320 | 339 |
account = account or self.account |
| 321 | 340 |
return self.delete('/%s/%s' % (account, container), params=params)
|
| 322 | 341 |
|
| ... | ... | |
| 334 | 353 |
return self._update_metadata('/%s/%s' % (account, container),
|
| 335 | 354 |
'container', **meta) |
| 336 | 355 |
|
| 337 |
def delete_container_metadata(self, container, meta=[], account=None):
|
|
| 356 |
def delete_container_metadata(self, container, meta=None, account=None):
|
|
| 338 | 357 |
"""deletes the container metadata""" |
| 358 |
meta = meta or [] |
|
| 339 | 359 |
account = account or self.account |
| 340 | 360 |
path = '/%s/%s' % (account, container) |
| 341 | 361 |
return self._delete_metadata(path, 'container', meta) |
| 342 | 362 |
|
| 343 | 363 |
# Storage Object Services |
| 344 | 364 |
|
| 345 |
def request_object(self, container, object, format='text', params={},
|
|
| 365 |
def request_object(self, container, object, format='text', params=None,
|
|
| 346 | 366 |
account=None, **headers): |
| 347 | 367 |
"""returns tuple containing the status, headers and data response for an object request""" |
| 368 |
params = params or {}
|
|
| 348 | 369 |
account = account or self.account |
| 349 | 370 |
path = '/%s/%s/%s' % (account, container, object) |
| 350 | 371 |
status, headers, data = self.get(path, format, headers, params) |
| 351 | 372 |
return status, headers, data |
| 352 | 373 |
|
| 353 |
def retrieve_object(self, container, object, format='text', params={},
|
|
| 374 |
def retrieve_object(self, container, object, format='text', params=None,
|
|
| 354 | 375 |
account=None, **headers): |
| 355 | 376 |
"""returns an object's data""" |
| 377 |
params = params or {}
|
|
| 356 | 378 |
account = account or self.account |
| 357 | 379 |
t = self.request_object(container, object, format, params, account, |
| 358 | 380 |
**headers) |
| ... | ... | |
| 364 | 386 |
return data |
| 365 | 387 |
|
| 366 | 388 |
def retrieve_object_hashmap( |
| 367 |
self, container, object, format='json', params={},
|
|
| 389 |
self, container, object, format='json', params=None,
|
|
| 368 | 390 |
account=None, **headers): |
| 369 | 391 |
"""returns the hashmap representing object's data""" |
| 392 |
params = params or {}
|
|
| 370 | 393 |
if not params: |
| 371 | 394 |
params = {}
|
| 372 | 395 |
params.update({'hashmap': None})
|
| ... | ... | |
| 382 | 405 |
container, object, account=account, |
| 383 | 406 |
**h) |
| 384 | 407 |
|
| 385 |
def create_object(self, container, object, f=stdin, format='text', meta={},
|
|
| 386 |
params={}, etag=None, content_type=None, content_encoding=None,
|
|
| 408 |
def create_object(self, container, object, f=stdin, format='text', meta=None,
|
|
| 409 |
params=None, etag=None, content_type=None, content_encoding=None,
|
|
| 387 | 410 |
content_disposition=None, account=None, **headers): |
| 388 | 411 |
"""creates a zero-length object""" |
| 412 |
meta = meta or {}
|
|
| 413 |
params = params or {}
|
|
| 389 | 414 |
account = account or self.account |
| 390 | 415 |
path = '/%s/%s/%s' % (account, container, object) |
| 391 | 416 |
for k, v in headers.items(): |
| ... | ... | |
| 403 | 428 |
data = f.read() if f else None |
| 404 | 429 |
return self.put(path, data, format, headers=headers, params=params) |
| 405 | 430 |
|
| 406 |
def create_zero_length_object(self, container, object, meta={}, etag=None,
|
|
| 431 |
def create_zero_length_object(self, container, object, meta=None, etag=None,
|
|
| 407 | 432 |
content_type=None, content_encoding=None, |
| 408 | 433 |
content_disposition=None, account=None, |
| 409 | 434 |
**headers): |
| 435 |
meta = meta or {}
|
|
| 410 | 436 |
account = account or self.account |
| 411 | 437 |
args = locals().copy() |
| 412 | 438 |
for elem in ['self', 'container', 'headers', 'account']: |
| ... | ... | |
| 415 | 441 |
return self.create_object(container, account=account, f=None, **args) |
| 416 | 442 |
|
| 417 | 443 |
def update_object(self, container, object, f=stdin, |
| 418 |
offset=None, meta={}, params={}, content_length=None,
|
|
| 444 |
offset=None, meta=None, params=None, content_length=None,
|
|
| 419 | 445 |
content_type=None, content_encoding=None, |
| 420 | 446 |
content_disposition=None, account=None, **headers): |
| 447 |
meta = meta or {}
|
|
| 448 |
params = params or {}
|
|
| 421 | 449 |
account = account or self.account |
| 422 | 450 |
path = '/%s/%s/%s' % (account, container, object) |
| 423 | 451 |
for k, v in headers.items(): |
| ... | ... | |
| 442 | 470 |
return self.post(path, data, headers=headers, params=params) |
| 443 | 471 |
|
| 444 | 472 |
def update_object_using_chunks(self, container, object, f=stdin, |
| 445 |
blocksize=1024, offset=None, meta={},
|
|
| 446 |
params={}, content_type=None, content_encoding=None,
|
|
| 473 |
blocksize=1024, offset=None, meta=None,
|
|
| 474 |
params=None, content_type=None, content_encoding=None,
|
|
| 447 | 475 |
content_disposition=None, account=None, **headers): |
| 448 | 476 |
"""updates an object (incremental upload)""" |
| 477 |
params = params or {}
|
|
| 478 |
meta = meta or {}
|
|
| 449 | 479 |
account = account or self.account |
| 450 | 480 |
path = '/%s/%s/%s' % (account, container, object) |
| 451 | 481 |
headers = headers if not headers else {}
|
| ... | ... | |
| 466 | 496 |
blocksize=blocksize, params=params) |
| 467 | 497 |
|
| 468 | 498 |
def _change_obj_location(self, src_container, src_object, dst_container, |
| 469 |
dst_object, remove=False, meta={}, account=None,
|
|
| 499 |
dst_object, remove=False, meta=None, account=None,
|
|
| 470 | 500 |
content_type=None, delimiter=None, **headers): |
| 501 |
meta = meta or {}
|
|
| 471 | 502 |
account = account or self.account |
| 472 | 503 |
path = '/%s/%s/%s' % (account, dst_container, dst_object) |
| 473 | 504 |
headers = {} if not headers else headers
|
| ... | ... | |
| 488 | 519 |
return self.put(path, headers=headers, params=params) |
| 489 | 520 |
|
| 490 | 521 |
def copy_object(self, src_container, src_object, dst_container, dst_object, |
| 491 |
meta={}, account=None, content_type=None, delimiter=None, **headers):
|
|
| 522 |
meta=None, account=None, content_type=None, delimiter=None, **headers):
|
|
| 492 | 523 |
"""copies an object""" |
| 524 |
meta = meta or {}
|
|
| 493 | 525 |
account = account or self.account |
| 494 | 526 |
return self._change_obj_location(src_container, src_object, |
| 495 | 527 |
dst_container, dst_object, account=account, |
| ... | ... | |
| 497 | 529 |
content_type=content_type, delimiter=delimiter, **headers) |
| 498 | 530 |
|
| 499 | 531 |
def move_object(self, src_container, src_object, dst_container, |
| 500 |
dst_object, meta={}, account=None,
|
|
| 532 |
dst_object, meta=None, account=None,
|
|
| 501 | 533 |
content_type=None, **headers): |
| 502 | 534 |
"""moves an object""" |
| 535 |
meta = meta or {}
|
|
| 503 | 536 |
account = account or self.account |
| 504 | 537 |
return self._change_obj_location(src_container, src_object, |
| 505 | 538 |
dst_container, dst_object, |
| ... | ... | |
| 507 | 540 |
meta=meta, content_type=content_type, |
| 508 | 541 |
**headers) |
| 509 | 542 |
|
| 510 |
def delete_object(self, container, object, params={}, account=None):
|
|
| 543 |
def delete_object(self, container, object, params=None, account=None):
|
|
| 511 | 544 |
"""deletes an object""" |
| 545 |
params = params or {}
|
|
| 512 | 546 |
account = account or self.account |
| 513 | 547 |
return self.delete('/%s/%s/%s' % (account, container, object),
|
| 514 | 548 |
params=params) |
| ... | ... | |
| 533 | 567 |
path = '/%s/%s/%s' % (account, container, object) |
| 534 | 568 |
return self._update_metadata(path, 'object', **meta) |
| 535 | 569 |
|
| 536 |
def delete_object_metadata(self, container, object, meta=[], account=None):
|
|
| 570 |
def delete_object_metadata(self, container, object, meta=None, account=None):
|
|
| 537 | 571 |
""" |
| 538 | 572 |
deletes object's metadata |
| 539 | 573 |
""" |
| 574 |
meta = meta or [] |
|
| 540 | 575 |
account = account or self.account |
| 541 | 576 |
path = '/%s/%s' % (account, container, object) |
| 542 | 577 |
return self._delete_metadata(path, 'object', meta) |
| ... | ... | |
| 557 | 592 |
headers[k] = v |
| 558 | 593 |
return self.post(path, headers=headers, params=params) |
| 559 | 594 |
|
| 560 |
def _delete_metadata(self, path, entity, meta=[]):
|
|
| 595 |
def _delete_metadata(self, path, entity, meta=None):
|
|
| 561 | 596 |
""" |
| 562 | 597 |
delete previously set metadata |
| 563 | 598 |
""" |
| 599 |
meta = meta or [] |
|
| 564 | 600 |
params = {'update': None}
|
| 565 | 601 |
headers = {}
|
| 566 | 602 |
prefix = 'x-%s-meta-' % entity |
| ... | ... | |
| 620 | 656 |
groups[key] = val |
| 621 | 657 |
return groups |
| 622 | 658 |
|
| 623 |
def unset_account_groups(self, groups=[], account=None):
|
|
| 659 |
def unset_account_groups(self, groups=None, account=None):
|
|
| 624 | 660 |
"""delete account groups""" |
| 661 |
groups = groups or [] |
|
| 625 | 662 |
account = account or self.account |
| 626 | 663 |
path = '/%s' % account |
| 627 | 664 |
headers = {}
|
| ... | ... | |
| 646 | 683 |
return self.post(path, headers=headers) |
| 647 | 684 |
|
| 648 | 685 |
# Storage Container Services |
| 649 |
def create_container(self, container, account=None, meta={}, policies={}):
|
|
| 686 |
def create_container(self, container, account=None, meta=None, policies=None):
|
|
| 650 | 687 |
"""creates a container""" |
| 688 |
meta = meta or {}
|
|
| 689 |
policies = policies or {}
|
|
| 651 | 690 |
args = {}
|
| 652 | 691 |
for k, v in policies.items(): |
| 653 | 692 |
args['X-Container-Policy-%s' % k.capitalize()] = v |
| ... | ... | |
| 655 | 694 |
|
| 656 | 695 |
def list_objects(self, container, format='text', |
| 657 | 696 |
limit=None, marker=None, prefix=None, delimiter=None, |
| 658 |
path=None, shared=False, include_trashed=False, params={},
|
|
| 697 |
path=None, shared=False, include_trashed=False, params=None,
|
|
| 659 | 698 |
if_modified_since=None, if_unmodified_since=None, meta='', |
| 660 | 699 |
until=None, account=None, public=False): |
| 661 | 700 |
"""returns a list with the container objects""" |
| 701 |
params = params or {}
|
|
| 662 | 702 |
account = account or self.account |
| 663 | 703 |
params = {'until': until, 'meta': meta}
|
| 664 | 704 |
if shared: |
| ... | ... | |
| 711 | 751 |
|
| 712 | 752 |
# Storage Object Services |
| 713 | 753 |
|
| 714 |
def retrieve_object(self, container, object, params={}, format='text',
|
|
| 754 |
def retrieve_object(self, container, object, params=None, format='text',
|
|
| 715 | 755 |
range=None, if_range=None, |
| 716 | 756 |
if_match=None, if_none_match=None, |
| 717 | 757 |
if_modified_since=None, if_unmodified_since=None, |
| 718 | 758 |
account=None, **headers): |
| 719 | 759 |
"""returns an object""" |
| 760 |
params = params or {}
|
|
| 720 | 761 |
account = account or self.account |
| 721 | 762 |
headers = {}
|
| 722 | 763 |
l = ['range', 'if_range', 'if_match', 'if_none_match', |
| ... | ... | |
| 759 | 800 |
format='json', **args) |
| 760 | 801 |
|
| 761 | 802 |
def create_zero_length_object(self, container, object, |
| 762 |
meta={}, etag=None, content_type=None,
|
|
| 803 |
meta=None, etag=None, content_type=None,
|
|
| 763 | 804 |
content_encoding=None, |
| 764 | 805 |
content_disposition=None, |
| 765 | 806 |
x_object_manifest=None, x_object_sharing=None, |
| 766 | 807 |
x_object_public=None, account=None): |
| 767 | 808 |
"""createas a zero length object""" |
| 809 |
meta = meta or {}
|
|
| 768 | 810 |
account = account or self.account |
| 769 | 811 |
args = locals().copy() |
| 770 | 812 |
for elem in ['self', 'container', 'object']: |
| ... | ... | |
| 773 | 815 |
**args) |
| 774 | 816 |
|
| 775 | 817 |
def create_folder(self, container, name, |
| 776 |
meta={}, etag=None,
|
|
| 818 |
meta=None, etag=None,
|
|
| 777 | 819 |
content_encoding=None, |
| 778 | 820 |
content_disposition=None, |
| 779 | 821 |
x_object_manifest=None, x_object_sharing=None, |
| 780 | 822 |
x_object_public=None, account=None): |
| 823 |
meta = meta or {}
|
|
| 781 | 824 |
args = locals().copy() |
| 782 | 825 |
for elem in ['self', 'container', 'name']: |
| 783 | 826 |
args.pop(elem) |
| ... | ... | |
| 785 | 828 |
return self.create_zero_length_object(container, name, **args) |
| 786 | 829 |
|
| 787 | 830 |
def create_object(self, container, object, f=stdin, format='text', |
| 788 |
meta={}, params={}, etag=None, content_type=None,
|
|
| 831 |
meta=None, params=None, etag=None, content_type=None,
|
|
| 789 | 832 |
content_encoding=None, content_disposition=None, |
| 790 | 833 |
x_object_manifest=None, x_object_sharing=None, |
| 791 | 834 |
x_object_public=None, account=None): |
| 792 | 835 |
"""creates an object""" |
| 836 |
meta = meta or {}
|
|
| 837 |
params = params or {}
|
|
| 793 | 838 |
account = account or self.account |
| 794 | 839 |
args = locals().copy() |
| 795 | 840 |
for elem in ['self', 'container', 'object']: |
| ... | ... | |
| 799 | 844 |
return OOS_Client.create_object(self, container, object, **args) |
| 800 | 845 |
|
| 801 | 846 |
def create_object_using_chunks(self, container, object, |
| 802 |
f=stdin, blocksize=1024, meta={}, etag=None,
|
|
| 847 |
f=stdin, blocksize=1024, meta=None, etag=None,
|
|
| 803 | 848 |
content_type=None, content_encoding=None, |
| 804 | 849 |
content_disposition=None, |
| 805 | 850 |
x_object_sharing=None, x_object_manifest=None, |
| 806 | 851 |
x_object_public=None, account=None): |
| 807 | 852 |
"""creates an object (incremental upload)""" |
| 853 |
meta = meta or {}
|
|
| 808 | 854 |
account = account or self.account |
| 809 | 855 |
path = '/%s/%s/%s' % (account, container, object) |
| 810 | 856 |
headers = {}
|
| ... | ... | |
| 822 | 868 |
return self._chunked_transfer(path, 'PUT', f, headers=headers, |
| 823 | 869 |
blocksize=blocksize) |
| 824 | 870 |
|
| 825 |
def create_object_by_hashmap(self, container, object, hashmap={},
|
|
| 826 |
meta={}, etag=None, content_encoding=None,
|
|
| 871 |
def create_object_by_hashmap(self, container, object, hashmap=None,
|
|
| 872 |
meta=None, etag=None, content_encoding=None,
|
|
| 827 | 873 |
content_disposition=None, content_type=None, |
| 828 | 874 |
x_object_sharing=None, x_object_manifest=None, |
| 829 | 875 |
x_object_public=None, account=None): |
| 830 | 876 |
"""creates an object by uploading hashes representing data instead of data""" |
| 877 |
meta = meta or {}
|
|
| 878 |
hashmap = hashmap or {}
|
|
| 831 | 879 |
account = account or self.account |
| 832 | 880 |
args = locals().copy() |
| 833 | 881 |
for elem in ['self', 'container', 'object', 'hashmap']: |
| ... | ... | |
| 850 | 898 |
**headers) |
| 851 | 899 |
|
| 852 | 900 |
def update_object(self, container, object, f=stdin, |
| 853 |
offset=None, meta={}, replace=False, content_length=None,
|
|
| 901 |
offset=None, meta=None, replace=False, content_length=None,
|
|
| 854 | 902 |
content_type=None, content_range=None, |
| 855 | 903 |
content_encoding=None, content_disposition=None, |
| 856 | 904 |
x_object_bytes=None, x_object_manifest=None, |
| 857 | 905 |
x_object_sharing=None, x_object_public=None, |
| 858 | 906 |
x_source_object=None, account=None): |
| 859 | 907 |
"""updates an object""" |
| 908 |
meta = meta or {}
|
|
| 860 | 909 |
account = account or self.account |
| 861 | 910 |
args = locals().copy() |
| 862 | 911 |
for elem in ['self', 'container', 'object', 'replace']: |
| ... | ... | |
| 866 | 915 |
return OOS_Client.update_object(self, container, object, **args) |
| 867 | 916 |
|
| 868 | 917 |
def update_object_using_chunks(self, container, object, f=stdin, |
| 869 |
blocksize=1024, offset=None, meta={},
|
|
| 918 |
blocksize=1024, offset=None, meta=None,
|
|
| 870 | 919 |
replace=False, content_type=None, content_encoding=None, |
| 871 | 920 |
content_disposition=None, x_object_bytes=None, |
| 872 | 921 |
x_object_manifest=None, x_object_sharing=None, |
| 873 | 922 |
x_object_public=None, account=None): |
| 874 | 923 |
"""updates an object (incremental upload)""" |
| 924 |
meta = meta or {}
|
|
| 875 | 925 |
account = account or self.account |
| 876 | 926 |
args = locals().copy() |
| 877 | 927 |
for elem in ['self', 'container', 'object', 'replace']: |
| ... | ... | |
| 881 | 931 |
return OOS_Client.update_object_using_chunks(self, container, object, **args) |
| 882 | 932 |
|
| 883 | 933 |
def update_from_other_source(self, container, object, source, |
| 884 |
offset=None, meta={}, content_range=None,
|
|
| 934 |
offset=None, meta=None, content_range=None,
|
|
| 885 | 935 |
content_encoding=None, content_disposition=None, |
| 886 | 936 |
x_object_bytes=None, x_object_manifest=None, |
| 887 | 937 |
x_object_sharing=None, x_object_public=None, account=None): |
| 888 | 938 |
"""updates an object""" |
| 939 |
meta = meta or {}
|
|
| 889 | 940 |
account = account or self.account |
| 890 | 941 |
args = locals().copy() |
| 891 | 942 |
for elem in ['self', 'container', 'object', 'source']: |
| ... | ... | |
| 933 | 984 |
return self.post(path, headers=headers, params=params) |
| 934 | 985 |
|
| 935 | 986 |
def copy_object(self, src_container, src_object, dst_container, dst_object, |
| 936 |
meta={}, public=False, version=None, account=None,
|
|
| 987 |
meta=None, public=False, version=None, account=None,
|
|
| 937 | 988 |
content_type=None, delimiter=None): |
| 938 | 989 |
"""copies an object""" |
| 990 |
meta = meta or {}
|
|
| 939 | 991 |
account = account or self.account |
| 940 | 992 |
headers = {}
|
| 941 | 993 |
headers['x_object_public'] = public |
| ... | ... | |
| 948 | 1000 |
**headers) |
| 949 | 1001 |
|
| 950 | 1002 |
def move_object(self, src_container, src_object, dst_container, |
| 951 |
dst_object, meta={}, public=False,
|
|
| 1003 |
dst_object, meta=None, public=False,
|
|
| 952 | 1004 |
account=None, content_type=None, delimiter=None): |
| 953 | 1005 |
"""moves an object""" |
| 1006 |
meta = meta or {}
|
|
| 954 | 1007 |
headers = {}
|
| 955 | 1008 |
headers['x_object_public'] = public |
| 956 | 1009 |
return OOS_Client.move_object(self, src_container, src_object, |
| ... | ... | |
| 977 | 1030 |
self.update_object(container, object, f=None, x_object_sharing=sharing) |
| 978 | 1031 |
|
| 979 | 1032 |
|
| 980 |
def _prepare_path(path, format='text', params={}):
|
|
| 1033 |
def _prepare_path(path, format='text', params=None): |
|
| 1034 |
params = params or {}
|
|
| 981 | 1035 |
full_path = '%s?format=%s' % (quote(path), format) |
| 982 | 1036 |
|
| 983 | 1037 |
for k, v in params.items(): |
| b/contrib/snf-pithos-tools/pithos/tools/test.py | ||
|---|---|---|
| 2182 | 2182 |
self.upload_random_data(self.container, self.object+'a/') |
| 2183 | 2183 |
self.dir_content_types = ('application/directory', 'application/folder')
|
| 2184 | 2184 |
|
| 2185 |
def assert_read(self, authorized=[], any=False, depth=0): |
|
| 2185 |
def assert_read(self, authorized=None, any=False, depth=0): |
|
| 2186 |
authorized = authorized or [] |
|
| 2186 | 2187 |
for token, account in OTHER_ACCOUNTS.items(): |
| 2187 | 2188 |
cl = Pithos_Client(get_url(), token, account) |
| 2188 | 2189 |
if account in authorized or any: |
| ... | ... | |
| 2213 | 2214 |
self.assert_raises_fault(403, cl.retrieve_object_metadata, |
| 2214 | 2215 |
self.container, o, account=get_user()) |
| 2215 | 2216 |
|
| 2216 |
def assert_write(self, authorized=[], any=False): |
|
| 2217 |
def assert_write(self, authorized=None, any=False): |
|
| 2218 |
authorized = authorized or [] |
|
| 2217 | 2219 |
o_data = self.client.retrieve_object(self.container, self.object) |
| 2218 | 2220 |
for token, account in OTHER_ACCOUNTS.items(): |
| 2219 | 2221 |
cl = Pithos_Client(get_url(), token, account) |
| b/snf-pithos-app/pithos/api/swiss_army/__init__.py | ||
|---|---|---|
| 303 | 303 |
return self.backend._lookup_account(account, create=True) |
| 304 | 304 |
|
| 305 | 305 |
def create_update_object(self, account, container, name, content_type, |
| 306 |
data, meta={}, permissions={}, request_user=None):
|
|
| 306 |
data, meta=None, permissions=None, request_user=None): |
|
| 307 |
meta = meta or {}
|
|
| 308 |
permissions = permissions or {}
|
|
| 307 | 309 |
md5 = hashlib.md5() |
| 308 | 310 |
size = 0 |
| 309 | 311 |
hashmap = [] |
| b/snf-pithos-app/pithos/api/swiss_army/tests.py | ||
|---|---|---|
| 59 | 59 |
self.utils._delete_account(i) |
| 60 | 60 |
self.utils.cleanup() |
| 61 | 61 |
|
| 62 |
def _verify_object(self, account, container, object, expected={},
|
|
| 62 |
def _verify_object(self, account, container, object, expected=None,
|
|
| 63 | 63 |
strict=True): |
| 64 |
expected = expected or {}
|
|
| 64 | 65 |
self._verify_object_metadata(account, container, object, |
| 65 | 66 |
expected.get('meta'))
|
| 66 | 67 |
self._verify_object_history(account, container, object, |
| b/snf-pithos-backend/pithos/backends/base.py | ||
|---|---|---|
| 176 | 176 |
""" |
| 177 | 177 |
return |
| 178 | 178 |
|
| 179 |
def put_account(self, user, account, policy={}):
|
|
| 179 |
def put_account(self, user, account, policy=None):
|
|
| 180 | 180 |
"""Create a new account with the given name. |
| 181 | 181 |
|
| 182 | 182 |
Raises: |
| ... | ... | |
| 289 | 289 |
""" |
| 290 | 290 |
return |
| 291 | 291 |
|
| 292 |
def put_container(self, user, account, container, policy={}, delimiter=None):
|
|
| 292 |
def put_container(self, user, account, container, policy=None, delimiter=None):
|
|
| 293 | 293 |
"""Create a new container with the given name. |
| 294 | 294 |
|
| 295 | 295 |
Parameters: |
| ... | ... | |
| 316 | 316 |
""" |
| 317 | 317 |
return |
| 318 | 318 |
|
| 319 |
def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None, public=False):
|
|
| 319 |
def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=None, shared=False, until=None, size_range=None, public=False):
|
|
| 320 | 320 |
"""Return a list of object (name, version_id) tuples existing under a container. |
| 321 | 321 |
|
| 322 | 322 |
Parameters: |
| ... | ... | |
| 355 | 355 |
""" |
| 356 | 356 |
return [] |
| 357 | 357 |
|
| 358 |
def list_object_meta(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None):
|
|
| 358 |
def list_object_meta(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=None, shared=False, until=None, size_range=None):
|
|
| 359 | 359 |
"""Return a list of object metadata dicts existing under a container. |
| 360 | 360 |
|
| 361 | 361 |
Same parameters with list_objects. Returned dicts have no user-defined |
| ... | ... | |
| 497 | 497 |
""" |
| 498 | 498 |
return 0, [] |
| 499 | 499 |
|
| 500 |
def update_object_hashmap(self, user, account, container, name, size, type, hashmap, checksum, domain, meta={}, replace_meta=False, permissions=None):
|
|
| 500 |
def update_object_hashmap(self, user, account, container, name, size, type, hashmap, checksum, domain, meta=None, replace_meta=False, permissions=None):
|
|
| 501 | 501 |
"""Create/update an object with the specified size and partial hashes and return the new version. |
| 502 | 502 |
|
| 503 | 503 |
Parameters: |
| ... | ... | |
| 524 | 524 |
"""Update an object's checksum.""" |
| 525 | 525 |
return |
| 526 | 526 |
|
| 527 |
def copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, src_version=None, delimiter=None):
|
|
| 527 |
def copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta=None, replace_meta=False, permissions=None, src_version=None, delimiter=None):
|
|
| 528 | 528 |
"""Copy an object's data and metadata and return the new version. |
| 529 | 529 |
|
| 530 | 530 |
Parameters: |
| ... | ... | |
| 553 | 553 |
""" |
| 554 | 554 |
return '' |
| 555 | 555 |
|
| 556 |
def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, delimiter=None):
|
|
| 556 |
def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta=None, replace_meta=False, permissions=None, delimiter=None):
|
|
| 557 | 557 |
"""Move an object's data and metadata and return the new version. |
| 558 | 558 |
|
| 559 | 559 |
Parameters: |
| b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/node.py | ||
|---|---|---|
| 868 | 868 |
'key': k, 'value': v} |
| 869 | 869 |
self.conn.execute(s, values).close() |
| 870 | 870 |
|
| 871 |
def latest_attribute_keys(self, parent, domain, before=inf, except_cluster=0, pathq=[]):
|
|
| 871 |
def latest_attribute_keys(self, parent, domain, before=inf, except_cluster=0, pathq=None):
|
|
| 872 | 872 |
"""Return a list with all keys pairs defined |
| 873 | 873 |
for all latest versions under parent that |
| 874 | 874 |
do not belong to the cluster. |
| 875 | 875 |
""" |
| 876 | 876 |
|
| 877 |
pathq = pathq or [] |
|
| 878 |
|
|
| 877 | 879 |
# TODO: Use another table to store before=inf results. |
| 878 | 880 |
a = self.attributes.alias('a')
|
| 879 | 881 |
v = self.versions.alias('v')
|
| b/snf-pithos-backend/pithos/backends/lib/sqlite/node.py | ||
|---|---|---|
| 804 | 804 |
args += [before] |
| 805 | 805 |
return q % where_cond, args |
| 806 | 806 |
|
| 807 |
def latest_attribute_keys(self, parent, domain, before=inf, except_cluster=0, pathq=[]):
|
|
| 807 |
def latest_attribute_keys(self, parent, domain, before=inf, except_cluster=0, pathq=None):
|
|
| 808 | 808 |
"""Return a list with all keys pairs defined |
| 809 | 809 |
for all latest versions under parent that |
| 810 | 810 |
do not belong to the cluster. |
| 811 | 811 |
""" |
| 812 | 812 |
|
| 813 |
pathq = pathq or [] |
|
| 814 |
|
|
| 813 | 815 |
# TODO: Use another table to store before=inf results. |
| 814 | 816 |
q = ("select distinct a.key "
|
| 815 | 817 |
"from attributes a, versions v, nodes n " |
| b/snf-pithos-backend/pithos/backends/modular.py | ||
|---|---|---|
| 351 | 351 |
self._put_policy(node, policy, replace) |
| 352 | 352 |
|
| 353 | 353 |
@backend_method |
| 354 |
def put_account(self, user, account, policy={}):
|
|
| 354 |
def put_account(self, user, account, policy=None):
|
|
| 355 | 355 |
"""Create a new account with the given name.""" |
| 356 | 356 |
|
| 357 | 357 |
logger.debug("put_account: %s %s %s", user, account, policy)
|
| 358 |
policy = policy or {}
|
|
| 358 | 359 |
if user != account: |
| 359 | 360 |
raise NotAllowedError |
| 360 | 361 |
node = self.node.node_lookup(account) |
| ... | ... | |
| 502 | 503 |
self._put_policy(node, policy, replace) |
| 503 | 504 |
|
| 504 | 505 |
@backend_method |
| 505 |
def put_container(self, user, account, container, policy={}):
|
|
| 506 |
def put_container(self, user, account, container, policy=None):
|
|
| 506 | 507 |
"""Create a new container with the given name.""" |
| 507 | 508 |
|
| 508 | 509 |
logger.debug( |
| 509 | 510 |
"put_container: %s %s %s %s", user, account, container, policy) |
| 511 |
policy = policy or {}
|
|
| 510 | 512 |
if user != account: |
| 511 | 513 |
raise NotAllowedError |
| 512 | 514 |
try: |
| ... | ... | |
| 669 | 671 |
return allowed |
| 670 | 672 |
|
| 671 | 673 |
@backend_method |
| 672 |
def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None, public=False):
|
|
| 674 |
def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=None, shared=False, until=None, size_range=None, public=False):
|
|
| 673 | 675 |
"""Return a list of object (name, version_id) tuples existing under a container.""" |
| 674 | 676 |
|
| 675 | 677 |
logger.debug("list_objects: %s %s %s %s %s %s %s %s %s %s %s %s %s %s", user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, public)
|
| 678 |
keys = keys or [] |
|
| 676 | 679 |
return self._list_objects(user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, False, public) |
| 677 | 680 |
|
| 678 | 681 |
@backend_method |
| 679 |
def list_object_meta(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None, public=False):
|
|
| 682 |
def list_object_meta(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=None, shared=False, until=None, size_range=None, public=False):
|
|
| 680 | 683 |
"""Return a list of object metadata dicts existing under a container.""" |
| 681 | 684 |
|
| 682 | 685 |
logger.debug("list_object_meta: %s %s %s %s %s %s %s %s %s %s %s %s %s %s", user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, public)
|
| 686 |
keys = keys or [] |
|
| 683 | 687 |
props = self._list_objects(user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, True, public) |
| 684 | 688 |
objects = [] |
| 685 | 689 |
for p in props: |
| ... | ... | |
| 893 | 897 |
return dest_version_id |
| 894 | 898 |
|
| 895 | 899 |
@backend_method |
| 896 |
def update_object_hashmap(self, user, account, container, name, size, type, hashmap, checksum, domain, meta={}, replace_meta=False, permissions=None):
|
|
| 900 |
def update_object_hashmap(self, user, account, container, name, size, type, hashmap, checksum, domain, meta=None, replace_meta=False, permissions=None):
|
|
| 897 | 901 |
"""Create/update an object with the specified size and partial hashes.""" |
| 898 | 902 |
|
| 899 | 903 |
logger.debug("update_object_hashmap: %s %s %s %s %s %s %s %s", user,
|
| 900 | 904 |
account, container, name, size, type, hashmap, checksum) |
| 905 |
meta = meta or {}
|
|
| 901 | 906 |
if size == 0: # No such thing as an empty hashmap. |
| 902 | 907 |
hashmap = [self.put_block('')]
|
| 903 | 908 |
map = HashMap(self.block_size, self.hash_algorithm) |
| ... | ... | |
| 929 | 934 |
self.node.version_put_property( |
| 930 | 935 |
x[self.SERIAL], 'checksum', checksum) |
| 931 | 936 |
|
| 932 |
def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, dest_domain=None, dest_meta={}, replace_meta=False, permissions=None, src_version=None, is_move=False, delimiter=None):
|
|
| 937 |
def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, dest_domain=None, dest_meta=None, replace_meta=False, permissions=None, src_version=None, is_move=False, delimiter=None): |
|
| 938 |
dest_meta = dest_meta or {}
|
|
| 933 | 939 |
dest_version_ids = [] |
| 934 | 940 |
self._can_read(user, src_account, src_container, src_name) |
| 935 | 941 |
path, node = self._lookup_object(src_account, src_container, src_name) |
| ... | ... | |
| 969 | 975 |
return dest_version_ids[0] if len(dest_version_ids) == 1 else dest_version_ids |
| 970 | 976 |
|
| 971 | 977 |
@backend_method |
| 972 |
def copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, src_version=None, delimiter=None):
|
|
| 978 |
def copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta=None, replace_meta=False, permissions=None, src_version=None, delimiter=None):
|
|
| 973 | 979 |
"""Copy an object's data and metadata.""" |
| 974 | 980 |
|
| 975 | 981 |
logger.debug("copy_object: %s %s %s %s %s %s %s %s %s %s %s %s %s %s", user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, src_version, delimiter)
|
| 982 |
meta = meta or {}
|
|
| 976 | 983 |
dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, src_version, False, delimiter) |
| 977 | 984 |
return dest_version_id |
| 978 | 985 |
|
| 979 | 986 |
@backend_method |
| 980 |
def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, delimiter=None):
|
|
| 987 |
def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta=None, replace_meta=False, permissions=None, delimiter=None):
|
|
| 981 | 988 |
"""Move an object's data and metadata.""" |
| 982 | 989 |
|
| 983 | 990 |
logger.debug("move_object: %s %s %s %s %s %s %s %s %s %s %s %s %s", user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, delimiter)
|
| 991 |
meta = meta or {}
|
|
| 984 | 992 |
if user != src_account: |
| 985 | 993 |
raise NotAllowedError |
| 986 | 994 |
dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, None, True, delimiter) |
| ... | ... | |
| 1281 | 1289 |
limit = 10000 |
| 1282 | 1290 |
return start, limit |
| 1283 | 1291 |
|
| 1284 |
def _list_object_properties(self, parent, path, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], until=None, size_range=None, allowed=[], all_props=False): |
|
| 1292 |
def _list_object_properties(self, parent, path, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=None, until=None, size_range=None, allowed=None, all_props=False): |
|
| 1293 |
keys = keys or [] |
|
| 1294 |
allowed = allowed or [] |
|
| 1285 | 1295 |
cont_prefix = path + '/' |
| 1286 | 1296 |
prefix = cont_prefix + prefix |
| 1287 | 1297 |
start = cont_prefix + marker if marker else None |
| ... | ... | |
| 1297 | 1307 |
|
| 1298 | 1308 |
# Reporting functions. |
| 1299 | 1309 |
|
| 1300 |
def _report_size_change(self, user, account, size, details={}):
|
|
| 1310 |
def _report_size_change(self, user, account, size, details=None): |
|
| 1311 |
details = details or {}
|
|
| 1312 |
|
|
| 1301 | 1313 |
if size == 0: |
| 1302 | 1314 |
return |
| 1303 | 1315 |
|
| ... | ... | |
| 1328 | 1340 |
else: |
| 1329 | 1341 |
self.serials.append(serial) |
| 1330 | 1342 |
|
| 1331 |
def _report_object_change(self, user, account, path, details={}):
|
|
| 1343 |
def _report_object_change(self, user, account, path, details=None): |
|
| 1344 |
details = details or {}
|
|
| 1332 | 1345 |
details.update({'user': user})
|
| 1333 | 1346 |
logger.debug("_report_object_change: %s %s %s %s", user,
|
| 1334 | 1347 |
account, path, details) |
| 1335 | 1348 |
self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('object',),
|
| 1336 | 1349 |
account, QUEUE_INSTANCE_ID, 'object', path, details)) |
| 1337 | 1350 |
|
| 1338 |
def _report_sharing_change(self, user, account, path, details={}):
|
|
| 1351 |
def _report_sharing_change(self, user, account, path, details=None):
|
|
| 1339 | 1352 |
logger.debug("_report_permissions_change: %s %s %s %s",
|
| 1340 | 1353 |
user, account, path, details) |
| 1354 |
details = details or {}
|
|
| 1341 | 1355 |
details.update({'user': user})
|
| 1342 | 1356 |
self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('sharing',),
|
| 1343 | 1357 |
account, QUEUE_INSTANCE_ID, 'sharing', path, details)) |
Also available in: Unified diff