root / pithos / lib / client.py @ e7b51248
History | View | Annotate | Download (34.1 kB)
| 1 |
# Copyright 2011 GRNET S.A. All rights reserved.
|
|---|---|
| 2 |
#
|
| 3 |
# Redistribution and use in source and binary forms, with or
|
| 4 |
# without modification, are permitted provided that the following
|
| 5 |
# conditions are met:
|
| 6 |
#
|
| 7 |
# 1. Redistributions of source code must retain the above
|
| 8 |
# copyright notice, this list of conditions and the following
|
| 9 |
# disclaimer.
|
| 10 |
#
|
| 11 |
# 2. Redistributions in binary form must reproduce the above
|
| 12 |
# copyright notice, this list of conditions and the following
|
| 13 |
# disclaimer in the documentation and/or other materials
|
| 14 |
# provided with the distribution.
|
| 15 |
#
|
| 16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
| 17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
| 18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
| 19 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
| 20 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
| 21 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
| 22 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
| 23 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
| 24 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
| 25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
| 26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
| 27 |
# POSSIBILITY OF SUCH DAMAGE.
|
| 28 |
#
|
| 29 |
# The views and conclusions contained in the software and
|
| 30 |
# documentation are those of the authors and should not be
|
| 31 |
# interpreted as representing official policies, either expressed
|
| 32 |
# or implied, of GRNET S.A.
|
| 33 |
|
| 34 |
from httplib import HTTPConnection, HTTP |
| 35 |
from sys import stdin |
| 36 |
from xml.dom import minidom |
| 37 |
|
| 38 |
import json |
| 39 |
import types |
| 40 |
import socket |
| 41 |
import urllib |
| 42 |
import pithos.api.faults |
| 43 |
import datetime |
| 44 |
|
| 45 |
ERROR_CODES = {304:'Not Modified',
|
| 46 |
400:'Bad Request', |
| 47 |
401:'Unauthorized', |
| 48 |
404:'Not Found', |
| 49 |
409:'Conflict', |
| 50 |
411:'Length Required', |
| 51 |
412:'Precondition Failed', |
| 52 |
416:'Range Not Satisfiable', |
| 53 |
422:'Unprocessable Entity', |
| 54 |
503:'Service Unavailable'} |
| 55 |
|
| 56 |
class Fault(Exception): |
| 57 |
def __init__(self, data='', status=None): |
| 58 |
if data == '' and status in ERROR_CODES.keys(): |
| 59 |
data = ERROR_CODES[status] |
| 60 |
Exception.__init__(self, data) |
| 61 |
self.data = data
|
| 62 |
self.status = status
|
| 63 |
|
| 64 |
class Client(object): |
| 65 |
def __init__(self, host, token, account, api='v1', verbose=False, debug=False): |
| 66 |
"""`host` can also include a port, e.g '127.0.0.1:8000'."""
|
| 67 |
|
| 68 |
self.host = host
|
| 69 |
self.account = account
|
| 70 |
self.api = api
|
| 71 |
self.verbose = verbose or debug |
| 72 |
self.debug = debug
|
| 73 |
self.token = token
|
| 74 |
|
| 75 |
def _req(self, method, path, body=None, headers={}, format='text', |
| 76 |
params={}, top_level_req=False):
|
| 77 |
#path = urllib.quote(path)
|
| 78 |
account = '' if top_level_req else self.account |
| 79 |
full_path = '/%s/%s%s?format=%s' % (self.api, account, path, format) |
| 80 |
|
| 81 |
for k,v in params.items(): |
| 82 |
if v:
|
| 83 |
full_path = '%s&%s=%s' %(full_path, k, v)
|
| 84 |
else:
|
| 85 |
full_path = '%s&%s' %(full_path, k)
|
| 86 |
conn = HTTPConnection(self.host)
|
| 87 |
|
| 88 |
#encode whitespace
|
| 89 |
full_path = full_path.replace(' ', '%20') |
| 90 |
|
| 91 |
kwargs = {}
|
| 92 |
for k,v in headers.items(): |
| 93 |
headers.pop(k) |
| 94 |
k = k.replace('_', '-') |
| 95 |
headers[k] = v |
| 96 |
|
| 97 |
kwargs['headers'] = headers
|
| 98 |
kwargs['headers']['X-Auth-Token'] = self.token |
| 99 |
if body:
|
| 100 |
kwargs['body'] = body
|
| 101 |
kwargs['headers'].setdefault('content-type', |
| 102 |
'application/octet-stream')
|
| 103 |
kwargs['headers'].setdefault('content-length', len(body) if body else 0) |
| 104 |
#kwargs['headers'] = _encode_headers(kwargs['headers'])
|
| 105 |
|
| 106 |
#print '#', method, full_path, kwargs
|
| 107 |
t1 = datetime.datetime.utcnow() |
| 108 |
conn.request(method, full_path, **kwargs) |
| 109 |
|
| 110 |
resp = conn.getresponse() |
| 111 |
t2 = datetime.datetime.utcnow() |
| 112 |
#print 'response time:', str(t2-t1)
|
| 113 |
headers = dict(resp.getheaders())
|
| 114 |
|
| 115 |
if self.verbose: |
| 116 |
print '%d %s' % (resp.status, resp.reason) |
| 117 |
for key, val in headers.items(): |
| 118 |
print '%s: %s' % (key.capitalize(), val) |
| 119 |
print
|
| 120 |
|
| 121 |
length = resp.getheader('content-length', None) |
| 122 |
data = resp.read(length) |
| 123 |
if self.debug: |
| 124 |
print data
|
| 125 |
print
|
| 126 |
|
| 127 |
if int(resp.status) in ERROR_CODES.keys(): |
| 128 |
raise Fault(data, int(resp.status)) |
| 129 |
|
| 130 |
#print '**', resp.status, headers, data
|
| 131 |
return resp.status, headers, data
|
| 132 |
|
| 133 |
def delete(self, path, format='text', params={}): |
| 134 |
return self._req('DELETE', path, format=format, params=params) |
| 135 |
|
| 136 |
def get(self, path, format='text', headers={}, params={}, |
| 137 |
top_level_req=False):
|
| 138 |
return self._req('GET', path, headers=headers, format=format, |
| 139 |
params=params, top_level_req=top_level_req) |
| 140 |
|
| 141 |
def head(self, path, format='text', params={}): |
| 142 |
return self._req('HEAD', path, format=format, params=params) |
| 143 |
|
| 144 |
def post(self, path, body=None, format='text', headers=None, params={}): |
| 145 |
return self._req('POST', path, body, headers=headers, format=format, |
| 146 |
params=params) |
| 147 |
|
| 148 |
def put(self, path, body=None, format='text', headers=None): |
| 149 |
return self._req('PUT', path, body, headers=headers, format=format) |
| 150 |
|
| 151 |
def _list(self, path, format='text', params={}, top_level_req=False, |
| 152 |
**headers): |
| 153 |
status, headers, data = self.get(path, format=format, headers=headers,
|
| 154 |
params=params, |
| 155 |
top_level_req=top_level_req) |
| 156 |
if format == 'json': |
| 157 |
data = json.loads(data) if data else '' |
| 158 |
elif format == 'xml': |
| 159 |
data = minidom.parseString(data) |
| 160 |
else:
|
| 161 |
data = data.strip().split('\n') if data else '' |
| 162 |
return data
|
| 163 |
|
| 164 |
def _get_metadata(self, path, prefix=None, params={}): |
| 165 |
status, headers, data = self.head(path, params=params)
|
| 166 |
prefixlen = len(prefix) if prefix else 0 |
| 167 |
meta = {}
|
| 168 |
for key, val in headers.items(): |
| 169 |
if prefix and not key.startswith(prefix): |
| 170 |
continue
|
| 171 |
elif prefix and key.startswith(prefix): |
| 172 |
key = key[prefixlen:] |
| 173 |
meta[key] = val |
| 174 |
return meta
|
| 175 |
|
| 176 |
def _filter(self, l, d): |
| 177 |
"""
|
| 178 |
filter out from l elements having the metadata values provided
|
| 179 |
"""
|
| 180 |
ll = l |
| 181 |
for elem in l: |
| 182 |
if type(elem) == types.DictionaryType: |
| 183 |
for key in d.keys(): |
| 184 |
k = 'x_object_meta_%s' % key
|
| 185 |
if k in elem.keys() and elem[k] == d[key]: |
| 186 |
ll.remove(elem) |
| 187 |
break
|
| 188 |
return ll
|
| 189 |
|
| 190 |
class OOS_Client(Client): |
| 191 |
"""Openstack Object Storage Client"""
|
| 192 |
|
| 193 |
def _update_metadata(self, path, entity, **meta): |
| 194 |
"""adds new and updates the values of previously set metadata"""
|
| 195 |
ex_meta = self.retrieve_account_metadata(restricted=True) |
| 196 |
ex_meta.update(meta) |
| 197 |
headers = {}
|
| 198 |
prefix = 'x-%s-meta-' % entity
|
| 199 |
for k,v in ex_meta.items(): |
| 200 |
k = '%s%s' % (prefix, k)
|
| 201 |
headers[k] = v |
| 202 |
return self.post(path, headers=headers) |
| 203 |
|
| 204 |
def _reset_metadata(self, path, entity, **meta): |
| 205 |
"""
|
| 206 |
overwrites all user defined metadata
|
| 207 |
"""
|
| 208 |
headers = {}
|
| 209 |
prefix = 'x-%s-meta-' % entity
|
| 210 |
for k,v in meta.items(): |
| 211 |
k = '%s%s' % (prefix, k)
|
| 212 |
headers[k] = v |
| 213 |
return self.post(path, headers=headers) |
| 214 |
|
| 215 |
def _delete_metadata(self, path, entity, meta=[]): |
| 216 |
"""delete previously set metadata"""
|
| 217 |
ex_meta = self.retrieve_account_metadata(restricted=True) |
| 218 |
headers = {}
|
| 219 |
prefix = 'x-%s-meta-' % entity
|
| 220 |
for k in ex_meta.keys(): |
| 221 |
if k in meta: |
| 222 |
headers['%s%s' % (prefix, k)] = ex_meta[k]
|
| 223 |
return self.post(path, headers=headers) |
| 224 |
|
| 225 |
# Storage Account Services
|
| 226 |
|
| 227 |
def list_containers(self, format='text', limit=None, marker=None, params={}, |
| 228 |
**headers): |
| 229 |
"""lists containers"""
|
| 230 |
params.update({'limit':limit, 'marker':marker})
|
| 231 |
return self._list('', format, params, **headers) |
| 232 |
|
| 233 |
def retrieve_account_metadata(self, restricted=False, **params): |
| 234 |
"""returns the account metadata"""
|
| 235 |
prefix = 'x-account-meta-' if restricted else None |
| 236 |
return self._get_metadata('', prefix, params) |
| 237 |
|
| 238 |
def update_account_metadata(self, **meta): |
| 239 |
"""updates the account metadata"""
|
| 240 |
return self._update_metadata('', 'account', **meta) |
| 241 |
|
| 242 |
def delete_account_metadata(self, meta=[]): |
| 243 |
"""deletes the account metadata"""
|
| 244 |
return self._delete_metadata('', 'account', meta) |
| 245 |
|
| 246 |
def reset_account_metadata(self, **meta): |
| 247 |
"""resets account metadata"""
|
| 248 |
return self._reset_metadata('', 'account', **meta) |
| 249 |
|
| 250 |
# Storage Container Services
|
| 251 |
|
| 252 |
def _filter_trashed(self, l): |
| 253 |
return self._filter(l, {'trash':'true'}) |
| 254 |
|
| 255 |
def list_objects(self, container, format='text', limit=None, marker=None, |
| 256 |
prefix=None, delimiter=None, path=None, |
| 257 |
include_trashed=False, params={}, **headers):
|
| 258 |
"""returns a list with the container objects"""
|
| 259 |
params.update({'limit':limit, 'marker':marker, 'prefix':prefix,
|
| 260 |
'delimiter':delimiter, 'path':path}) |
| 261 |
l = self._list('/' + container, format, params, **headers) |
| 262 |
#TODO support filter trashed with xml also
|
| 263 |
if format != 'xml' and not include_trashed: |
| 264 |
l = self._filter_trashed(l)
|
| 265 |
return l
|
| 266 |
|
| 267 |
def create_container(self, container, **meta): |
| 268 |
"""creates a container"""
|
| 269 |
headers = {}
|
| 270 |
for k,v in meta.items(): |
| 271 |
headers['x-container-meta-%s' %k.strip().upper()] = v.strip()
|
| 272 |
status, header, data = self.put('/' + container, headers=headers) |
| 273 |
if status == 202: |
| 274 |
return False |
| 275 |
elif status != 201: |
| 276 |
raise Fault(data, int(status)) |
| 277 |
return True |
| 278 |
|
| 279 |
def delete_container(self, container, params={}): |
| 280 |
"""deletes a container"""
|
| 281 |
return self.delete('/' + container, params=params) |
| 282 |
|
| 283 |
def retrieve_container_metadata(self, container, restricted=False, **params): |
| 284 |
"""returns the container metadata"""
|
| 285 |
prefix = 'x-container-meta-' if restricted else None |
| 286 |
return self._get_metadata('/%s' % container, prefix, params) |
| 287 |
|
| 288 |
def update_container_metadata(self, container, **meta): |
| 289 |
"""unpdates the container metadata"""
|
| 290 |
return self._update_metadata('/' + container, 'container', **meta) |
| 291 |
|
| 292 |
def delete_container_metadata(self, container, meta=[]): |
| 293 |
"""deletes the container metadata"""
|
| 294 |
path = '/%s' % (container)
|
| 295 |
return self._delete_metadata(path, 'container', meta) |
| 296 |
|
| 297 |
# Storage Object Services
|
| 298 |
|
| 299 |
def request_object(self, container, object, format='text', params={}, |
| 300 |
**headers): |
| 301 |
"""returns tuple containing the status, headers and data response for an object request"""
|
| 302 |
path = '/%s/%s' % (container, object) |
| 303 |
status, headers, data = self.get(path, format, headers, params)
|
| 304 |
return status, headers, data
|
| 305 |
|
| 306 |
def retrieve_object(self, container, object, format='text', params={}, |
| 307 |
**headers): |
| 308 |
"""returns an object's data"""
|
| 309 |
t = self.request_object(container, object, format, params, **headers) |
| 310 |
return t[2] |
| 311 |
|
| 312 |
def create_directory_marker(self, container, object): |
| 313 |
"""creates a dierectory marker"""
|
| 314 |
if not object: |
| 315 |
raise Fault('Directory markers have to be nested in a container') |
| 316 |
h = {'content_type':'application/directory'}
|
| 317 |
return self.create_zero_length_object(container, object, **h) |
| 318 |
|
| 319 |
def create_object(self, container, object, f=stdin, format='text', meta={}, |
| 320 |
etag=None, content_type=None, content_encoding=None, |
| 321 |
content_disposition=None, **headers):
|
| 322 |
"""creates a zero-length object"""
|
| 323 |
path = '/%s/%s' % (container, object) |
| 324 |
for k, v in headers.items(): |
| 325 |
if not v: |
| 326 |
headers.pop(k) |
| 327 |
|
| 328 |
l = ['etag', 'content_encoding', 'content_disposition', 'content_type'] |
| 329 |
l = [elem for elem in l if eval(elem)] |
| 330 |
for elem in l: |
| 331 |
headers.update({elem:eval(elem)})
|
| 332 |
|
| 333 |
for k,v in meta.items(): |
| 334 |
headers['x-object-meta-%s' %k.strip()] = v.strip()
|
| 335 |
data = f.read() if f else None |
| 336 |
return self.put(path, data, format, headers=headers) |
| 337 |
|
| 338 |
def create_zero_length_object(self, container, object, meta={}, etag=None, |
| 339 |
content_type=None, content_encoding=None, |
| 340 |
content_disposition=None, **headers):
|
| 341 |
args = locals()
|
| 342 |
for elem in ['self', 'container', 'headers']: |
| 343 |
args.pop(elem) |
| 344 |
args.update(headers) |
| 345 |
return self.create_object(container, f=None, **args) |
| 346 |
|
| 347 |
def update_object(self, container, object, f=stdin, offset=None, meta={}, |
| 348 |
content_length=None, content_type=None, |
| 349 |
content_encoding=None, content_disposition=None, |
| 350 |
**headers): |
| 351 |
path = '/%s/%s' % (container, object) |
| 352 |
for k, v in headers.items(): |
| 353 |
if not v: |
| 354 |
headers.pop(k) |
| 355 |
|
| 356 |
l = ['content_encoding', 'content_disposition', 'content_type', |
| 357 |
'content_length']
|
| 358 |
l = [elem for elem in l if eval(elem)] |
| 359 |
for elem in l: |
| 360 |
headers.update({elem:eval(elem)})
|
| 361 |
|
| 362 |
if 'content_range' not in headers.keys(): |
| 363 |
if offset != None: |
| 364 |
headers['content_range'] = 'bytes %s-/*' % offset |
| 365 |
else:
|
| 366 |
headers['content_range'] = 'bytes */*' |
| 367 |
|
| 368 |
for k,v in meta.items(): |
| 369 |
headers['x-object-meta-%s' %k.strip()] = v.strip()
|
| 370 |
data = f.read() if f else None |
| 371 |
return self.post(path, data, headers=headers) |
| 372 |
|
| 373 |
def _change_obj_location(self, src_container, src_object, dst_container, |
| 374 |
dst_object, remove=False, meta={}, **headers):
|
| 375 |
path = '/%s/%s' % (dst_container, dst_object)
|
| 376 |
headers = {} if not headers else headers
|
| 377 |
for k, v in meta.items(): |
| 378 |
headers['x-object-meta-%s' % k] = v
|
| 379 |
if remove:
|
| 380 |
headers['x-move-from'] = '/%s/%s' % (src_container, src_object) |
| 381 |
else:
|
| 382 |
headers['x-copy-from'] = '/%s/%s' % (src_container, src_object) |
| 383 |
headers['content_length'] = 0 |
| 384 |
return self.put(path, headers=headers) |
| 385 |
|
| 386 |
def copy_object(self, src_container, src_object, dst_container, dst_object, |
| 387 |
meta, **headers): |
| 388 |
"""copies an object"""
|
| 389 |
return self._change_obj_location(src_container, src_object, |
| 390 |
dst_container, dst_object, remove=False,
|
| 391 |
meta=meta, **headers) |
| 392 |
|
| 393 |
def move_object(self, src_container, src_object, dst_container, |
| 394 |
dst_object, meta={}, **headers):
|
| 395 |
"""moves an object"""
|
| 396 |
return self._change_obj_location(src_container, src_object, |
| 397 |
dst_container, dst_object, remove=True,
|
| 398 |
meta=meta, **headers) |
| 399 |
|
| 400 |
def delete_object(self, container, object, params={}): |
| 401 |
"""deletes an object"""
|
| 402 |
return self.delete('/%s/%s' % (container, object), params=params) |
| 403 |
|
| 404 |
def retrieve_object_metadata(self, container, object, restricted=False, |
| 405 |
version=None):
|
| 406 |
"""
|
| 407 |
set restricted to True to get only user defined metadata
|
| 408 |
"""
|
| 409 |
path = '/%s/%s' % (container, object) |
| 410 |
prefix = 'x-object-meta-' if restricted else None |
| 411 |
params = {'version':version} if version else {}
|
| 412 |
return self._get_metadata(path, prefix, params=params) |
| 413 |
|
| 414 |
def update_object_metadata(self, container, object, **meta): |
| 415 |
"""
|
| 416 |
updates object's metadata
|
| 417 |
"""
|
| 418 |
path = '/%s/%s' % (container, object) |
| 419 |
return self._update_metadata(path, 'object', **meta) |
| 420 |
|
| 421 |
def delete_object_metadata(self, container, object, meta=[]): |
| 422 |
"""
|
| 423 |
deletes object's metadata
|
| 424 |
"""
|
| 425 |
path = '/%s/%s' % (container, object) |
| 426 |
return self._delete_metadata(path, 'object', meta) |
| 427 |
|
| 428 |
class Pithos_Client(OOS_Client): |
| 429 |
"""Pithos Storage Client. Extends OOS_Client"""
|
| 430 |
|
| 431 |
def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None, |
| 432 |
blocksize=1024):
|
| 433 |
"""perfomrs a chunked request"""
|
| 434 |
http = HTTPConnection(self.host)
|
| 435 |
|
| 436 |
# write header
|
| 437 |
path = '/%s/%s%s' % (self.api, self.account, path) |
| 438 |
http.putrequest(method, path) |
| 439 |
http.putheader('x-auth-token', self.token) |
| 440 |
http.putheader('content-type', 'application/octet-stream') |
| 441 |
http.putheader('transfer-encoding', 'chunked') |
| 442 |
if headers:
|
| 443 |
for header,value in headers.items(): |
| 444 |
http.putheader(header, value) |
| 445 |
http.endheaders() |
| 446 |
|
| 447 |
# write body
|
| 448 |
data = ''
|
| 449 |
while True: |
| 450 |
if f.closed:
|
| 451 |
break
|
| 452 |
block = f.read(blocksize) |
| 453 |
if block == '': |
| 454 |
break
|
| 455 |
data = '%x\r\n%s\r\n' % (len(block), block) |
| 456 |
try:
|
| 457 |
http.send(data) |
| 458 |
except:
|
| 459 |
#retry
|
| 460 |
http.send(data) |
| 461 |
data = '0\r\n\r\n'
|
| 462 |
try:
|
| 463 |
http.send(data) |
| 464 |
except:
|
| 465 |
#retry
|
| 466 |
http.send(data) |
| 467 |
|
| 468 |
# get response
|
| 469 |
resp = http.getresponse() |
| 470 |
|
| 471 |
headers = dict(resp.getheaders())
|
| 472 |
|
| 473 |
if self.verbose: |
| 474 |
print '%d %s' % (resp.status, resp.reason) |
| 475 |
for key, val in headers.items(): |
| 476 |
print '%s: %s' % (key.capitalize(), val) |
| 477 |
print
|
| 478 |
|
| 479 |
length = resp.getheader('Content-length', None) |
| 480 |
data = resp.read(length) |
| 481 |
if self.debug: |
| 482 |
print data
|
| 483 |
print
|
| 484 |
|
| 485 |
if int(resp.status) in ERROR_CODES.keys(): |
| 486 |
raise Fault(data, int(resp.status)) |
| 487 |
|
| 488 |
#print '*', resp.status, headers, data
|
| 489 |
return resp.status, headers, data
|
| 490 |
|
| 491 |
def _update_metadata(self, path, entity, **meta): |
| 492 |
"""
|
| 493 |
adds new and updates the values of previously set metadata
|
| 494 |
"""
|
| 495 |
params = {'update':None}
|
| 496 |
headers = {}
|
| 497 |
prefix = 'x-%s-meta-' % entity
|
| 498 |
for k,v in meta.items(): |
| 499 |
k = '%s%s' % (prefix, k)
|
| 500 |
headers[k] = v |
| 501 |
return self.post(path, headers=headers, params=params) |
| 502 |
|
| 503 |
def _delete_metadata(self, path, entity, meta=[]): |
| 504 |
"""
|
| 505 |
delete previously set metadata
|
| 506 |
"""
|
| 507 |
params = {'update':None}
|
| 508 |
headers = {}
|
| 509 |
prefix = 'x-%s-meta-' % entity
|
| 510 |
for m in meta: |
| 511 |
headers['%s%s' % (prefix, m)] = '' |
| 512 |
return self.post(path, headers=headers, params=params) |
| 513 |
|
| 514 |
# Storage Account Services
|
| 515 |
|
| 516 |
def list_containers(self, format='text', if_modified_since=None, |
| 517 |
if_unmodified_since=None, limit=1000, marker=None, |
| 518 |
until=None):
|
| 519 |
"""returns a list with the account containers"""
|
| 520 |
params = {'until':until} if until else {}
|
| 521 |
headers = {'if-modified-since':if_modified_since,
|
| 522 |
'if-unmodified-since':if_unmodified_since}
|
| 523 |
return OOS_Client.list_containers(self, format=format, limit=limit, |
| 524 |
marker=marker, params=params, |
| 525 |
**headers) |
| 526 |
|
| 527 |
def retrieve_account_metadata(self, restricted=False, until=None): |
| 528 |
"""returns the account metadata"""
|
| 529 |
params = {'until':until} if until else {}
|
| 530 |
return OOS_Client.retrieve_account_metadata(self, restricted=restricted, |
| 531 |
**params) |
| 532 |
|
| 533 |
def set_account_groups(self, **groups): |
| 534 |
"""create account groups"""
|
| 535 |
headers = {}
|
| 536 |
for k, v in groups.items(): |
| 537 |
headers['x-account-group-%s' % k] = v
|
| 538 |
params = {'update':None}
|
| 539 |
return self.post('', headers=headers, params=params) |
| 540 |
|
| 541 |
def retrieve_account_groups(self): |
| 542 |
"""returns the account groups"""
|
| 543 |
meta = self.retrieve_account_metadata()
|
| 544 |
prefix = 'x-account-group-'
|
| 545 |
prefixlen = len(prefix)
|
| 546 |
groups = {}
|
| 547 |
for key, val in meta.items(): |
| 548 |
if prefix and not key.startswith(prefix): |
| 549 |
continue
|
| 550 |
elif prefix and key.startswith(prefix): |
| 551 |
key = key[prefixlen:] |
| 552 |
groups[key] = val |
| 553 |
return groups
|
| 554 |
|
| 555 |
def unset_account_groups(self, groups=[]): |
| 556 |
"""delete account groups"""
|
| 557 |
headers = {}
|
| 558 |
for elem in groups: |
| 559 |
headers['x-account-group-%s' % elem] = '' |
| 560 |
params = {'update':None}
|
| 561 |
return self.post('', headers=headers, params=params) |
| 562 |
|
| 563 |
def reset_account_groups(self, **groups): |
| 564 |
"""overrides account groups"""
|
| 565 |
headers = {}
|
| 566 |
for k, v in groups.items(): |
| 567 |
v = v.strip() |
| 568 |
headers['x-account-group-%s' % k] = v
|
| 569 |
meta = self.retrieve_account_metadata(restricted=True) |
| 570 |
prefix = 'x-account-meta-'
|
| 571 |
for k,v in meta.items(): |
| 572 |
k = '%s%s' % (prefix, k)
|
| 573 |
headers[k] = v |
| 574 |
return self.post('', headers=headers) |
| 575 |
|
| 576 |
# Storage Container Services
|
| 577 |
|
| 578 |
def list_objects(self, container, format='text', limit=None, marker=None, |
| 579 |
prefix=None, delimiter=None, path=None, |
| 580 |
include_trashed=False, params={}, if_modified_since=None, |
| 581 |
if_unmodified_since=None, meta='', until=None): |
| 582 |
"""returns a list with the container objects"""
|
| 583 |
params = {'until':until, 'meta':meta}
|
| 584 |
args = locals()
|
| 585 |
for elem in ['self', 'container', 'params', 'until', 'meta']: |
| 586 |
args.pop(elem) |
| 587 |
return OOS_Client.list_objects(self, container, params=params, |
| 588 |
**args) |
| 589 |
|
| 590 |
def retrieve_container_metadata(self, container, restricted=False, |
| 591 |
until=None):
|
| 592 |
"""returns container's metadata"""
|
| 593 |
params = {'until':until} if until else {}
|
| 594 |
return OOS_Client.retrieve_container_metadata(self, container, |
| 595 |
restricted=restricted, |
| 596 |
**params) |
| 597 |
|
| 598 |
def set_container_policies(self, container, **policies): |
| 599 |
"""sets containers policies"""
|
| 600 |
path = '/%s' % (container)
|
| 601 |
headers = {}
|
| 602 |
print '' |
| 603 |
for key, val in policies.items(): |
| 604 |
headers['x-container-policy-%s' % key] = val
|
| 605 |
return self.post(path, headers=headers) |
| 606 |
|
| 607 |
def delete_container(self, container, until=None): |
| 608 |
"""deletes a container or the container history until the date provided"""
|
| 609 |
params = {'until':until} if until else {}
|
| 610 |
return OOS_Client.delete_container(self, container, params) |
| 611 |
|
| 612 |
# Storage Object Services
|
| 613 |
|
| 614 |
def retrieve_object(self, container, object, params={}, format='text', range=None, |
| 615 |
if_range=None, if_match=None, if_none_match=None, |
| 616 |
if_modified_since=None, if_unmodified_since=None, |
| 617 |
**headers): |
| 618 |
"""returns an object"""
|
| 619 |
headers={}
|
| 620 |
l = ['range', 'if_range', 'if_match', 'if_none_match', |
| 621 |
'if_modified_since', 'if_unmodified_since'] |
| 622 |
l = [elem for elem in l if eval(elem)] |
| 623 |
for elem in l: |
| 624 |
headers.update({elem:eval(elem)})
|
| 625 |
return OOS_Client.retrieve_object(self, container, object, format=format, |
| 626 |
params=params, **headers) |
| 627 |
|
| 628 |
def retrieve_object_version(self, container, object, version, detail=False, |
| 629 |
range=None, if_range=None, if_match=None, |
| 630 |
if_none_match=None, if_modified_since=None, |
| 631 |
if_unmodified_since=None):
|
| 632 |
"""returns a specific object version"""
|
| 633 |
args = locals()
|
| 634 |
l = ['self', 'container', 'object'] |
| 635 |
for elem in l: |
| 636 |
args.pop(elem) |
| 637 |
params = {'version':version}
|
| 638 |
return self.retrieve_object(container, object, params, **args) |
| 639 |
|
| 640 |
def retrieve_object_versionlist(self, container, object, range=None, |
| 641 |
if_range=None, if_match=None, |
| 642 |
if_none_match=None, if_modified_since=None, |
| 643 |
if_unmodified_since=None):
|
| 644 |
"""returns the object version list"""
|
| 645 |
args = locals()
|
| 646 |
l = ['self', 'container', 'object'] |
| 647 |
for elem in l: |
| 648 |
args.pop(elem) |
| 649 |
|
| 650 |
return self.retrieve_object_version(container, object, version='list', |
| 651 |
detail=True, **args)
|
| 652 |
|
| 653 |
def create_zero_length_object(self, container, object, meta={}, |
| 654 |
etag=None, content_type=None, content_encoding=None, |
| 655 |
content_disposition=None, x_object_manifest=None, |
| 656 |
x_object_sharing=None, x_object_public=None): |
| 657 |
"""createas a zero length object"""
|
| 658 |
args = locals()
|
| 659 |
for elem in ['self', 'container', 'object']: |
| 660 |
args.pop(elem) |
| 661 |
return OOS_Client.create_zero_length_object(self, container, object, |
| 662 |
**args) |
| 663 |
|
| 664 |
def create_object(self, container, object, f=stdin, meta={}, |
| 665 |
etag=None, content_type=None, content_encoding=None, |
| 666 |
content_disposition=None, x_object_manifest=None, |
| 667 |
x_object_sharing=None, x_object_public=None): |
| 668 |
"""creates an object"""
|
| 669 |
args = locals()
|
| 670 |
for elem in ['self', 'container', 'object']: |
| 671 |
args.pop(elem) |
| 672 |
return OOS_Client.create_object(self, container, object, **args) |
| 673 |
|
| 674 |
def create_object_using_chunks(self, container, object, f=stdin, |
| 675 |
blocksize=1024, meta={}, etag=None, |
| 676 |
content_type=None, content_encoding=None, |
| 677 |
content_disposition=None,
|
| 678 |
x_object_sharing=None,
|
| 679 |
x_object_manifest=None,
|
| 680 |
x_object_public=None):
|
| 681 |
"""creates an object (incremental upload)"""
|
| 682 |
path = '/%s/%s' % (container, object) |
| 683 |
headers = {}
|
| 684 |
l = ['etag', 'content_type', 'content_encoding', 'content_disposition', |
| 685 |
'x_object_sharing', 'x_object_manifest', 'x_object_public'] |
| 686 |
l = [elem for elem in l if eval(elem)] |
| 687 |
for elem in l: |
| 688 |
headers.update({elem:eval(elem)})
|
| 689 |
|
| 690 |
for k,v in meta.items(): |
| 691 |
v = v.strip() |
| 692 |
headers['x-object-meta-%s' %k.strip()] = v
|
| 693 |
|
| 694 |
return self._chunked_transfer(path, 'PUT', f, headers=headers, |
| 695 |
blocksize=blocksize) |
| 696 |
|
| 697 |
def create_object_by_hashmap(container, object, f=stdin, format='json', |
| 698 |
meta={}, etag=None, content_encoding=None,
|
| 699 |
content_disposition=None, content_type=None, |
| 700 |
x_object_sharing=None, x_object_manifest=None, |
| 701 |
x_object_public = None):
|
| 702 |
"""creates an object by uploading hashes representing data instead of data"""
|
| 703 |
args = locals()
|
| 704 |
for elem in ['self', 'container', 'object']: |
| 705 |
args.pop(elem) |
| 706 |
|
| 707 |
data = f.read() if f else None |
| 708 |
if data and format == 'json': |
| 709 |
try:
|
| 710 |
data = eval(data)
|
| 711 |
data = json.dumps(data) |
| 712 |
except SyntaxError: |
| 713 |
raise Fault('Invalid formatting') |
| 714 |
|
| 715 |
#TODO check with xml
|
| 716 |
return self.create_object(container, object, **args) |
| 717 |
|
| 718 |
def create_manifestation(self, container, object, manifest): |
| 719 |
"""creates a manifestation"""
|
| 720 |
headers={'x_object_manifest':manifest}
|
| 721 |
return self.create_object(container, object, f=None, **headers) |
| 722 |
|
| 723 |
def update_object(self, container, object, f=stdin, offset=None, meta={}, |
| 724 |
content_length=None, content_type=None, content_range=None, |
| 725 |
content_encoding=None, content_disposition=None, |
| 726 |
x_object_bytes=None, x_object_manifest=None, |
| 727 |
x_object_sharing=None, x_object_public=None): |
| 728 |
"""updates an object"""
|
| 729 |
spath = '/%s/%s' % (container, object) |
| 730 |
args = locals()
|
| 731 |
for elem in ['self', 'container', 'object']: |
| 732 |
args.pop(elem) |
| 733 |
|
| 734 |
return OOS_Client.update_object(self, container, object, **args) |
| 735 |
|
| 736 |
def update_object_using_chunks(self, container, object, f=stdin, |
| 737 |
blocksize=1024, offset=None, meta={}, |
| 738 |
content_type=None, content_encoding=None, |
| 739 |
content_disposition=None, x_object_bytes=None, |
| 740 |
x_object_manifest=None, x_object_sharing=None, |
| 741 |
x_object_public=None):
|
| 742 |
"""updates an object (incremental upload)"""
|
| 743 |
path = '/%s/%s' % (container, object) |
| 744 |
headers = {}
|
| 745 |
l = ['content_type', 'content_encoding', 'content_disposition', |
| 746 |
'x_object_bytes', 'x_object_manifest', 'x_object_sharing', |
| 747 |
'x_object_public']
|
| 748 |
l = [elem for elem in l if eval(elem)] |
| 749 |
for elem in l: |
| 750 |
headers.update({elem:eval(elem)})
|
| 751 |
|
| 752 |
if offset != None: |
| 753 |
headers['content_range'] = 'bytes %s-/*' % offset |
| 754 |
else:
|
| 755 |
headers['content_range'] = 'bytes */*' |
| 756 |
|
| 757 |
for k,v in meta.items(): |
| 758 |
v = v.strip() |
| 759 |
headers['x-object-meta-%s' %k.strip()] = v
|
| 760 |
|
| 761 |
return self._chunked_transfer(path, 'POST', f, headers=headers, |
| 762 |
blocksize=blocksize) |
| 763 |
|
| 764 |
def delete_object(self, container, object, until=None): |
| 765 |
"""deletes an object or the object history until the date provided"""
|
| 766 |
params = {'until':until} if until else {}
|
| 767 |
return OOS_Client.delete_object(self, container, object, params) |
| 768 |
|
| 769 |
def trash_object(self, container, object): |
| 770 |
"""trashes an object"""
|
| 771 |
path = '/%s/%s' % (container, object) |
| 772 |
meta = {'trash':'true'}
|
| 773 |
return self._update_metadata(path, 'object', **meta) |
| 774 |
|
| 775 |
def restore_object(self, container, object): |
| 776 |
"""restores a trashed object"""
|
| 777 |
return self.delete_object_metadata(container, object, ['trash']) |
| 778 |
|
| 779 |
def publish_object(self, container, object): |
| 780 |
"""sets a previously created object publicly accessible"""
|
| 781 |
path = '/%s/%s' % (container, object) |
| 782 |
headers = {'content_range':'bytes */*'}
|
| 783 |
headers['x_object_public'] = True |
| 784 |
return self.post(path, headers=headers) |
| 785 |
|
| 786 |
def unpublish_object(self, container, object): |
| 787 |
"""unpublish an object"""
|
| 788 |
path = '/%s/%s' % (container, object) |
| 789 |
headers = {'content_range':'bytes */*'}
|
| 790 |
headers['x_object_public'] = False |
| 791 |
return self.post(path, headers=headers) |
| 792 |
|
| 793 |
def _change_obj_location(self, src_container, src_object, dst_container, |
| 794 |
dst_object, remove=False, meta={}, **headers):
|
| 795 |
path = '/%s/%s' % (dst_container, dst_object)
|
| 796 |
headers = {} if not headers else headers
|
| 797 |
for k, v in meta.items(): |
| 798 |
headers['x-object-meta-%s' % k] = v
|
| 799 |
if remove:
|
| 800 |
headers['x-move-from'] = '/%s/%s' % (src_container, src_object) |
| 801 |
else:
|
| 802 |
headers['x-copy-from'] = '/%s/%s' % (src_container, src_object) |
| 803 |
headers['content_length'] = 0 |
| 804 |
return self.put(path, headers=headers) |
| 805 |
|
| 806 |
def copy_object(self, src_container, src_object, dst_container, dst_object, |
| 807 |
meta={}, public=False, version=None):
|
| 808 |
"""copies an object"""
|
| 809 |
headers = {}
|
| 810 |
headers['x_object_public'] = public
|
| 811 |
if version:
|
| 812 |
headers['x_object_version'] = version
|
| 813 |
return OOS_Client.copy_object(self, src_container, src_object, |
| 814 |
dst_container, dst_object, meta=meta, |
| 815 |
**headers) |
| 816 |
|
| 817 |
def move_object(self, src_container, src_object, dst_container, |
| 818 |
dst_object, meta={}, public=False, version=None):
|
| 819 |
"""moves an object"""
|
| 820 |
headers = {}
|
| 821 |
headers['x_object_public'] = public
|
| 822 |
if version:
|
| 823 |
headers['x_object_version'] = version
|
| 824 |
return OOS_Client.move_object(self, src_container, src_object, |
| 825 |
dst_container, dst_object, meta=meta, |
| 826 |
**headers) |
| 827 |
|
| 828 |
def list_shared_by_others(self, limit=None, marker=None, format='text'): |
| 829 |
"""lists other accounts that share objects to the user"""
|
| 830 |
l = ['limit', 'marker'] |
| 831 |
params = {}
|
| 832 |
for elem in [elem for elem in l if eval(elem)]: |
| 833 |
params[elem] = eval(elem)
|
| 834 |
return self._list('', format, params, top_level_req=True) |
| 835 |
|
| 836 |
def share_object(self, container, object, l, read=True): |
| 837 |
"""gives access(read by default) to an object to a user/group list"""
|
| 838 |
action = 'read' if read else 'write' |
| 839 |
sharing = '%s=%s' % (action, ','.join(l)) |
| 840 |
self.update_object(container, object, f=None, x_object_sharing=sharing) |
| 841 |
|
| 842 |
def _encode_headers(headers): |
| 843 |
h = {}
|
| 844 |
for k, v in headers.items(): |
| 845 |
k = urllib.quote(k) |
| 846 |
if v and type(v) == types.StringType: |
| 847 |
v = urllib.quote(v, '/=,-* :"')
|
| 848 |
h[k] = v |
| 849 |
return h
|