-# Copyright 2011 GRNET S.A. All rights reserved.
+# Copyright 2011-2012 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
-from httplib import HTTPConnection, HTTP
+from httplib import HTTPConnection, HTTPSConnection, HTTP
from sys import stdin
from xml.dom import minidom
+from StringIO import StringIO
+from urllib import quote, unquote
+from urlparse import urlparse
import json
import types
import socket
-import pithos.api.faults
+import urllib
+import datetime
ERROR_CODES = {304:'Not Modified',
400:'Bad Request',
401:'Unauthorized',
+ 403:'Forbidden',
404:'Not Found',
409:'Conflict',
411:'Length Required',
412:'Precondition Failed',
+ 413:'Request Entity Too Large',
416:'Range Not Satisfiable',
422:'Unprocessable Entity',
- 503:'Service Unavailable'}
+ 500:'Internal Server Error',
+ 501:'Not Implemented'}
class Fault(Exception):
def __init__(self, data='', status=None):
self.status = status
class Client(object):
- def __init__(self, host, token, account, api='v1', verbose=False, debug=False):
- """`host` can also include a port, e.g '127.0.0.1:8000'."""
+ def __init__(self, url, token, account, verbose=False, debug=False):
+ """`url` can also include a port, e.g '127.0.0.1:8000'."""
- self.host = host
+ self.url = url
self.account = account
- self.api = api
self.verbose = verbose or debug
self.debug = debug
self.token = token
- def _req(self, method, path, body=None, headers={}, format='text',
- params={}):
- full_path = '/%s/%s%s?format=%s' % (self.api, self.account, path,
- format)
- for k,v in params.items():
- if v:
- full_path = '%s&%s=%s' %(full_path, k, v)
- else:
- full_path = '%s&%s' %(full_path, k)
- conn = HTTPConnection(self.host)
+ def _req(self, method, path, body=None, headers={}, format='text', params={}):
+ p = urlparse(self.url)
+ if p.scheme == 'http':
+ conn = HTTPConnection(p.netloc)
+ elif p.scheme == 'https':
+ conn = HTTPSConnection(p.netloc)
+ else:
+ raise Exception('Unknown URL scheme')
- #encode whitespace
- full_path = full_path.replace(' ', '%20')
+ full_path = _prepare_path(p.path + path, format, params)
kwargs = {}
- for k,v in headers.items():
- headers.pop(k)
- k = k.replace('_', '-')
- headers[k] = v
-
- kwargs['headers'] = headers
+ kwargs['headers'] = _prepare_headers(headers)
kwargs['headers']['X-Auth-Token'] = self.token
if body:
kwargs['body'] = body
- kwargs['headers'].setdefault('content-type',
- 'application/octet-stream')
+ kwargs['headers'].setdefault('content-type', 'application/octet-stream')
kwargs['headers'].setdefault('content-length', len(body) if body else 0)
- try:
- #print '*', method, full_path, kwargs
- conn.request(method, full_path, **kwargs)
- except socket.error, e:
- raise Fault(status=503)
-
+
+ #print '#', method, full_path, kwargs
+ #t1 = datetime.datetime.utcnow()
+ conn.request(method, full_path, **kwargs)
+
resp = conn.getresponse()
- headers = dict(resp.getheaders())
+ #t2 = datetime.datetime.utcnow()
+ #print 'response time:', str(t2-t1)
+ return _handle_response(resp, self.verbose, self.debug)
+
+ def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
+ blocksize=1024, params={}):
+ """perfomrs a chunked request"""
+ p = urlparse(self.url)
+ if p.scheme == 'http':
+ conn = HTTPConnection(p.netloc)
+ elif p.scheme == 'https':
+ conn = HTTPSConnection(p.netloc)
+ else:
+ raise Exception('Unknown URL scheme')
- if self.verbose:
- print '%d %s' % (resp.status, resp.reason)
- for key, val in headers.items():
- print '%s: %s' % (key.capitalize(), val)
- print
+ full_path = _prepare_path(p.path + path, params=params)
- length = resp.getheader('content-length', None)
- data = resp.read(length)
- if self.debug:
- print data
- print
+ conn.putrequest(method, full_path)
+ conn.putheader('x-auth-token', self.token)
+ conn.putheader('content-type', 'application/octet-stream')
+ conn.putheader('transfer-encoding', 'chunked')
+ for k,v in _prepare_headers(headers).items():
+ conn.putheader(k, v)
+ conn.endheaders()
- if int(resp.status) in ERROR_CODES.keys():
- #print '**', resp.status
- raise Fault(data, int(resp.status))
+ # write body
+ data = ''
+ while True:
+ if f.closed:
+ break
+ block = f.read(blocksize)
+ if block == '':
+ break
+ data = '%x\r\n%s\r\n' % (len(block), block)
+ try:
+ conn.send(data)
+ except:
+ #retry
+ conn.send(data)
+ data = '0\r\n\r\n'
+ try:
+ conn.send(data)
+ except:
+ #retry
+ conn.send(data)
- #print '**', resp.status, headers, data
- return resp.status, headers, data
+ resp = conn.getresponse()
+ return _handle_response(resp, self.verbose, self.debug)
def delete(self, path, format='text', params={}):
return self._req('DELETE', path, format=format, params=params)
- def get(self, path, format='text', headers=None, params={}):
+ def get(self, path, format='text', headers={}, params={}):
return self._req('GET', path, headers=headers, format=format,
params=params)
def head(self, path, format='text', params={}):
- return self._req('HEAD', path, format=format, params=params)
+ return self._req('HEAD', path, format=format, params=params)
def post(self, path, body=None, format='text', headers=None, params={}):
return self._req('POST', path, body, headers=headers, format=format,
params=params)
- def put(self, path, body=None, format='text', headers=None):
- return self._req('PUT', path, body, headers=headers, format=format)
+ def put(self, path, body=None, format='text', headers=None, params={}):
+ return self._req('PUT', path, body, headers=headers, format=format,
+ params=params)
def _list(self, path, format='text', params={}, **headers):
status, headers, data = self.get(path, format=format, headers=headers,
elif format == 'xml':
data = minidom.parseString(data)
else:
- data = data.strip().split('\n') if data else ''
+ data = data.split('\n')[:-1] if data else ''
return data
def _get_metadata(self, path, prefix=None, params={}):
# Storage Account Services
- def list_containers(self, format='text', limit=10000, marker=None, params={},
- **headers):
+ def list_containers(self, format='text', limit=None,
+ marker=None, params={}, account=None, **headers):
"""lists containers"""
- if not params:
- params = {}
+ account = account or self.account
+ path = '/%s' % account
params.update({'limit':limit, 'marker':marker})
- return self._list('', format, params, **headers)
+ return self._list(path, format, params, **headers)
- def retrieve_account_metadata(self, restricted=False, **params):
+ def retrieve_account_metadata(self, restricted=False, account=None, **params):
"""returns the account metadata"""
+ account = account or self.account
+ path = '/%s' % account
prefix = 'x-account-meta-' if restricted else None
- return self._get_metadata('', prefix, params)
+ return self._get_metadata(path, prefix, params)
- def update_account_metadata(self, **meta):
+ def update_account_metadata(self, account=None, **meta):
"""updates the account metadata"""
- return self._update_metadata('', 'account', **meta)
+ account = account or self.account
+ path = '/%s' % account
+ return self._update_metadata(path, 'account', **meta)
- def delete_account_metadata(self, meta=[]):
+ def delete_account_metadata(self, meta=[], account=None):
"""deletes the account metadata"""
- return self._delete_metadata('', 'account', meta)
+ account = account or self.account
+ path = '/%s' % account
+ return self._delete_metadata(path, 'account', meta)
- def reset_account_metadata(self, **meta):
+ def reset_account_metadata(self, account=None, **meta):
"""resets account metadata"""
- return self._reset_metadata('', 'account', **meta)
+ account = account or self.account
+ path = '/%s' % account
+ return self._reset_metadata(path, 'account', **meta)
# Storage Container Services
def _filter_trashed(self, l):
return self._filter(l, {'trash':'true'})
- def list_objects(self, container, format='text', limit=10000, marker=None,
- prefix=None, delimiter=None, path=None,
- include_trashed=False, params={}, **headers):
+ def list_objects(self, container, format='text',
+ limit=None, marker=None, prefix=None, delimiter=None,
+ path=None, include_trashed=False, params={}, account=None,
+ **headers):
"""returns a list with the container objects"""
+ account = account or self.account
params.update({'limit':limit, 'marker':marker, 'prefix':prefix,
'delimiter':delimiter, 'path':path})
- l = self._list('/' + container, format, params, **headers)
+ l = self._list('/%s/%s' % (account, container), format, params,
+ **headers)
#TODO support filter trashed with xml also
if format != 'xml' and not include_trashed:
l = self._filter_trashed(l)
return l
- def create_container(self, container, **meta):
+ def create_container(self, container, account=None, **meta):
"""creates a container"""
+ account = account or self.account
headers = {}
for k,v in meta.items():
headers['x-container-meta-%s' %k.strip().upper()] = v.strip()
- status, header, data = self.put('/' + container, headers=headers)
+ status, header, data = self.put('/%s/%s' % (account, container),
+ headers=headers)
if status == 202:
return False
elif status != 201:
raise Fault(data, int(status))
return True
- def delete_container(self, container, params={}):
+ def delete_container(self, container, params={}, account=None):
"""deletes a container"""
- return self.delete('/' + container, params=params)
+ account = account or self.account
+ return self.delete('/%s/%s' % (account, container), params=params)
- def retrieve_container_metadata(self, container, restricted=False, **params):
+ def retrieve_container_metadata(self, container, restricted=False,
+ account=None, **params):
"""returns the container metadata"""
+ account = account or self.account
prefix = 'x-container-meta-' if restricted else None
- return self._get_metadata('/%s' % container, prefix, params)
+ return self._get_metadata('/%s/%s' % (account, container), prefix,
+ params)
- def update_container_metadata(self, container, **meta):
+ def update_container_metadata(self, container, account=None, **meta):
"""unpdates the container metadata"""
- return self._update_metadata('/' + container, 'container', **meta)
+ account = account or self.account
+ return self._update_metadata('/%s/%s' % (account, container),
+ 'container', **meta)
- def delete_container_metadata(self, container, meta=[]):
+ def delete_container_metadata(self, container, meta=[], account=None):
"""deletes the container metadata"""
- path = '/%s' % (container)
+ account = account or self.account
+ path = '/%s/%s' % (account, container)
return self._delete_metadata(path, 'container', meta)
# Storage Object Services
def request_object(self, container, object, format='text', params={},
- **headers):
+ account=None, **headers):
"""returns tuple containing the status, headers and data response for an object request"""
- path = '/%s/%s' % (container, object)
+ account = account or self.account
+ path = '/%s/%s/%s' % (account, container, object)
status, headers, data = self.get(path, format, headers, params)
return status, headers, data
def retrieve_object(self, container, object, format='text', params={},
- **headers):
+ account=None, **headers):
"""returns an object's data"""
- t = self.request_object(container, object, format, params, **headers)
- return t[2]
+ account = account or self.account
+ t = self.request_object(container, object, format, params, account,
+ **headers)
+ data = t[2]
+ if format == 'json':
+ data = json.loads(data) if data else ''
+ elif format == 'xml':
+ data = minidom.parseString(data)
+ return data
+
+ def retrieve_object_hashmap(self, container, object, params={},
+ account=None, **headers):
+ """returns the hashmap representing object's data"""
+ args = locals().copy()
+ for elem in ['self', 'container', 'object']:
+ args.pop(elem)
+ return self.retrieve_object(container, object, format='json', **args)
- def create_directory_marker(self, container, object):
+ def create_directory_marker(self, container, object, account=None):
"""creates a dierectory marker"""
+ account = account or self.account
if not object:
raise Fault('Directory markers have to be nested in a container')
h = {'content_type':'application/directory'}
- return self.create_zero_length_object(container, object, **h)
+ return self.create_zero_length_object(container, object, account=account,
+ **h)
def create_object(self, container, object, f=stdin, format='text', meta={},
- etag=None, content_type=None, content_encoding=None,
- content_disposition=None, **headers):
+ params={}, etag=None, content_type=None, content_encoding=None,
+ content_disposition=None, account=None, **headers):
"""creates a zero-length object"""
- path = '/%s/%s' % (container, object)
+ account = account or self.account
+ path = '/%s/%s/%s' % (account, container, object)
for k, v in headers.items():
- if not v:
+ if v == None:
headers.pop(k)
l = ['etag', 'content_encoding', 'content_disposition', 'content_type']
l = [elem for elem in l if eval(elem)]
for elem in l:
headers.update({elem:eval(elem)})
-
+ headers.setdefault('content-type', 'application/octet-stream')
+
for k,v in meta.items():
headers['x-object-meta-%s' %k.strip()] = v.strip()
data = f.read() if f else None
- return self.put(path, data, format, headers=headers)
+ return self.put(path, data, format, headers=headers, params=params)
def create_zero_length_object(self, container, object, meta={}, etag=None,
content_type=None, content_encoding=None,
- content_disposition=None, **headers):
- args = locals()
- for elem in ['self', 'container', 'headers']:
+ content_disposition=None, account=None,
+ **headers):
+ account = account or self.account
+ args = locals().copy()
+ for elem in ['self', 'container', 'headers', 'account']:
args.pop(elem)
args.update(headers)
- return self.create_object(container, f=None, **args)
-
- def update_object(self, container, object, f=stdin, offset=None, meta={},
- content_length=None, content_type=None,
- content_encoding=None, content_disposition=None,
- **headers):
- path = '/%s/%s' % (container, object)
+ return self.create_object(container, account=account, f=None, **args)
+
+ def update_object(self, container, object, f=stdin,
+ offset=None, meta={}, params={}, content_length=None,
+ content_type=None, content_encoding=None,
+ content_disposition=None, account=None, **headers):
+ account = account or self.account
+ path = '/%s/%s/%s' % (account, container, object)
for k, v in headers.items():
- if not v:
+ if v == None:
headers.pop(k)
l = ['content_encoding', 'content_disposition', 'content_type',
l = [elem for elem in l if eval(elem)]
for elem in l:
headers.update({elem:eval(elem)})
-
+
if 'content_range' not in headers.keys():
if offset != None:
headers['content_range'] = 'bytes %s-/*' % offset
for k,v in meta.items():
headers['x-object-meta-%s' %k.strip()] = v.strip()
data = f.read() if f else None
- return self.post(path, data, headers=headers)
+ return self.post(path, data, headers=headers, params=params)
+
+ def update_object_using_chunks(self, container, object, f=stdin,
+ blocksize=1024, offset=None, meta={},
+ params={}, content_type=None, content_encoding=None,
+ content_disposition=None, account=None, **headers):
+ """updates an object (incremental upload)"""
+ account = account or self.account
+ path = '/%s/%s/%s' % (account, container, object)
+ headers = headers if not headers else {}
+ l = ['content_type', 'content_encoding', 'content_disposition']
+ l = [elem for elem in l if eval(elem)]
+ for elem in l:
+ headers.update({elem:eval(elem)})
+
+ if offset != None:
+ headers['content_range'] = 'bytes %s-/*' % offset
+ else:
+ headers['content_range'] = 'bytes */*'
+
+ for k,v in meta.items():
+ v = v.strip()
+ headers['x-object-meta-%s' %k.strip()] = v
+ return self._chunked_transfer(path, 'POST', f, headers=headers,
+ blocksize=blocksize, params=params)
def _change_obj_location(self, src_container, src_object, dst_container,
- dst_object, remove=False, public=False, **meta):
- path = '/%s/%s' % (dst_container, dst_object)
- headers = {}
+ dst_object, remove=False, meta={}, account=None,
+ content_type=None, **headers):
+ account = account or self.account
+ path = '/%s/%s/%s' % (account, dst_container, dst_object)
+ headers = {} if not headers else headers
+ params = {}
for k, v in meta.items():
- headers['x-object-meta-%s' % k] = v
+ headers['x-object-meta-%s' % k] = v
if remove:
headers['x-move-from'] = '/%s/%s' % (src_container, src_object)
else:
headers['x-copy-from'] = '/%s/%s' % (src_container, src_object)
- self._set_public_header(headers, public)
- self.headers = headers if headers else None
- headers['content-length'] = 0
- return self.put(path, headers=headers)
+ headers['content_length'] = 0
+ if content_type:
+ headers['content_type'] = content_type
+ else:
+ params['ignore_content_type'] = ''
+ return self.put(path, headers=headers, params=params)
- def copy_object(self, src_container, src_object, dst_container,
- dst_object, public=False, **meta):
+ def copy_object(self, src_container, src_object, dst_container, dst_object,
+ meta={}, account=None, content_type=None, **headers):
+ """copies an object"""
+ account = account or self.account
return self._change_obj_location(src_container, src_object,
- dst_container, dst_object, False,
- public, **meta)
+ dst_container, dst_object, account=account,
+ remove=False, meta=meta,
+ content_type=content_type, **headers)
def move_object(self, src_container, src_object, dst_container,
- dst_object, public=False, **meta):
+ dst_object, meta={}, account=None,
+ content_type=None, **headers):
+ """moves an object"""
+ account = account or self.account
return self._change_obj_location(src_container, src_object,
- dst_container, dst_object, True,
- public, **meta)
+ dst_container, dst_object,
+ account=account, remove=True,
+ meta=meta, content_type=content_type,
+ **headers)
- def delete_object(self, container, object, params={}):
- return self.delete('/%s/%s' % (container, object), params=params)
+ def delete_object(self, container, object, params={}, account=None):
+ """deletes an object"""
+ account = account or self.account
+ return self.delete('/%s/%s/%s' % (account, container, object),
+ params=params)
def retrieve_object_metadata(self, container, object, restricted=False,
- version=None):
+ version=None, account=None):
"""
set restricted to True to get only user defined metadata
"""
- path = '/%s/%s' % (container, object)
+ account = account or self.account
+ path = '/%s/%s/%s' % (account, container, object)
prefix = 'x-object-meta-' if restricted else None
params = {'version':version} if version else {}
return self._get_metadata(path, prefix, params=params)
- def update_object_metadata(self, container, object, **meta):
- path = '/%s/%s' % (container, object)
+ def update_object_metadata(self, container, object, account=None,
+ **meta):
+ """
+ updates object's metadata
+ """
+ account = account or self.account
+ path = '/%s/%s/%s' % (account, container, object)
return self._update_metadata(path, 'object', **meta)
- def delete_object_metadata(self, container, object, meta=[]):
- path = '/%s/%s' % (container, object)
+ def delete_object_metadata(self, container, object, meta=[], account=None):
+ """
+ deletes object's metadata
+ """
+ account = account or self.account
+ path = '/%s/%s' % (account, container, object)
return self._delete_metadata(path, 'object', meta)
class Pithos_Client(OOS_Client):
"""Pithos Storage Client. Extends OOS_Client"""
- def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
- blocksize=1024):
- """perfomrs a chunked request"""
- http = HTTPConnection(self.host)
-
- # write header
- path = '/%s/%s%s' % (self.api, self.account, path)
- http.putrequest(method, path)
- http.putheader('x-auth-token', self.token)
- http.putheader('content-type', 'application/octet-stream')
- http.putheader('transfer-encoding', 'chunked')
- if headers:
- for header,value in headers.items():
- http.putheader(header, value)
- http.endheaders()
-
- # write body
- data = ''
- while True:
- if f.closed:
- break
- block = f.read(blocksize)
- if block == '':
- break
- data = '%s\r\n%s\r\n' % (hex(len(block)), block)
- try:
- http.send(data)
- except:
- #retry
- http.send(data)
- data = '0x0\r\n'
- try:
- http.send(data)
- except:
- #retry
- http.send(data)
-
- # get response
- resp = http.getresponse()
-
- headers = dict(resp.getheaders())
-
- if self.verbose:
- print '%d %s' % (resp.status, resp.reason)
- for key, val in headers.items():
- print '%s: %s' % (key.capitalize(), val)
- print
-
- length = resp.getheader('Content-length', None)
- data = resp.read(length)
- if self.debug:
- print data
- print
-
- if int(resp.status) in ERROR_CODES.keys():
- raise Fault(data, int(resp.status))
-
- #print '*', resp.status, headers, data
- return resp.status, headers, data
-
def _update_metadata(self, path, entity, **meta):
"""
adds new and updates the values of previously set metadata
# Storage Account Services
def list_containers(self, format='text', if_modified_since=None,
- if_unmodified_since=None, limit=1000, marker=None,
- until=None):
+ if_unmodified_since=None, limit=None, marker=None,
+ shared=False, until=None, account=None):
"""returns a list with the account containers"""
- params = {'until':until} if until else None
+ account = account or self.account
+ params = {'until':until} if until else {}
+ if shared:
+ params['shared'] = None
headers = {'if-modified-since':if_modified_since,
'if-unmodified-since':if_unmodified_since}
- return OOS_Client.list_containers(self, format=format, limit=limit,
- marker=marker, params=params,
- **headers)
+ return OOS_Client.list_containers(self, account=account, format=format,
+ limit=limit, marker=marker,
+ params=params, **headers)
- def retrieve_account_metadata(self, restricted=False, until=None):
+ def retrieve_account_metadata(self, restricted=False, until=None,
+ account=None):
"""returns the account metadata"""
+ account = account or self.account
params = {'until':until} if until else {}
- return OOS_Client.retrieve_account_metadata(self, restricted=restricted,
- **params)
+ return OOS_Client.retrieve_account_metadata(self, account=account,
+ restricted=restricted,
+ **params)
- def set_account_groups(self, **groups):
+ def set_account_groups(self, account=None, **groups):
"""create account groups"""
+ account = account or self.account
+ path = '/%s' % account
headers = {}
- for key, val in groups.items():
- headers['x-account-group-%s' % key] = val
+ for k, v in groups.items():
+ headers['x-account-group-%s' % k] = v
params = {'update':None}
- return self.post('', headers=headers, params=params)
+ return self.post(path, headers=headers, params=params)
- def retrieve_account_groups(self):
+ def retrieve_account_groups(self, account=None):
"""returns the account groups"""
- meta = self.retrieve_account_metadata()
+ account = account or self.account
+ meta = self.retrieve_account_metadata(account=account)
prefix = 'x-account-group-'
prefixlen = len(prefix)
groups = {}
groups[key] = val
return groups
- def unset_account_groups(self, groups=[]):
+ def unset_account_groups(self, groups=[], account=None):
"""delete account groups"""
+ account = account or self.account
+ path = '/%s' % account
headers = {}
for elem in groups:
headers['x-account-group-%s' % elem] = ''
params = {'update':None}
- return self.post('', headers=headers, params=params)
+ return self.post(path, headers=headers, params=params)
- def reset_account_groups(self, **groups):
+ def reset_account_groups(self, account=None, **groups):
"""overrides account groups"""
+ account = account or self.account
+ path = '/%s' % account
headers = {}
- for key, val in groups.items():
- headers['x-account-group-%s' % key] = val
- meta = self.retrieve_account_metadata()
- headers.update(meta)
- return self.post('', headers=headers)
+ for k, v in groups.items():
+ v = v.strip()
+ headers['x-account-group-%s' % k] = v
+ meta = self.retrieve_account_metadata(restricted=True)
+ prefix = 'x-account-meta-'
+ for k,v in meta.items():
+ k = '%s%s' % (prefix, k)
+ headers[k] = v
+ return self.post(path, headers=headers)
# Storage Container Services
- def list_objects(self, container, format='text', limit=10000, marker=None,
- prefix=None, delimiter=None, path=None,
- include_trashed=False, params={}, if_modified_since=None,
- if_unmodified_since=None, meta={}, until=None):
+ def list_objects(self, container, format='text',
+ limit=None, marker=None, prefix=None, delimiter=None,
+ path=None, shared=False, include_trashed=False, params={},
+ if_modified_since=None, if_unmodified_since=None, meta='',
+ until=None, account=None):
"""returns a list with the container objects"""
+ account = account or self.account
params = {'until':until, 'meta':meta}
- args = locals()
+ if shared:
+ params['shared'] = None
+ args = locals().copy()
for elem in ['self', 'container', 'params', 'until', 'meta']:
args.pop(elem)
- return OOS_Client.list_objects(self, container, params=params,
- **args)
+ return OOS_Client.list_objects(self, container, params=params, **args)
def retrieve_container_metadata(self, container, restricted=False,
- until=None):
+ until=None, account=None):
"""returns container's metadata"""
+ account = account or self.account
params = {'until':until} if until else {}
return OOS_Client.retrieve_container_metadata(self, container,
+ account=account,
restricted=restricted,
**params)
- def set_container_policies(self, container, **policies):
+ def set_container_policies(self, container, account=None,
+ **policies):
"""sets containers policies"""
- path = '/%s' % (container)
+ account = account or self.account
+ path = '/%s/%s' % (account, container)
headers = {}
- print ''
for key, val in policies.items():
headers['x-container-policy-%s' % key] = val
return self.post(path, headers=headers)
- def delete_container(self, container, until=None):
+ def update_container_data(self, container, f=stdin):
+ """adds blocks of data to the container"""
+ account = self.account
+ path = '/%s/%s' % (account, container)
+ params = {'update': None}
+ headers = {'content_type': 'application/octet-stream'}
+ data = f.read() if f else None
+ headers['content_length'] = len(data)
+ return self.post(path, data, headers=headers, params=params)
+
+ def delete_container(self, container, until=None, account=None):
"""deletes a container or the container history until the date provided"""
+ account = account or self.account
params = {'until':until} if until else {}
- return OOS_Client.delete_container(self, container, params)
+ return OOS_Client.delete_container(self, container, account=account,
+ params=params)
# Storage Object Services
- def retrieve_object(self, container, object, params={}, format='text', range=None,
- if_range=None, if_match=None, if_none_match=None,
+ def retrieve_object(self, container, object, params={}, format='text',
+ range=None, if_range=None,
+ if_match=None, if_none_match=None,
if_modified_since=None, if_unmodified_since=None,
- **headers):
+ account=None, **headers):
"""returns an object"""
+ account = account or self.account
headers={}
l = ['range', 'if_range', 'if_match', 'if_none_match',
'if_modified_since', 'if_unmodified_since']
l = [elem for elem in l if eval(elem)]
for elem in l:
headers.update({elem:eval(elem)})
- return OOS_Client.retrieve_object(self, container, object, format=format,
+ if format != 'text':
+ params['hashmap'] = None
+ return OOS_Client.retrieve_object(self, container, object,
+ account=account, format=format,
params=params, **headers)
- def retrieve_object_version(self, container, object, version, detail=False,
- range=None, if_range=None, if_match=None,
- if_none_match=None, if_modified_since=None,
- if_unmodified_since=None):
+ def retrieve_object_version(self, container, object, version,
+ format='text', range=None, if_range=None,
+ if_match=None, if_none_match=None,
+ if_modified_since=None, if_unmodified_since=None,
+ account=None):
"""returns a specific object version"""
- args = locals()
+ account = account or self.account
+ args = locals().copy()
l = ['self', 'container', 'object']
for elem in l:
args.pop(elem)
params = {'version':version}
- return self.retrieve_object(container, object, params, **args)
+ return self.retrieve_object(container, object, params=params, **args)
def retrieve_object_versionlist(self, container, object, range=None,
if_range=None, if_match=None,
if_none_match=None, if_modified_since=None,
- if_unmodified_since=None):
+ if_unmodified_since=None, account=None):
"""returns the object version list"""
- args = locals()
+ account = account or self.account
+ args = locals().copy()
l = ['self', 'container', 'object']
for elem in l:
args.pop(elem)
-
+
return self.retrieve_object_version(container, object, version='list',
- detail=True, **args)
-
- def create_zero_length_object(self, container, object, meta={},
- etag=None, content_type=None, content_encoding=None,
- content_disposition=None, x_object_manifest=None,
- x_object_sharing=None, x_object_public=None):
+ format='json', **args)
+
+ def create_zero_length_object(self, container, object,
+ meta={}, etag=None, content_type=None,
+ content_encoding=None,
+ content_disposition=None,
+ x_object_manifest=None, x_object_sharing=None,
+ x_object_public=None, account=None):
"""createas a zero length object"""
- args = locals()
+ account = account or self.account
+ args = locals().copy()
for elem in ['self', 'container', 'object']:
args.pop(elem)
return OOS_Client.create_zero_length_object(self, container, object,
**args)
- def create_object(self, container, object, f=stdin, meta={},
- etag=None, content_type=None, content_encoding=None,
- content_disposition=None, x_object_manifest=None,
- x_object_sharing=None, x_object_public=None):
+ def create_object(self, container, object, f=stdin, format='text',
+ meta={}, params={}, etag=None, content_type=None,
+ content_encoding=None, content_disposition=None,
+ x_object_manifest=None, x_object_sharing=None,
+ x_object_public=None, account=None):
"""creates an object"""
- args = locals()
+ account = account or self.account
+ args = locals().copy()
for elem in ['self', 'container', 'object']:
args.pop(elem)
+ if format != 'text':
+ params.update({'hashmap':None})
return OOS_Client.create_object(self, container, object, **args)
- def create_object_using_chunks(self, container, object, f=stdin,
- blocksize=1024, meta={}, etag=None,
- content_type=None, content_encoding=None,
- content_disposition=None,
- x_object_sharing=None,
- x_object_manifest=None,
- x_object_public=None):
+ def create_object_using_chunks(self, container, object,
+ f=stdin, blocksize=1024, meta={}, etag=None,
+ content_type=None, content_encoding=None,
+ content_disposition=None,
+ x_object_sharing=None, x_object_manifest=None,
+ x_object_public=None, account=None):
"""creates an object (incremental upload)"""
- path = '/%s/%s' % (container, object)
+ account = account or self.account
+ path = '/%s/%s/%s' % (account, container, object)
headers = {}
l = ['etag', 'content_type', 'content_encoding', 'content_disposition',
'x_object_sharing', 'x_object_manifest', 'x_object_public']
l = [elem for elem in l if eval(elem)]
for elem in l:
headers.update({elem:eval(elem)})
+ headers.setdefault('content-type', 'application/octet-stream')
for k,v in meta.items():
- headers['x-object-meta-%s' %k.strip()] = v.strip()
+ v = v.strip()
+ headers['x-object-meta-%s' %k.strip()] = v
return self._chunked_transfer(path, 'PUT', f, headers=headers,
blocksize=blocksize)
- def create_object_by_hashmap(container, object, f=stdin, format='json',
+ def create_object_by_hashmap(self, container, object, hashmap={},
meta={}, etag=None, content_encoding=None,
content_disposition=None, content_type=None,
x_object_sharing=None, x_object_manifest=None,
- x_object_public = None):
+ x_object_public = None, account=None):
"""creates an object by uploading hashes representing data instead of data"""
- args = locals()
- for elem in ['self', 'container', 'object']:
+ account = account or self.account
+ args = locals().copy()
+ for elem in ['self', 'container', 'object', 'hashmap']:
args.pop(elem)
- data = f.read() if f else None
- if data and format == 'json':
- try:
- data = eval(data)
- data = json.dumps(data)
- except SyntaxError:
- raise Fault('Invalid formatting')
+ try:
+ data = json.dumps(hashmap)
+ except SyntaxError:
+ raise Fault('Invalid formatting')
+ args['params'] = {'hashmap':None}
+ args['format'] = 'json'
- #TODO check with xml
- return self.create_object(container, object, **args)
+ return self.create_object(container, object, f=StringIO(data), **args)
- def create_manifestation(self, container, object, manifest):
+ def create_manifestation(self, container, object, manifest, account=None):
"""creates a manifestation"""
+ account = account or self.account
headers={'x_object_manifest':manifest}
- return self.create_object(container, object, f=None, **headers)
+ return self.create_object(container, object, f=None, account=account,
+ **headers)
- def update_object(self, container, object, f=stdin, offset=None, meta={},
- content_length=None, content_type=None, content_range=None,
+ def update_object(self, container, object, f=stdin,
+ offset=None, meta={}, replace=False, content_length=None,
+ content_type=None, content_range=None,
content_encoding=None, content_disposition=None,
x_object_bytes=None, x_object_manifest=None,
- x_object_sharing=None, x_object_public=None):
+ x_object_sharing=None, x_object_public=None,
+ x_source_object=None, account=None):
"""updates an object"""
- spath = '/%s/%s' % (container, object)
- args = locals()
- for elem in ['self', 'container', 'object']:
+ account = account or self.account
+ args = locals().copy()
+ for elem in ['self', 'container', 'object', 'replace']:
args.pop(elem)
-
+ if not replace:
+ args['params'] = {'update':None}
return OOS_Client.update_object(self, container, object, **args)
-
+
def update_object_using_chunks(self, container, object, f=stdin,
- blocksize=1024, offset=None, meta={},
- content_type=None, content_encoding=None,
- content_disposition=None, x_object_bytes=None,
- x_object_manifest=None, x_object_sharing=None,
- x_object_public=None):
+ blocksize=1024, offset=None, meta={},
+ replace=False, content_type=None, content_encoding=None,
+ content_disposition=None, x_object_bytes=None,
+ x_object_manifest=None, x_object_sharing=None,
+ x_object_public=None, account=None):
"""updates an object (incremental upload)"""
- path = '/%s/%s' % (container, object)
- headers = {}
- l = ['content_type', 'content_encoding', 'content_disposition',
- 'x_object_bytes', 'x_object_manifest', 'x_object_sharing',
- 'x_object_public']
- l = [elem for elem in l if eval(elem)]
- for elem in l:
- headers.update({elem:eval(elem)})
-
- if offset != None:
- headers['content_range'] = 'bytes %s-/*' % offset
- else:
- headers['content_range'] = 'bytes */*'
-
- for k,v in meta.items():
- headers['x-object-meta-%s' %k.strip()] = v.strip()
+ account = account or self.account
+ args = locals().copy()
+ for elem in ['self', 'container', 'object', 'replace']:
+ args.pop(elem)
+ if not replace:
+ args['params'] = {'update':None}
+ return OOS_Client.update_object_using_chunks(self, container, object, **args)
+
+ def update_from_other_source(self, container, object, source,
+ offset=None, meta={}, content_range=None,
+ content_encoding=None, content_disposition=None,
+ x_object_bytes=None, x_object_manifest=None,
+ x_object_sharing=None, x_object_public=None, account=None):
+ """updates an object"""
+ account = account or self.account
+ args = locals().copy()
+ for elem in ['self', 'container', 'object', 'source']:
+ args.pop(elem)
- return self._chunked_transfer(path, 'POST', f, headers=headers,
- blocksize=blocksize)
+ args['x_source_object'] = source
+ return self.update_object(container, object, f=None, **args)
- def delete_object(self, container, object, until=None):
+ def delete_object(self, container, object, until=None, account=None):
"""deletes an object or the object history until the date provided"""
+ account = account or self.account
params = {'until':until} if until else {}
- return OOS_Client.delete_object(self, container, object, params)
+ return OOS_Client.delete_object(self, container, object, params, account)
def trash_object(self, container, object):
"""trashes an object"""
+ account = account or self.account
path = '/%s/%s' % (container, object)
meta = {'trash':'true'}
return self._update_metadata(path, 'object', **meta)
- def restore_object(self, container, object):
+ def restore_object(self, container, object, account=None):
"""restores a trashed object"""
- return self.delete_object_metadata(container, object, ['trash'])
-
- def _set_public_header(self, headers, public=False):
- """sets the public header"""
- if not headers:
- headers = {}
- if public == None:
- return
- else:
- headers['x-object-public'] = public if public else ''
+ account = account or self.account
+ return self.delete_object_metadata(container, object, account, ['trash'])
- def publish_object(self, container, object):
+ def publish_object(self, container, object, account=None):
"""sets a previously created object publicly accessible"""
- path = '/%s/%s' % (container, object)
- headers = {'content_range':'bytes */*'}
- self._set_public_header(headers, public=True)
- return self.post(path, headers=headers)
+ account = account or self.account
+ path = '/%s/%s/%s' % (account, container, object)
+ headers = {}
+ headers['x_object_public'] = True
+ params = {'update':None}
+ return self.post(path, headers=headers, params=params)
- def unpublish_object(self, container, object):
+ def unpublish_object(self, container, object, account=None):
"""unpublish an object"""
- path = '/%s/%s' % (container, object)
- headers = {'content_range':'bytes */*'}
- self._set_public_header(headers, public=False)
- return self.post(path, headers=headers)
+ account = account or self.account
+ path = '/%s/%s/%s' % (account, container, object)
+ headers = {}
+ headers['x_object_public'] = False
+ params = {'update':None}
+ return self.post(path, headers=headers, params=params)
+
+ def copy_object(self, src_container, src_object, dst_container, dst_object,
+ meta={}, public=False, version=None, account=None,
+ content_type=None):
+ """copies an object"""
+ account = account or self.account
+ headers = {}
+ headers['x_object_public'] = public
+ if version:
+ headers['x_source_version'] = version
+ return OOS_Client.copy_object(self, src_container, src_object,
+ dst_container, dst_object, meta=meta,
+ account=account, content_type=content_type,
+ **headers)
+
+ def move_object(self, src_container, src_object, dst_container,
+ dst_object, meta={}, public=False,
+ account=None, content_type=None):
+ """moves an object"""
+ headers = {}
+ headers['x_object_public'] = public
+ return OOS_Client.move_object(self, src_container, src_object,
+ dst_container, dst_object, meta=meta,
+ account=account, content_type=content_type,
+ **headers)
+
+ def list_shared_by_others(self, limit=None, marker=None, format='text'):
+ """lists other accounts that share objects to the user"""
+ l = ['limit', 'marker']
+ params = {}
+ for elem in [elem for elem in l if eval(elem)]:
+ params[elem] = eval(elem)
+ return self._list('', format, params)
+
+ def share_object(self, container, object, l, read=True):
+ """gives access(read by default) to an object to a user/group list"""
+ action = 'read' if read else 'write'
+ sharing = '%s=%s' % (action, ','.join(l))
+ self.update_object(container, object, f=None, x_object_sharing=sharing)
+
+def _prepare_path(path, format='text', params={}):
+ full_path = '%s?format=%s' % (quote(path), format)
+
+ for k,v in params.items():
+ value = quote(str(v)) if v else ''
+ full_path = '%s&%s=%s' %(full_path, quote(k), value)
+ return full_path
+
+def _prepare_headers(headers):
+ for k,v in headers.items():
+ headers.pop(k)
+ k = k.replace('_', '-')
+ headers[quote(k)] = quote(v, safe='/=,:@ *"') if type(v) == types.StringType else v
+ return headers
+
+def _handle_response(response, verbose=False, debug=False):
+ headers = response.getheaders()
+ headers = dict((unquote(h), unquote(v)) for h,v in headers)
+
+ if verbose:
+ print '%d %s' % (response.status, response.reason)
+ for key, val in headers.items():
+ print '%s: %s' % (key.capitalize(), val)
+ print
+
+ length = response.getheader('content-length', None)
+ data = response.read(length)
+ if debug:
+ print data
+ print
+
+ if int(response.status) in ERROR_CODES.keys():
+ raise Fault(data, int(response.status))
+
+ #print '**', response.status, headers, data, '\n'
+ return response.status, headers, data
projects / pithos / blobdiff