+# Copyright 2011 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above
+# copyright notice, this list of conditions and the following
+# disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials
+# provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
from httplib import HTTPConnection, HTTP
from sys import stdin
self.status = status
class Client(object):
- def __init__(self, host, account, api='v1', verbose=False, debug=False):
+ def __init__(self, host, token, account, api='v1', verbose=False, debug=False):
"""`host` can also include a port, e.g '127.0.0.1:8000'."""
self.host = host
self.api = api
self.verbose = verbose or debug
self.debug = debug
-
+ self.token = token
+
def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
blocksize=1024):
+
http = HTTPConnection(self.host)
# write header
path = '/%s/%s%s' % (self.api, self.account, path)
http.putrequest(method, path)
- http.putheader('Content-Type', 'application/octet-stream')
- http.putheader('Transfer-Encoding', 'chunked')
+ http.putheader('x-auth-token', self.token)
+ http.putheader('content-type', 'application/octet-stream')
+ http.putheader('transfer-encoding', 'chunked')
if headers:
for header,value in headers.items():
http.putheader(header, value)
print '%s: %s' % (key.capitalize(), val)
print
- data = resp.read()
+ length = resp.getheader('Content-length', None)
+ data = resp.read(length)
if self.debug:
print data
print
- if data:
- assert data[-1] == '\n'
- #remove trailing enter
- data = data and data[:-1] or data
-
if int(resp.status) in ERROR_CODES.keys():
raise Fault(data, int(resp.status))
+ #print '*', resp.status, headers, data
return resp.status, headers, data
-
- def req(self, method, path, body=None, headers=None, format='text',
+
+ def _req(self, method, path, body=None, headers=None, format='text',
params=None):
full_path = '/%s/%s%s?format=%s' % (self.api, self.account, path,
format)
for k,v in params.items():
if v:
full_path = '%s&%s=%s' %(full_path, k, v)
+ else:
+ full_path = '%s&%s' %(full_path, k)
conn = HTTPConnection(self.host)
#encode whitespace
kwargs = {}
kwargs['headers'] = headers or {}
+ kwargs['headers']['X-Auth-Token'] = self.token
if not headers or \
- 'Transfer-Encoding' not in headers \
- or headers['Transfer-Encoding'] != 'chunked':
- kwargs['headers']['Content-Length'] = len(body) if body else 0
+ 'transfer-encoding' not in headers \
+ or headers['transfer-encoding'] != 'chunked':
+ kwargs['headers']['content-length'] = len(body) if body else 0
if body:
kwargs['body'] = body
- kwargs['headers']['Content-Type'] = 'application/octet-stream'
- #print '****', method, full_path, kwargs
+ else:
+ kwargs['headers']['content-type'] = ''
+ kwargs['headers'].setdefault('content-type', 'application/octet-stream')
try:
+ #print '*', method, full_path, kwargs
conn.request(method, full_path, **kwargs)
except socket.error, e:
raise Fault(status=503)
print '%s: %s' % (key.capitalize(), val)
print
- data = resp.read()
+ length = resp.getheader('content-length', None)
+ data = resp.read(length)
if self.debug:
print data
print
- if data:
- assert data[-1] == '\n'
- #remove trailing enter
- data = data and data[:-1] or data
-
if int(resp.status) in ERROR_CODES.keys():
raise Fault(data, int(resp.status))
#print '*', resp.status, headers, data
return resp.status, headers, data
-
+
def delete(self, path, format='text'):
- return self.req('DELETE', path, format=format)
-
+ return self._req('DELETE', path, format=format)
+
def get(self, path, format='text', headers=None, params=None):
- return self.req('GET', path, headers=headers, format=format,
+ return self._req('GET', path, headers=headers, format=format,
params=params)
-
+
def head(self, path, format='text', params=None):
- return self.req('HEAD', path, format=format, params=params)
-
- def post(self, path, body=None, format='text', headers=None):
- return self.req('POST', path, body, headers=headers, format=format)
-
+ return self._req('HEAD', path, format=format, params=params)
+
+ def post(self, path, body=None, format='text', headers=None, params=None):
+ return self._req('POST', path, body, headers=headers, format=format,
+ params=params)
+
def put(self, path, body=None, format='text', headers=None):
- return self.req('PUT', path, body, headers=headers, format=format)
-
+ return self._req('PUT', path, body, headers=headers, format=format)
+
def _list(self, path, detail=False, params=None, headers=None):
format = 'json' if detail else 'text'
status, headers, data = self.get(path, format=format, headers=headers,
params=params)
if detail:
- data = json.loads(data)
+ data = json.loads(data) if data else ''
else:
data = data.strip().split('\n')
return data
-
+
def _get_metadata(self, path, prefix=None, params=None):
status, headers, data = self.head(path, params=params)
- prefixlen = prefix and len(prefix) or 0
+ prefixlen = len(prefix) if prefix else 0
meta = {}
for key, val in headers.items():
if prefix and not key.startswith(prefix):
key = key[prefixlen:]
meta[key] = val
return meta
-
- def _set_metadata(self, path, entity, **meta):
+
+ def _update_metadata(self, path, entity, **meta):
+ """
+ adds new and updates the values of previously set metadata
+ """
+ params = {'update':None}
headers = {}
- for key, val in meta.items():
- http_key = 'X-%s-Meta-%s' %(entity.capitalize(), key.capitalize())
- headers[http_key] = val
- self.post(path, headers=headers)
-
+ prefix = 'x-%s-meta-' % entity
+ for k,v in meta.items():
+ k = '%s%s' % (prefix, k)
+ headers[k] = v
+ return self.post(path, headers=headers, params=params)
+
+ def _delete_metadata(self, path, entity, meta=[]):
+ """
+ delete previously set metadata
+ """
+ params = {'update':None}
+ headers = {}
+ prefix = 'x-%s-meta-' % entity
+ for m in meta:
+ headers['%s%s' % (prefix, m)] = None
+ return self.post(path, headers=headers)
+
# Storage Account Services
-
+
def list_containers(self, detail=False, params=None, headers=None):
return self._list('', detail, params, headers)
-
+
def account_metadata(self, restricted=False, until=None):
- prefix = restricted and 'x-account-meta-' or None
- params = until and {'until':until} or None
+ prefix = 'x-account-meta-' if restricted else None
+ params = {'until':until} if until else None
return self._get_metadata('', prefix, params=params)
-
+
def update_account_metadata(self, **meta):
- self._set_metadata('', 'account', **meta)
-
+ return self._update_metadata('', 'account', **meta)
+
+ def delete_account_metadata(self, meta=[]):
+ return self._delete_metadata('', 'account', meta)
+
+ def set_account_groups(self, **groups):
+ """
+ create account groups
+ """
+ headers = {}
+ for key, val in groups.items():
+ headers['x-account-group-%s' % key] = val
+ params = {'update':None}
+ return self.post('', headers=headers, params=params)
+
+ def unset_account_groups(self, groups=[]):
+ """
+ delete account groups
+ """
+ headers = {}
+ for elem in groups:
+ headers['x-account-group-%s' % elem] = ''
+ params = {'update':None}
+ return self.post('', headers=headers, params=params)
+
# Storage Container Services
-
- def list_objects(self, container, detail=False, params=None, headers=None):
- return self._list('/' + container, detail, params, headers)
-
- def create_container(self, container, headers=None):
+
+ def _filter(self, l, d):
+ """
+ filter out from l elements having the metadata values provided
+ """
+ ll = l
+ for elem in l:
+ if type(elem) == types.DictionaryType:
+ for key in d.keys():
+ k = 'x_object_meta_%s' % key
+ if k in elem.keys() and elem[k] == d[key]:
+ ll.remove(elem)
+ break
+ return ll
+
+ def _filter_trashed(self, l):
+ return self._filter(l, {'trash':'true'})
+
+ def list_objects(self, container, detail=False, headers=None,
+ include_trashed=False, **params):
+ l = self._list('/' + container, detail, params, headers)
+ if not include_trashed:
+ l = self._filter_trashed(l)
+ return l
+
+ def create_container(self, container, headers=None, **meta):
+ for k,v in meta.items():
+ headers['x-container-meta-%s' %k.strip().upper()] = v.strip()
status, header, data = self.put('/' + container, headers=headers)
if status == 202:
return False
elif status != 201:
raise Fault(data, int(status))
return True
-
+
def delete_container(self, container):
- self.delete('/' + container)
-
+ return self.delete('/' + container)
+
def retrieve_container_metadata(self, container, restricted=False,
until=None):
- prefix = restricted and 'x-container-meta-' or None
- params = until and {'until':until} or None
+ prefix = 'x-container-meta-' if restricted else None
+ params = {'until':until} if until else None
return self._get_metadata('/%s' % container, prefix, params=params)
-
+
def update_container_metadata(self, container, **meta):
- self._set_metadata('/' + container, 'container', **meta)
-
+ return self._update_metadata('/' + container, 'container', **meta)
+
+ def delete_container_metadata(self, container, meta=[]):
+ path = '/%s' % (container)
+ return self._delete_metadata(path, 'container', meta)
+
+ def set_container_policies(self, container, **policies):
+ path = '/%s' % (container)
+ headers = {}
+ print ''
+ for key, val in policies.items():
+ headers['x-container-policy-%s' % key] = val
+ return self.post(path, headers=headers)
+
# Storage Object Services
-
+
def retrieve_object(self, container, object, detail=False, headers=None,
version=None):
path = '/%s/%s' % (container, object)
format = 'json' if detail else 'text'
- params = version and {'version':version} or None
+ params = {'version':version} if version else None
status, headers, data = self.get(path, format, headers, params)
return data
-
+
+ def create_directory_marker(self, container, object):
+ if not object:
+ raise Fault('Directory markers have to be nested in a container')
+ h = {'Content-Type':'application/directory'}
+ return self.create_object(container, object, f=None, headers=h)
+
+ def _set_public_header(self, headers, public=False):
+ """
+ sets the public header
+ """
+ if public == None:
+ return
+ elif public:
+ headers['x-object-public'] = public
+ else:
+ headers['x-object-public'] = ''
+
def create_object(self, container, object, f=stdin, chunked=False,
- blocksize=1024, headers=None):
+ blocksize=1024, headers={}, use_hashes=False,
+ public=None, **meta):
"""
creates an object
if f is None then creates a zero length object
if f is stdin or chunked is set then performs chunked transfer
"""
path = '/%s/%s' % (container, object)
- if not chunked and f != stdin:
- data = f and f.read() or None
- return self.put(path, data, headers=headers)
+ for k,v in meta.items():
+ headers['x-object-meta-%s' %k.strip().upper()] = v.strip()
+ self._set_public_header(headers, public)
+ headers = headers if headers else None
+ if not chunked:
+ format = 'json' if use_hashes else 'text'
+ data = f.read() if f else None
+ if data:
+ if format == 'json':
+ try:
+ data = eval(data)
+ data = json.dumps(data)
+ except SyntaxError:
+ raise Fault('Invalid formatting')
+ return self.put(path, data, headers=headers, format=format)
else:
return self._chunked_transfer(path, 'PUT', f, headers=headers,
blocksize=1024)
-
+
+ def update_object_data(self, container, object, data=None, headers={},
+ offset=None, public=None, **meta):
+ path = '/%s/%s' % (container, object)
+ for k,v in meta.items():
+ headers['x-object-meta-%s' %k.strip()] = v.strip()
+ if 'content-range' not in headers.keys():
+ if offset:
+ headers['content-range'] = 'bytes %s-/*' % offset
+ else:
+ headers['content-range'] = 'bytes */*'
+ self._set_public_header(headers, public)
+ headers = headers if headers else None
+ return self.post(path, data, headers=headers)
+
def update_object(self, container, object, f=stdin, chunked=False,
- blocksize=1024, headers=None):
- if not f:
- return
+ blocksize=1024, headers={}, offset=None, public=None,
+ **meta):
path = '/%s/%s' % (container, object)
+ for k,v in meta.items():
+ headers['x-object-meta-%s' %k.strip()] = v.strip()
+ if offset:
+ headers['content-range'] = 'bytes %s-/*' % offset
+ else:
+ headers['content-range'] = 'bytes */*'
+ self._set_public_header(headers, public)
+ headers = headers if headers else None
if not chunked and f != stdin:
- data = f.read()
- self.post(path, data, headers=headers)
+ data = f.read() if f else None
+ return self.post(path, data, headers=headers)
else:
- self._chunked_transfer(path, 'POST', f, headers=headers,
+ return self._chunked_transfer(path, 'POST', f, headers=headers,
blocksize=1024)
-
+
def _change_obj_location(self, src_container, src_object, dst_container,
- dst_object, remove=False, headers=None):
+ dst_object, remove=False, public=False, **meta):
path = '/%s/%s' % (dst_container, dst_object)
- if not headers:
- headers = {}
+ headers = {}
+ for k, v in meta.items():
+ headers['x-object-meta-%s' % k] = v
if remove:
- headers['X-Move-From'] = '/%s/%s' % (src_container, src_object)
+ headers['x-move-from'] = '/%s/%s' % (src_container, src_object)
else:
- headers['X-Copy-From'] = '/%s/%s' % (src_container, src_object)
- headers['Content-Length'] = 0
- self.put(path, headers=headers)
-
+ headers['x-copy-from'] = '/%s/%s' % (src_container, src_object)
+ self._set_public_header(headers, public)
+ self.headers = headers if headers else None
+ headers['content-length'] = 0
+ return self.put(path, headers=headers)
+
def copy_object(self, src_container, src_object, dst_container,
- dst_object, headers=None):
- self._change_obj_location(src_container, src_object,
- dst_container, dst_object, headers)
-
+ dst_object, public=False, **meta):
+ return self._change_obj_location(src_container, src_object,
+ dst_container, dst_object, False,
+ public, **meta)
+
def move_object(self, src_container, src_object, dst_container,
- dst_object, headers=None):
- self._change_obj_location(src_container, src_object,
- dst_container, dst_object, True, headers)
-
+ dst_object, public=False, **meta):
+ return self._change_obj_location(src_container, src_object,
+ dst_container, dst_object, True,
+ public, **meta)
+
def delete_object(self, container, object):
- self.delete('/%s/%s' % (container, object))
-
+ return self.delete('/%s/%s' % (container, object))
+
def retrieve_object_metadata(self, container, object, restricted=False,
version=None):
+ """
+ set restricted to True to get only user defined metadata
+ """
path = '/%s/%s' % (container, object)
- prefix = restricted and 'x-object-meta-' or None
- params = version and {'version':version} or None
+ prefix = 'x-object-meta-' if restricted else None
+ params = {'version':version} if version else None
return self._get_metadata(path, prefix, params=params)
-
+
def update_object_metadata(self, container, object, **meta):
path = '/%s/%s' % (container, object)
- self._set_metadata(path, 'object', **meta)
+ return self._update_metadata(path, 'object', **meta)
+
+ def delete_object_metadata(self, container, object, meta=[]):
+ path = '/%s/%s' % (container, object)
+ return self._delete_metadata(path, 'object', meta)
+
+ def trash_object(self, container, object):
+ """
+ trashes an object
+ actually resets all object metadata with trash = true
+ """
+ path = '/%s/%s' % (container, object)
+ meta = {'trash':'true'}
+ return self._update_metadata(path, 'object', **meta)
+
+ def restore_object(self, container, object):
+ """
+ restores a trashed object
+ actualy removes trash object metadata info
+ """
+ return self.delete_object_metadata(container, object, ['trash'])
+
+ def publish_object(self, container, object):
+ """
+ sets a previously created object publicly accessible
+ """
+ path = '/%s/%s' % (container, object)
+ headers = {'content-range':'bytes */*'}
+ self._set_public_header(headers, public=True)
+ return self.post(path, headers=headers)
+
+ def unpublish_object(self, container, object):
+ """
+ unpublish an object
+ """
+ path = '/%s/%s' % (container, object)
+ headers = {'content-range':'bytes */*'}
+ self._set_public_header(headers, public=False)
+ return self.post(path, headers=headers)