from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, ItemNotFound, Conflict,
LengthRequired, PreconditionFailed, RangeNotSatisfiable, UnprocessableEntity)
-from pithos.api.util import (format_meta_key, printable_meta_dict, get_account_meta,
- put_account_meta, get_container_meta, put_container_meta, get_object_meta, put_object_meta,
+from pithos.api.util import (format_header_key, printable_header_dict, get_account_headers,
+ put_account_headers, get_container_headers, put_container_headers, get_object_headers, put_object_headers,
update_manifest_meta, update_sharing_meta, validate_modification_preconditions,
validate_matching_preconditions, split_container_object_string, copy_or_move_object,
get_int_parameter, get_content_length, get_content_range, get_sharing, raw_input_socket,
until = get_int_parameter(request, 'until')
try:
meta = backend.get_account_meta(request.user, v_account, until)
+ groups = backend.get_account_groups(request.user, v_account)
except NotAllowedError:
raise Unauthorized('Access denied')
response = HttpResponse(status=204)
- put_account_meta(response, meta)
+ put_account_headers(response, meta, groups)
return response
@api_method('POST')
# unauthorized (401),
# badRequest (400)
- meta = get_account_meta(request)
+ meta, groups = get_account_headers(request)
replace = True
if 'update' in request.GET:
- replace = False
+ replace = False
+ if groups:
+ try:
+ backend.update_account_groups(request.user, v_account, groups, replace)
+ except NotAllowedError:
+ raise Unauthorized('Access denied')
+ except ValueError:
+ raise BadRequest('Invalid groups header')
try:
backend.update_account_meta(request.user, v_account, meta, replace)
except NotAllowedError:
until = get_int_parameter(request, 'until')
try:
meta = backend.get_account_meta(request.user, v_account, until)
+ groups = backend.get_account_groups(request.user, v_account)
except NotAllowedError:
raise Unauthorized('Access denied')
validate_modification_preconditions(request, meta)
response = HttpResponse()
- put_account_meta(response, meta)
+ put_account_headers(response, meta, groups)
marker = request.GET.get('marker')
limit = request.GET.get('limit')
if x[1] is not None:
try:
meta = backend.get_container_meta(request.user, v_account, x[0], until)
- container_meta.append(printable_meta_dict(meta))
+ container_meta.append(printable_header_dict(meta))
except NotAllowedError:
raise Unauthorized('Access denied')
except NameError:
raise ItemNotFound('Container does not exist')
response = HttpResponse(status=204)
- put_container_meta(response, meta)
+ put_container_headers(response, meta)
return response
@api_method('PUT')
# unauthorized (401),
# badRequest (400)
- meta = get_container_meta(request)
+ meta = get_container_headers(request)
try:
backend.put_container(request.user, v_account, v_container)
# unauthorized (401),
# badRequest (400)
- meta = get_container_meta(request)
+ meta = get_container_headers(request)
replace = True
if 'update' in request.GET:
replace = False
validate_modification_preconditions(request, meta)
response = HttpResponse()
- put_container_meta(response, meta)
+ put_container_headers(response, meta)
path = request.GET.get('path')
prefix = request.GET.get('prefix')
keys = request.GET.get('meta')
if keys:
keys = keys.split(',')
- keys = [format_meta_key('X-Object-Meta-' + x.strip()) for x in keys if x.strip() != '']
+ keys = [format_header_key('X-Object-Meta-' + x.strip()) for x in keys if x.strip() != '']
else:
keys = []
except NameError:
pass
update_sharing_meta(permissions, v_account, v_container, x[0], meta)
- object_meta.append(printable_meta_dict(meta))
+ object_meta.append(printable_header_dict(meta))
if request.serialization == 'xml':
data = render_to_string('objects.xml', {'container': v_container, 'objects': object_meta})
elif request.serialization == 'json':
update_sharing_meta(permissions, v_account, v_container, v_object, meta)
response = HttpResponse(status=200)
- put_object_meta(response, meta)
+ put_object_headers(response, meta)
return response
@api_method('GET', format_allowed=True)
data = json.dumps(d)
response = HttpResponse(data, status=200)
- put_object_meta(response, meta)
+ put_object_headers(response, meta)
response['Content-Length'] = len(data)
return response
copy_or_move_object(request, v_account, src_container, src_name, v_container, v_object, move=False)
return HttpResponse(status=201)
- meta = get_object_meta(request)
+ meta = get_object_headers(request)
permissions = get_sharing(request)
content_length = -1
if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
# unauthorized (401),
# badRequest (400)
- meta = get_object_meta(request)
+ meta = get_object_headers(request)
permissions = get_sharing(request)
content_type = meta.get('Content-Type')
if content_type:
class AaiClient(Client):
def request(self, **request):
- request['HTTP_X_AUTH_TOKEN'] = '46e427d657b20defe352804f0eb6f8a2'
+ request['HTTP_X_AUTH_TOKEN'] = '0000'
return super(AaiClient, self).request(**request)
class BaseTestCase(TestCase):
from django.utils.http import http_date, parse_etags
from pithos.api.compat import parse_http_date_safe
-from pithos.api.faults import (Fault, NotModified, BadRequest, ItemNotFound, LengthRequired,
- PreconditionFailed, RangeNotSatisfiable, ServiceUnavailable)
+from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, ItemNotFound,
+ LengthRequired, PreconditionFailed, RangeNotSatisfiable,
+ ServiceUnavailable)
from pithos.backends import backend
from pithos.backends.base import NotAllowedError
logger = logging.getLogger(__name__)
-def printable_meta_dict(d):
+def printable_header_dict(d):
"""Format a meta dictionary for printing out json/xml.
Convert all keys to lower case and replace dashes to underscores.
del(d['modified'])
return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])
-def format_meta_key(k):
+def format_header_key(k):
"""Convert underscores to dashes and capitalize intra-dash strings."""
return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])
-def get_meta_prefix(request, prefix):
- """Get all prefix-* request headers in a dict. Reformat keys with format_meta_key()."""
+def get_header_prefix(request, prefix):
+ """Get all prefix-* request headers in a dict. Reformat keys with format_header_key()."""
prefix = 'HTTP_' + prefix.upper().replace('-', '_')
- return dict([(format_meta_key(k[5:]), v) for k, v in request.META.iteritems() if k.startswith(prefix) and len(k) > len(prefix)])
-
-def get_account_meta(request):
- """Get metadata from an account request."""
-
- meta = get_meta_prefix(request, 'X-Account-Meta-')
- return meta
-
-def put_account_meta(response, meta):
- """Put metadata in an account response."""
-
+ return dict([(format_header_key(k[5:]), v.replace('_', '')) for k, v in request.META.iteritems() if k.startswith(prefix) and len(k) > len(prefix)])
+
+def get_account_headers(request):
+ meta = get_header_prefix(request, 'X-Account-Meta-')
+ groups = {}
+ for k, v in get_header_prefix(request, 'X-Account-Group-').iteritems():
+ n = k[16:].lower()
+ if '-' in n or '_' in n:
+ raise BadRequest('Bad characters in group name')
+ groups[n] = v.replace(' ', '').split(',')
+ if '' in groups[n]:
+ groups[n].remove('')
+ return meta, groups
+
+def put_account_headers(response, meta, groups):
response['X-Account-Container-Count'] = meta['count']
response['X-Account-Bytes-Used'] = meta['bytes']
if 'modified' in meta:
response[k.encode('utf-8')] = meta[k].encode('utf-8')
if 'until_timestamp' in meta:
response['X-Account-Until-Timestamp'] = http_date(int(meta['until_timestamp']))
+ for k, v in groups.iteritems():
+ response[format_header_key('X-Account-Group-' + k).encode('utf-8')] = (','.join(v)).encode('utf-8')
-def get_container_meta(request):
- """Get metadata from a container request."""
-
- meta = get_meta_prefix(request, 'X-Container-Meta-')
+def get_container_headers(request):
+ meta = get_header_prefix(request, 'X-Container-Meta-')
return meta
-def put_container_meta(response, meta):
- """Put metadata in a container response."""
-
+def put_container_headers(response, meta):
response['X-Container-Object-Count'] = meta['count']
response['X-Container-Bytes-Used'] = meta['bytes']
response['Last-Modified'] = http_date(int(meta['modified']))
if 'until_timestamp' in meta:
response['X-Container-Until-Timestamp'] = http_date(int(meta['until_timestamp']))
-def get_object_meta(request):
- """Get metadata from an object request."""
-
- meta = get_meta_prefix(request, 'X-Object-Meta-')
+def get_object_headers(request):
+ meta = get_header_prefix(request, 'X-Object-Meta-')
if request.META.get('CONTENT_TYPE'):
meta['Content-Type'] = request.META['CONTENT_TYPE']
if request.META.get('HTTP_CONTENT_ENCODING'):
meta['X-Object-Manifest'] = request.META['HTTP_X_OBJECT_MANIFEST']
return meta
-def put_object_meta(response, meta, public=False):
- """Put metadata in an object response."""
-
+def put_object_headers(response, meta, public=False):
response['ETag'] = meta['hash']
response['Content-Length'] = meta['bytes']
response['Content-Type'] = meta.get('Content-Type', 'application/octet-stream')
def copy_or_move_object(request, v_account, src_container, src_name, dest_container, dest_name, move=False):
"""Copy or move an object."""
- meta = get_object_meta(request)
+ meta = get_object_headers(request)
permissions = get_sharing(request)
src_version = request.META.get('HTTP_X_SOURCE_VERSION')
try:
return ret
for perm in (x for x in permissions.split(';')):
if perm.startswith('read='):
- ret['read'] = [v.replace(' ','') for v in perm[5:].split(',')]
- ret['read'].remove('')
+ ret['read'] = [v.replace(' ','').lower() for v in perm[5:].split(',')]
+ if '' in ret['read']:
+ ret['read'].remove('')
if '*' in ret['read']:
ret['read'] = ['*']
if len(ret['read']) == 0:
raise BadRequest('Bad X-Object-Sharing header value')
elif perm.startswith('write='):
- ret['write'] = [v.replace(' ','') for v in perm[6:].split(',')]
- ret['write'].remove('')
+ ret['write'] = [v.replace(' ','').lower() for v in perm[6:].split(',')]
+ if '' in ret['write']:
+ ret['write'].remove('')
if '*' in ret['write']:
ret['write'] = ['*']
if len(ret['write']) == 0:
boundary = ''
wrapper = ObjectWrapper(ranges, sizes, hashmaps, boundary)
response = HttpResponse(wrapper, status=ret)
- put_object_meta(response, meta, public)
+ put_object_headers(response, meta, public)
if ret == 206:
if len(ranges) == 1:
offset, length = ranges[0]
Note that the account level is always valid as it is checked from another subsystem.
- When not replacing metadata, keys with empty values should be deleted.
+ When not replacing metadata/groups/policy, keys with empty values should be deleted.
The following variables should be available:
'hash_algorithm': Suggested is 'sha256'
'block_size': Suggested is 4MB
"""
- def delete_account(self, user, account):
- """Delete the account with the given name.
-
- Raises:
- NotAllowedError: Operation not permitted
- IndexError: Account is not empty
- """
- return
-
def get_account_meta(self, user, account, until=None):
"""Return a dictionary with the account metadata.
"""
return
- def list_containers(self, user, account, marker=None, limit=10000, until=None):
- """Return a list of container (name, version_id) tuples existing under an account.
-
- Parameters:
- 'marker': Start list from the next item after 'marker'
- 'limit': Number of containers to return
+ def get_account_groups(self, user, account):
+ """Return a dictionary with the user groups defined for this account.
Raises:
NotAllowedError: Operation not permitted
"""
- return []
+ return {}
- def put_container(self, user, account, container):
- """Create a new container with the given name.
+ def update_account_groups(self, user, account, groups, replace=False):
+ """Update the groups associated with the account.
Raises:
NotAllowedError: Operation not permitted
- NameError: Container already exists
+ ValueError: Invalid data in groups
"""
return
- def delete_container(self, user, account, container):
- """Delete the container with the given name.
+ def delete_account(self, user, account):
+ """Delete the account with the given name.
Raises:
NotAllowedError: Operation not permitted
- NameError: Container does not exist
- IndexError: Container is not empty
+ IndexError: Account is not empty
"""
return
+ def list_containers(self, user, account, marker=None, limit=10000, until=None):
+ """Return a list of container (name, version_id) tuples existing under an account.
+
+ Parameters:
+ 'marker': Start list from the next item after 'marker'
+ 'limit': Number of containers to return
+
+ Raises:
+ NotAllowedError: Operation not permitted
+ """
+ return []
+
def get_container_meta(self, user, account, container, until=None):
"""Return a dictionary with the container metadata.
"""
return
+ def get_container_policy(self, user, account, container):
+ """Return a dictionary with the container policy.
+
+ The keys returned are:
+ 'quota': The maximum bytes allowed (default is 0 - unlimited)
+ 'versioning': Can be 'auto', 'manual' or 'none' (default is 'manual')
+
+ Raises:
+ NotAllowedError: Operation not permitted
+ NameError: Container does not exist
+ """
+ return {}
+
+ def update_container_policy(self, user, account, container, policy, replace=False):
+ """Update the policy associated with the account.
+
+ Raises:
+ NotAllowedError: Operation not permitted
+ NameError: Container does not exist
+ ValueError: Invalid policy defined
+ """
+ return
+
+ def put_container(self, user, account, container, policy=None):
+ """Create a new container with the given name.
+
+ Raises:
+ NotAllowedError: Operation not permitted
+ NameError: Container already exists
+ ValueError: Invalid policy defined
+ """
+ return
+
+ def delete_container(self, user, account, container):
+ """Delete the container with the given name.
+
+ Raises:
+ NotAllowedError: Operation not permitted
+ NameError: Container does not exist
+ IndexError: Container is not empty
+ """
+ return
+
def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[], until=None):
"""Return a list of object (name, version_id) tuples existing under a container.
"""
return
+ def get_object_public(self, user, account, container, name):
+ """Return the public URL of the object if applicable.
+
+ Raises:
+ NotAllowedError: Operation not permitted
+ NameError: Container/object does not exist
+ """
+ return None
+
+ def update_object_public(self, user, account, container, name, public):
+ """Update the public status of the object.
+
+ Parameters:
+ 'public': Boolean value
+
+ Raises:
+ NotAllowedError: Operation not permitted
+ NameError: Container/object does not exist
+ """
+ return
+
def get_object_hashmap(self, user, account, container, name, version=None):
"""Return the object's size and a list with partial hashes.
sql = '''create table if not exists hashmaps (
version_id integer, pos integer, block_id text, primary key (version_id, pos))'''
self.con.execute(sql)
+ sql = '''create table if not exists groups (
+ account text, name text, users text, primary key (account, name))'''
+ self.con.execute(sql)
sql = '''create table if not exists permissions (
name text, read text, write text, primary key (name))'''
self.con.execute(sql)
+ sql = '''create table if not exists policy (
+ name text, key text, value text, primary key (name, key))'''
+ self.con.execute(sql)
self.con.commit()
- def delete_account(self, user, account):
- """Delete the account with the given name."""
-
- logger.debug("delete_account: %s", account)
- if user != account:
- raise NotAllowedError
- count, bytes, tstamp = self._get_pathstats(account)
- if count > 0:
- raise IndexError('Account is not empty')
- self._del_path(account) # Point of no return.
-
def get_account_meta(self, user, account, until=None):
"""Return a dictionary with the account metadata."""
raise NotAllowedError
self._put_metadata(user, account, meta, replace)
- def list_containers(self, user, account, marker=None, limit=10000, until=None):
- """Return a list of containers existing under an account."""
+ def get_account_groups(self, user, account):
+ """Return a dictionary with the user groups defined for this account."""
- logger.debug("list_containers: %s %s %s %s", account, marker, limit, until)
+ logger.debug("get_account_groups: %s", account)
if user != account:
raise NotAllowedError
- return self._list_objects(account, '', '/', marker, limit, False, [], until)
+ return self._get_groups(account)
- def put_container(self, user, account, container):
- """Create a new container with the given name."""
+ def update_account_groups(self, user, account, groups, replace=False):
+ """Update the groups associated with the account."""
- logger.debug("put_container: %s %s", account, container)
+ logger.debug("update_account_groups: %s %s %s", account, groups, replace)
if user != account:
raise NotAllowedError
- try:
- path, version_id, mtime = self._get_containerinfo(account, container)
- except NameError:
- path = os.path.join(account, container)
- version_id = self._put_version(path, user)
- else:
- raise NameError('Container already exists')
+ for k, v in groups.iteritems():
+ if True in [False or ',' in x for x in v]:
+ raise ValueError('Bad characters in groups')
+ if replace:
+ sql = 'delete from groups where account = ?'
+ self.con.execute(sql, (account,))
+ for k, v in groups.iteritems():
+ if len(v) == 0:
+ if not replace:
+ sql = 'delete from groups where account = ? and name = ?'
+ self.con.execute(sql, (account, k))
+ else:
+ sql = 'insert or replace into groups (account, name, users) values (?, ?, ?)'
+ self.con.execute(sql, (account, k, ','.join(v)))
+ self.con.commit()
- def delete_container(self, user, account, container):
- """Delete the container with the given name."""
+ def delete_account(self, user, account):
+ """Delete the account with the given name."""
- logger.debug("delete_container: %s %s", account, container)
+ logger.debug("delete_account: %s", account)
if user != account:
raise NotAllowedError
- path, version_id, mtime = self._get_containerinfo(account, container)
- count, bytes, tstamp = self._get_pathstats(path)
+ count, bytes, tstamp = self._get_pathstats(account)
if count > 0:
- raise IndexError('Container is not empty')
- self._del_path(path) # Point of no return.
- self._copy_version(user, account, account, True, True) # New account version.
+ raise IndexError('Account is not empty')
+ self._del_path(account) # Point of no return.
+
+ def list_containers(self, user, account, marker=None, limit=10000, until=None):
+ """Return a list of containers existing under an account."""
+
+ logger.debug("list_containers: %s %s %s %s", account, marker, limit, until)
+ if user != account:
+ raise NotAllowedError
+ return self._list_objects(account, '', '/', marker, limit, False, [], until)
def get_container_meta(self, user, account, container, until=None):
"""Return a dictionary with the container metadata."""
path, version_id, mtime = self._get_containerinfo(account, container)
self._put_metadata(user, path, meta, replace)
+ def get_container_policy(self, user, account, container):
+ """Return a dictionary with the container policy."""
+
+ logger.debug("get_container_policy: %s %s", account, container)
+ return {}
+
+ def update_container_policy(self, user, account, container, policy, replace=False):
+ """Update the policy associated with the account."""
+
+ logger.debug("update_container_policy: %s %s %s %s", account, container, policy, replace)
+ return
+
+ def put_container(self, user, account, container, policy=None):
+ """Create a new container with the given name."""
+
+ logger.debug("put_container: %s %s %s", account, container, policy)
+ if user != account:
+ raise NotAllowedError
+ try:
+ path, version_id, mtime = self._get_containerinfo(account, container)
+ except NameError:
+ path = os.path.join(account, container)
+ version_id = self._put_version(path, user)
+ else:
+ raise NameError('Container already exists')
+
+ def delete_container(self, user, account, container):
+ """Delete the container with the given name."""
+
+ logger.debug("delete_container: %s %s", account, container)
+ if user != account:
+ raise NotAllowedError
+ path, version_id, mtime = self._get_containerinfo(account, container)
+ count, bytes, tstamp = self._get_pathstats(path)
+ if count > 0:
+ raise IndexError('Container is not empty')
+ self._del_path(path) # Point of no return.
+ self._copy_version(user, account, account, True, True) # New account version.
+
def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[], until=None):
"""Return a list of objects existing under a container."""
r, w = self._check_permissions(path, permissions)
self._put_permissions(path, r, w)
+ def get_object_public(self, user, account, container, name):
+ """Return the public URL of the object if applicable."""
+
+ logger.debug("get_object_public: %s %s %s", account, container, name)
+ return None
+
+ def update_object_public(self, user, account, container, name, public):
+ """Update the public status of the object."""
+
+ logger.debug("update_object_public: %s %s %s %s", account, container, name, public)
+ return
+
def get_object_hashmap(self, user, account, container, name, version=None):
"""Return the object's size and a list with partial hashes."""
self.con.execute(sql, (dest_version_id, k, v))
self.con.commit()
+ def _get_groups(self, account):
+ sql = 'select name, users from groups where account = ?'
+ c = self.con.execute(sql, (account,))
+ return dict([(x[0], x[1].split(',')) for x in c.fetchall()])
+
def _is_allowed(self, user, account, container, name, op='read'):
if user == account:
return True
path = os.path.join(account, container, name)
perm_path, perms = self._get_permissions(path)
+
+ # Expand groups.
+ for x in ('read', 'write'):
+ g_perms = []
+ for y in perms.get(x, []):
+ if ':' in y:
+ g_account, g_name = y.split(':', 1)
+ groups = self._get_groups(g_account)
+ if g_name in groups:
+ g_perms += groups[g_name]
+ else:
+ g_perms.append(y)
+ perms[x] = g_perms
+
if op == 'read' and user in perms.get('read', []):
return True
if user in perms.get('write', []):
def setUp(self):
self.basepath = './test/content'
self.b = SimpleBackend(self.basepath)
- self.account = 'account1'
+ self.account = 'test'
def tearDown(self):
containers = [x[0] for x in self.b.list_containers('test', self.account)]
def test_get_account_meta(self):
meta = {
- "name": "account1",
+ "name": "test",
"username": "aaitest@uth.gr",
"email": "aaitest@uth.gr",
"fileroot": "http://hostname/gss/rest/aaitest@uth.gr/files",
self.assertEquals(unicode(v), d[k])
def test_get_non_existing_account_meta(self):
- meta = self.b.get_account_meta('test', 'account2')
- self.assertEquals(meta, {'name': 'account2', 'count': 0, 'bytes': 0})
+ meta = self.b.get_account_meta('account1', 'account1')
+ self.assertEquals(meta, {'name': 'account1', 'count': 0, 'bytes': 0})
def test_update_account_meta(self):
meta = {
- "name": "account1",
+ "name": "test",
"username": "aaitest@uth.gr",
"email": "aaitest@uth.gr",
"fileroot": "http://hostname/gss/rest/aaitest@uth.gr/files",
def setUp(self):
self.basepath = './test/content'
self.b = SimpleBackend(self.basepath)
- self.account = 'account1'
+ self.account = 'test'
def tearDown(self):
containers = [x[0] for x in self.b.list_containers('test', self.account)]
self.b.delete_container('test', self.account, container)
def test_list_non_existing_account_objects(self):
- self.assertRaises(NameError, self.b.list_objects, 'test', 'account2', 'container1')
+ self.assertRaises(NameError, self.b.list_objects, 'test', 'test', 'container1')
def test_list_objects(self):
self.b.put_container('test', self.account, 'container1')
def setUp(self):
self.basepath = './test/content'
self.b = SimpleBackend(self.basepath)
- self.account = 'account1'
+ self.account = 'test'
def tearDown(self):
containers = [x[0] for x in self.b.list_containers('test', self.account)]
self.assertRaises(NameError,
self.b.copy_object,
'test',
- 'account',
+ 'test',
src_cname,
src_obj,
dest_cname,
projects / pithos / commitdiff