From: Sofia Papagiannaki Date: Wed, 27 Jun 2012 15:35:22 +0000 (+0300) Subject: Merge branch 'next' X-Git-Tag: pithos/v0.10.1~22 X-Git-Url: https://code.grnet.gr/git/pithos/commitdiff_plain/2c10110f2078b2cfaf04964759faccd15803ba4b?hp=00744daf13f120001dc8bba44ad0b36845f41e2d Merge branch 'next' --- diff --git a/snf-pithos-app/pithos/api/delegate.py b/snf-pithos-app/pithos/api/delegate.py index c3083ea..e665723 100644 --- a/snf-pithos-app/pithos/api/delegate.py +++ b/snf-pithos-app/pithos/api/delegate.py @@ -82,7 +82,9 @@ def delegate_to_feedback_service(request): try: urllib2.urlopen(req) except urllib2.HTTPError, e: + logger.exception(e) return HttpResponse(status=e.code) except urllib2.URLError, e: - return HttpResponseNotFound(e) + logger.exception(e) + return HttpResponse(status=e.reason) return HttpResponse() \ No newline at end of file diff --git a/snf-pithos-app/pithos/api/functions.py b/snf-pithos-app/pithos/api/functions.py index f1488b8..65e56d1 100644 --- a/snf-pithos-app/pithos/api/functions.py +++ b/snf-pithos-app/pithos/api/functions.py @@ -52,7 +52,7 @@ from pithos.api.util import (json_encode_decimal, rename_meta_key, format_header validate_modification_preconditions, validate_matching_preconditions, split_container_object_string, copy_or_move_object, get_int_parameter, get_content_length, get_content_range, socket_read_iterator, SaveToBackendHandler, object_data_response, put_object_block, hashmap_md5, simple_list_response, api_method) -from pithos.api.settings import AUTHENTICATION_URL, AUTHENTICATION_USERS, COOKIE_NAME, UPDATE_MD5 +from pithos.api.settings import UPDATE_MD5 from pithos.backends.base import NotAllowedError, QuotaError from pithos.backends.filter import parse_filters @@ -66,17 +66,13 @@ logger = logging.getLogger(__name__) @csrf_exempt def top_demux(request): - get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS) if request.method == 'GET': - if getattr(request, 'user', None) is not None: - return account_list(request) - return authenticate(request) + return account_list(request) else: return method_not_allowed(request) @csrf_exempt def account_demux(request, v_account): - get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS) if request.method == 'HEAD': return account_meta(request, v_account) elif request.method == 'POST': @@ -88,7 +84,6 @@ def account_demux(request, v_account): @csrf_exempt def container_demux(request, v_account, v_container): - get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS) if request.method == 'HEAD': return container_meta(request, v_account, v_container) elif request.method == 'PUT': @@ -105,12 +100,6 @@ def container_demux(request, v_account, v_container): @csrf_exempt def object_demux(request, v_account, v_container, v_object): # Helper to avoid placing the token in the URL when loading objects from a browser. - token = None - if request.method in ('HEAD', 'GET') and COOKIE_NAME in request.COOKIES: - cookie_value = unquote(request.COOKIES.get(COOKIE_NAME, '')) - if cookie_value and '|' in cookie_value: - token = cookie_value.split('|', 1)[1] - get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS, token) if request.method == 'HEAD': return object_meta(request, v_account, v_container, v_object) elif request.method == 'GET': @@ -156,6 +145,8 @@ def account_list(request): # Normal Response Codes: 200, 204 # Error Response Codes: internalServerError (500), # badRequest (400) + if getattr(request, 'user', None) is None: + return authenticate(request) response = HttpResponse() @@ -797,6 +788,7 @@ def object_write(request, v_account, v_container, v_object): copy_from = request.META.get('HTTP_X_COPY_FROM') move_from = request.META.get('HTTP_X_MOVE_FROM') if copy_from or move_from: + delimiter = request.GET.get('delimiter') content_length = get_content_length(request) # Required by the API. src_account = request.META.get('HTTP_X_SOURCE_ACCOUNT') @@ -808,14 +800,14 @@ def object_write(request, v_account, v_container, v_object): except ValueError: raise BadRequest('Invalid X-Move-From header') version_id = copy_or_move_object(request, src_account, src_container, src_name, - v_account, v_container, v_object, move=True) + v_account, v_container, v_object, move=True, delimiter=delimiter) else: try: src_container, src_name = split_container_object_string(copy_from) except ValueError: raise BadRequest('Invalid X-Copy-From header') version_id = copy_or_move_object(request, src_account, src_container, src_name, - v_account, v_container, v_object, move=False) + v_account, v_container, v_object, move=False, delimiter=delimiter) response = HttpResponse(status=201) response['X-Object-Version'] = version_id return response @@ -976,8 +968,10 @@ def object_copy(request, v_account, v_container, v_object): raise ItemNotFound('Container or object does not exist') validate_matching_preconditions(request, meta) + delimiter = request.GET.get('delimiter') + version_id = copy_or_move_object(request, v_account, v_container, v_object, - dest_account, dest_container, dest_name, move=False) + dest_account, dest_container, dest_name, move=False, delimiter=delimiter) response = HttpResponse(status=201) response['X-Object-Version'] = version_id return response @@ -1012,8 +1006,10 @@ def object_move(request, v_account, v_container, v_object): raise ItemNotFound('Container or object does not exist') validate_matching_preconditions(request, meta) + delimiter = request.GET.get('delimiter') + version_id = copy_or_move_object(request, v_account, v_container, v_object, - dest_account, dest_container, dest_name, move=True) + dest_account, dest_container, dest_name, move=True, delimiter=delimiter) response = HttpResponse(status=201) response['X-Object-Version'] = version_id return response @@ -1230,9 +1226,11 @@ def object_delete(request, v_account, v_container, v_object): # badRequest (400) until = get_int_parameter(request.GET.get('until')) + delimiter = request.GET.get('delimiter') + try: request.backend.delete_object(request.user_uniq, v_account, v_container, - v_object, until) + v_object, until, delimiter=delimiter) except NotAllowedError: raise Forbidden('Not allowed') except NameError: diff --git a/snf-pithos-app/pithos/api/util.py b/snf-pithos-app/pithos/api/util.py index 7dbaf4c..c3a8fd8 100644 --- a/snf-pithos-app/pithos/api/util.py +++ b/snf-pithos-app/pithos/api/util.py @@ -49,6 +49,7 @@ from django.core.files.uploadhandler import FileUploadHandler from django.core.files.uploadedfile import UploadedFile from synnefo.lib.parsedate import parse_http_date_safe, parse_http_date +from synnefo.lib.astakos import get_user from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, Conflict, LengthRequired, PreconditionFailed, RequestEntityTooLarge, @@ -58,7 +59,10 @@ from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION, BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH, BACKEND_BLOCK_UMASK, BACKEND_QUEUE_MODULE, BACKEND_QUEUE_CONNECTION, - BACKEND_QUOTA, BACKEND_VERSIONING) + BACKEND_QUOTA, BACKEND_VERSIONING, + AUTHENTICATION_URL, AUTHENTICATION_USERS, + SERVICE_TOKEN, COOKIE_NAME) + from pithos.backends import connect_backend from pithos.backends.base import NotAllowedError, QuotaError @@ -316,22 +320,24 @@ def split_container_object_string(s): raise ValueError return s[:pos], s[(pos + 1):] -def copy_or_move_object(request, src_account, src_container, src_name, dest_account, dest_container, dest_name, move=False): +def copy_or_move_object(request, src_account, src_container, src_name, dest_account, dest_container, dest_name, move=False, delimiter=None): """Copy or move an object.""" if 'ignore_content_type' in request.GET and 'CONTENT_TYPE' in request.META: del(request.META['CONTENT_TYPE']) content_type, meta, permissions, public = get_object_headers(request) + if delimiter: + public = False # ignore public in that case src_version = request.META.get('HTTP_X_SOURCE_VERSION') try: if move: version_id = request.backend.move_object(request.user_uniq, src_account, src_container, src_name, dest_account, dest_container, dest_name, - content_type, 'pithos', meta, False, permissions) + content_type, 'pithos', meta, False, permissions, delimiter) else: version_id = request.backend.copy_object(request.user_uniq, src_account, src_container, src_name, dest_account, dest_container, dest_name, - content_type, 'pithos', meta, False, permissions, src_version) + content_type, 'pithos', meta, False, permissions, src_version, delimiter) except NotAllowedError: raise Forbidden('Not allowed') except (NameError, IndexError): @@ -875,8 +881,16 @@ def api_method(http_method=None, format_allowed=False, user_required=True): try: if http_method and request.method != http_method: raise BadRequest('Method not allowed.') - if user_required and getattr(request, 'user', None) is None: - raise Unauthorized('Access denied') + + if user_required: + token = None + if request.method in ('HEAD', 'GET') and COOKIE_NAME in request.COOKIES: + cookie_value = unquote(request.COOKIES.get(COOKIE_NAME, '')) + if cookie_value and '|' in cookie_value: + token = cookie_value.split('|', 1)[1] + get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS, token) + if getattr(request, 'user', None) is None: + raise Unauthorized('Access denied') # The args variable may contain up to (account, container, object). if len(args) > 1 and len(args[1]) > 256: @@ -898,7 +912,7 @@ def api_method(http_method=None, format_allowed=False, user_required=True): return render_fault(request, fault) except BaseException, e: logger.exception('Unexpected error: %s' % e) - fault = InternalServerError('Unexpected error') + fault = InternalServerError('Unexpected error: %s' % e) return render_fault(request, fault) finally: if getattr(request, 'backend', None) is not None: diff --git a/snf-pithos-backend/pithos/backends/base.py b/snf-pithos-backend/pithos/backends/base.py index 3cf2251..0d4e198 100644 --- a/snf-pithos-backend/pithos/backends/base.py +++ b/snf-pithos-backend/pithos/backends/base.py @@ -169,7 +169,7 @@ class BaseBackend(object): """ return - def list_containers(self, user, account, marker=None, limit=10000, shared=False, until=None): + def list_containers(self, user, account, marker=None, limit=10000, shared=False, until=None, public=False): """Return a list of container names existing under an account. Parameters: @@ -179,6 +179,8 @@ class BaseBackend(object): 'shared': Only list containers with permissions set + 'public': Only list containers containing public objects + Raises: NotAllowedError: Operation not permitted @@ -284,7 +286,7 @@ class BaseBackend(object): """ return - def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None): + def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None, public=False): """Return a list of object (name, version_id) tuples existing under a container. Parameters: @@ -312,6 +314,9 @@ class BaseBackend(object): 'size_range': Include objects with byte size in (from, to). Use None to specify unlimited + + 'public': Only list public objects + Raises: NotAllowedError: Operation not permitted @@ -489,7 +494,7 @@ class BaseBackend(object): """Update an object's checksum.""" return - def copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, src_version=None): + def copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, src_version=None, delimiter=None): """Copy an object's data and metadata and return the new version. Parameters: @@ -502,6 +507,8 @@ class BaseBackend(object): 'permissions': New object permissions 'src_version': Copy from the version provided + + 'delimiter': Copy objects whose path starts with src_name + delimiter Raises: NotAllowedError: Operation not permitted @@ -516,7 +523,7 @@ class BaseBackend(object): """ return '' - def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None): + def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, delimiter=None): """Move an object's data and metadata and return the new version. Parameters: @@ -527,6 +534,8 @@ class BaseBackend(object): 'replace_meta': Replace metadata instead of update 'permissions': New object permissions + + 'delimiter': Move objects whose path starts with src_name + delimiter Raises: NotAllowedError: Operation not permitted @@ -539,9 +548,12 @@ class BaseBackend(object): """ return '' - def delete_object(self, user, account, container, name, until=None): + def delete_object(self, user, account, container, name, until=None, delimiter=None): """Delete/purge an object. + Parameters: + 'delimiter': Delete objects whose path starting with name + delimiter + Raises: NotAllowedError: Operation not permitted diff --git a/snf-pithos-backend/pithos/backends/lib/sqlalchemy/node.py b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/node.py index bf996f6..130c764 100644 --- a/snf-pithos-backend/pithos/backends/lib/sqlalchemy/node.py +++ b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/node.py @@ -219,6 +219,18 @@ class Node(DBWorker): return row[0] return None + def node_lookup_bulk(self, paths): + """Lookup the current nodes for the given paths. + Return () if the path is not found. + """ + + # Use LIKE for comparison to avoid MySQL problems with trailing spaces. + s = select([self.nodes.c.node], self.nodes.c.path.in_(paths)) + r = self.conn.execute(s) + rows = r.fetchall() + r.close() + return [row[0] for row in rows] + def node_get_properties(self, node): """Return the node's (parent, path). Return None if the node is not found. @@ -571,7 +583,7 @@ class Node(DBWorker): self.statistics_update_ancestors(node, 1, size, mtime, cluster) return serial, mtime - def version_lookup(self, node, before=inf, cluster=0): + def version_lookup(self, node, before=inf, cluster=0, all_props=True): """Lookup the current version of the given node. Return a list with its properties: (serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster) @@ -579,10 +591,13 @@ class Node(DBWorker): """ v = self.versions.alias('v') - s = select([v.c.serial, v.c.node, v.c.hash, - v.c.size, v.c.type, v.c.source, - v.c.mtime, v.c.muser, v.c.uuid, - v.c.checksum, v.c.cluster]) + if not all_props: + s = select([v.c.serial]) + else: + s = select([v.c.serial, v.c.node, v.c.hash, + v.c.size, v.c.type, v.c.source, + v.c.mtime, v.c.muser, v.c.uuid, + v.c.checksum, v.c.cluster]) c = select([func.max(self.versions.c.serial)], self.versions.c.node == node) if before != inf: @@ -596,6 +611,31 @@ class Node(DBWorker): return props return None + def version_lookup_bulk(self, nodes, before=inf, cluster=0, all_props=True): + """Lookup the current versions of the given nodes. + Return a list with their properties: + (serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster). + """ + + v = self.versions.alias('v') + if not all_props: + s = select([v.c.serial]) + else: + s = select([v.c.serial, v.c.node, v.c.hash, + v.c.size, v.c.type, v.c.source, + v.c.mtime, v.c.muser, v.c.uuid, + v.c.checksum, v.c.cluster]) + c = select([func.max(self.versions.c.serial)], + self.versions.c.node.in_(nodes)).group_by(self.versions.c.node) + if before != inf: + c = c.where(self.versions.c.mtime < before) + s = s.where(and_(v.c.serial.in_(c), + v.c.cluster == cluster)) + r = self.conn.execute(s) + rproxy = r.fetchall() + r.close() + return (tuple(row.values()) for row in rproxy) + def version_get_properties(self, serial, keys=(), propnames=_propnames): """Return a sequence of values for the properties of the version specified by serial and the keys, in the order given. diff --git a/snf-pithos-backend/pithos/backends/lib/sqlalchemy/permissions.py b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/permissions.py index 5ebdea2..9644e85 100644 --- a/snf-pithos-backend/pithos/backends/lib/sqlalchemy/permissions.py +++ b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/permissions.py @@ -115,6 +115,12 @@ class Permissions(XFeatures, Groups, Public): self.xfeature_destroy(path) self.public_unset(path) + def access_clear_bulk(self, paths): + """Revoke access to path (both permissions and public).""" + + self.xfeature_destroy_bulk(paths) + self.public_unset_bulk(paths) + def access_check(self, path, access, member): """Return true if the member has this access to the path.""" diff --git a/snf-pithos-backend/pithos/backends/lib/sqlalchemy/public.py b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/public.py index 25bf0f3..bb06282 100644 --- a/snf-pithos-backend/pithos/backends/lib/sqlalchemy/public.py +++ b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/public.py @@ -74,6 +74,13 @@ class Public(DBWorker): r = self.conn.execute(s) r.close() + def public_unset_bulk(self, paths): + s = self.public.update() + s = s.where(self.public.c.path.in_(paths)) + s = s.values(active=False) + r = self.conn.execute(s) + r.close() + def public_get(self, path): s = select([self.public.c.public_id]) s = s.where(and_(self.public.c.path == path, diff --git a/snf-pithos-backend/pithos/backends/lib/sqlalchemy/xfeatures.py b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/xfeatures.py index d9ebd66..d4c45d8 100644 --- a/snf-pithos-backend/pithos/backends/lib/sqlalchemy/xfeatures.py +++ b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/xfeatures.py @@ -113,6 +113,13 @@ class XFeatures(DBWorker): r = self.conn.execute(s) r.close() + def xfeature_destroy_bulk(self, paths): + """Destroy features and all their key, value pairs.""" + + s = self.xfeatures.delete().where(self.xfeatures.c.path.in_(paths)) + r = self.conn.execute(s) + r.close() + def feature_dict(self, feature): """Return a dict mapping keys to list of values for feature.""" diff --git a/snf-pithos-backend/pithos/backends/lib/sqlite/node.py b/snf-pithos-backend/pithos/backends/lib/sqlite/node.py index ab79c52..ce73d48 100644 --- a/snf-pithos-backend/pithos/backends/lib/sqlite/node.py +++ b/snf-pithos-backend/pithos/backends/lib/sqlite/node.py @@ -201,6 +201,19 @@ class Node(DBWorker): return r[0] return None + def node_lookup_bulk(self, paths): + """Lookup the current nodes for the given paths. + Return () if the path is not found. + """ + + placeholders = ','.join('?' for path in paths) + q = "select node from nodes where path in (%s)" % placeholders + self.execute(q, paths) + r = self.fetchall() + if r is not None: + return [row[0] for row in r] + return None + def node_get_properties(self, node): """Return the node's (parent, path). Return None if the node is not found. @@ -482,24 +495,52 @@ class Node(DBWorker): self.statistics_update_ancestors(node, 1, size, mtime, cluster) return serial, mtime - def version_lookup(self, node, before=inf, cluster=0): + def version_lookup(self, node, before=inf, cluster=0, all_props=True): """Lookup the current version of the given node. Return a list with its properties: (serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster) or None if the current version is not found in the given cluster. """ - q = ("select serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster " + q = ("select %s " "from versions " "where serial = (select max(serial) " "from versions " "where node = ? and mtime < ?) " "and cluster = ?") + if not all_props: + q = q % "serial" + else: + q = q % "serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster" + self.execute(q, (node, before, cluster)) props = self.fetchone() if props is not None: return props return None + + def version_lookup_bulk(self, nodes, before=inf, cluster=0, all_props=True): + """Lookup the current versions of the given nodes. + Return a list with their properties: + (serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster). + """ + + q = ("select %s " + "from versions " + "where serial in (select max(serial) " + "from versions " + "where node in (%s) and mtime < ? group by node) " + "and cluster = ?") + placeholders = ','.join('?' for node in nodes) + if not all_props: + q = q % ("serial", placeholders) + else: + q = q % ("serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster", placeholders) + + args = nodes + args.extend((before, cluster)) + self.execute(q, args) + return self.fetchall() def version_get_properties(self, serial, keys=(), propnames=_propnames): """Return a sequence of values for the properties of @@ -656,7 +697,6 @@ class Node(DBWorker): subqlist = [] args = [] - print pathq for path, match in pathq: if match == MATCH_PREFIX: subqlist.append("n.path like ? escape '\\'") diff --git a/snf-pithos-backend/pithos/backends/lib/sqlite/permissions.py b/snf-pithos-backend/pithos/backends/lib/sqlite/permissions.py index b29422d..8faed87 100644 --- a/snf-pithos-backend/pithos/backends/lib/sqlite/permissions.py +++ b/snf-pithos-backend/pithos/backends/lib/sqlite/permissions.py @@ -112,6 +112,12 @@ class Permissions(XFeatures, Groups, Public): self.xfeature_destroy(path) self.public_unset(path) + def access_clear_bulk(self, paths): + """Revoke access to path (both permissions and public).""" + + self.xfeature_destroy_bulk(paths) + self.public_unset_bulk(paths) + def access_check(self, path, access, member): """Return true if the member has this access to the path.""" diff --git a/snf-pithos-backend/pithos/backends/lib/sqlite/public.py b/snf-pithos-backend/pithos/backends/lib/sqlite/public.py index 2d6e9cb..e314bfd 100644 --- a/snf-pithos-backend/pithos/backends/lib/sqlite/public.py +++ b/snf-pithos-backend/pithos/backends/lib/sqlite/public.py @@ -58,6 +58,11 @@ class Public(DBWorker): q = "update public set active = 0 where path = ?" self.execute(q, (path,)) + def public_unset_bulk(self, paths): + placeholders = ','.join('?' for path in paths) + q = "update public set active = 0 where path in (%s)" % placeholders + self.execute(q, paths) + def public_get(self, path): q = "select public_id from public where path = ? and active = 1" self.execute(q, (path,)) diff --git a/snf-pithos-backend/pithos/backends/lib/sqlite/xfeatures.py b/snf-pithos-backend/pithos/backends/lib/sqlite/xfeatures.py index 7d682d0..de948d0 100644 --- a/snf-pithos-backend/pithos/backends/lib/sqlite/xfeatures.py +++ b/snf-pithos-backend/pithos/backends/lib/sqlite/xfeatures.py @@ -99,6 +99,13 @@ class XFeatures(DBWorker): q = "delete from xfeatures where path = ?" self.execute(q, (path,)) + def xfeature_destroy_bulk(self, paths): + """Destroy features and all their key, value pairs.""" + + placeholders = ','.join('?' for path in paths) + q = "delete from xfeatures where path in (%s)" % placeholders + self.execute(q, paths) + def feature_dict(self, feature): """Return a dict mapping keys to list of values for feature.""" diff --git a/snf-pithos-backend/pithos/backends/modular.py b/snf-pithos-backend/pithos/backends/modular.py index 9e1b4ea..d1e3126 100644 --- a/snf-pithos-backend/pithos/backends/modular.py +++ b/snf-pithos-backend/pithos/backends/modular.py @@ -190,7 +190,7 @@ class ModularBackend(BaseBackend): def get_account_meta(self, user, account, domain, until=None, include_user_defined=True): """Return a dictionary with the account metadata for the domain.""" - logger.debug("get_account_meta: %s %s %s", account, domain, until) + logger.debug("get_account_meta: %s %s %s %s", user, account, domain, until) path, node = self._lookup_account(account, user == account) if user != account: if until or node is None or account not in self._allowed_accounts(user): @@ -225,7 +225,7 @@ class ModularBackend(BaseBackend): def update_account_meta(self, user, account, domain, meta, replace=False): """Update the metadata associated with the account for the domain.""" - logger.debug("update_account_meta: %s %s %s %s", account, domain, meta, replace) + logger.debug("update_account_meta: %s %s %s %s %s", user, account, domain, meta, replace) if user != account: raise NotAllowedError path, node = self._lookup_account(account, True) @@ -235,7 +235,7 @@ class ModularBackend(BaseBackend): def get_account_groups(self, user, account): """Return a dictionary with the user groups defined for this account.""" - logger.debug("get_account_groups: %s", account) + logger.debug("get_account_groups: %s %s", user, account) if user != account: if account not in self._allowed_accounts(user): raise NotAllowedError @@ -247,7 +247,7 @@ class ModularBackend(BaseBackend): def update_account_groups(self, user, account, groups, replace=False): """Update the groups associated with the account.""" - logger.debug("update_account_groups: %s %s %s", account, groups, replace) + logger.debug("update_account_groups: %s %s %s %s", user, account, groups, replace) if user != account: raise NotAllowedError self._lookup_account(account, True) @@ -264,7 +264,7 @@ class ModularBackend(BaseBackend): def get_account_policy(self, user, account): """Return a dictionary with the account policy.""" - logger.debug("get_account_policy: %s", account) + logger.debug("get_account_policy: %s %s", user, account) if user != account: if account not in self._allowed_accounts(user): raise NotAllowedError @@ -276,7 +276,7 @@ class ModularBackend(BaseBackend): def update_account_policy(self, user, account, policy, replace=False): """Update the policy associated with the account.""" - logger.debug("update_account_policy: %s %s %s", account, policy, replace) + logger.debug("update_account_policy: %s %s %s %s", user, account, policy, replace) if user != account: raise NotAllowedError path, node = self._lookup_account(account, True) @@ -287,7 +287,7 @@ class ModularBackend(BaseBackend): def put_account(self, user, account, policy={}): """Create a new account with the given name.""" - logger.debug("put_account: %s %s", account, policy) + logger.debug("put_account: %s %s %s", user, account, policy) if user != account: raise NotAllowedError node = self.node.node_lookup(account) @@ -302,7 +302,7 @@ class ModularBackend(BaseBackend): def delete_account(self, user, account): """Delete the account with the given name.""" - logger.debug("delete_account: %s", account) + logger.debug("delete_account: %s %s", user, account) if user != account: raise NotAllowedError node = self.node.node_lookup(account) @@ -316,7 +316,7 @@ class ModularBackend(BaseBackend): def list_containers(self, user, account, marker=None, limit=10000, shared=False, until=None, public=False): """Return a list of containers existing under an account.""" - logger.debug("list_containers: %s %s %s %s %s %s", account, marker, limit, shared, until, public) + logger.debug("list_containers: %s %s %s %s %s %s %s", user, account, marker, limit, shared, until, public) if user != account: if until or account not in self._allowed_accounts(user): raise NotAllowedError @@ -324,23 +324,24 @@ class ModularBackend(BaseBackend): start, limit = self._list_limits(allowed, marker, limit) return allowed[start:start + limit] if shared or public: - allowed = [] + allowed = set() if shared: - allowed.extend([x.split('/', 2)[1] for x in self.permissions.access_list_shared(account)]) + allowed.update([x.split('/', 2)[1] for x in self.permissions.access_list_shared(account)]) if public: - allowed.extend([x[0].split('/', 2)[1] for x in self.permissions.public_list(account)]) - allowed = list(set(allowed)) - allowed.sort() + allowed.update([x[0].split('/', 2)[1] for x in self.permissions.public_list(account)]) + allowed = sorted(allowed) start, limit = self._list_limits(allowed, marker, limit) return allowed[start:start + limit] node = self.node.node_lookup(account) - return [x[0] for x in self._list_object_properties(node, account, '', '/', marker, limit, False, None, [], until)] + containers = [x[0] for x in self._list_object_properties(node, account, '', '/', marker, limit, False, None, [], until)] + start, limit = self._list_limits([x[0] for x in containers], marker, limit) + return containers[start:start + limit] @backend_method def list_container_meta(self, user, account, container, domain, until=None): """Return a list with all the container's object meta keys for the domain.""" - logger.debug("list_container_meta: %s %s %s %s", account, container, domain, until) + logger.debug("list_container_meta: %s %s %s %s %s", user, account, container, domain, until) allowed = [] if user != account: if until: @@ -357,7 +358,7 @@ class ModularBackend(BaseBackend): def get_container_meta(self, user, account, container, domain, until=None, include_user_defined=True): """Return a dictionary with the container metadata for the domain.""" - logger.debug("get_container_meta: %s %s %s %s", account, container, domain, until) + logger.debug("get_container_meta: %s %s %s %s %s", user, account, container, domain, until) if user != account: if until or container not in self._allowed_containers(user, account): raise NotAllowedError @@ -388,7 +389,7 @@ class ModularBackend(BaseBackend): def update_container_meta(self, user, account, container, domain, meta, replace=False): """Update the metadata associated with the container for the domain.""" - logger.debug("update_container_meta: %s %s %s %s %s", account, container, domain, meta, replace) + logger.debug("update_container_meta: %s %s %s %s %s %s", user, account, container, domain, meta, replace) if user != account: raise NotAllowedError path, node = self._lookup_container(account, container) @@ -402,7 +403,7 @@ class ModularBackend(BaseBackend): def get_container_policy(self, user, account, container): """Return a dictionary with the container policy.""" - logger.debug("get_container_policy: %s %s", account, container) + logger.debug("get_container_policy: %s %s %s", user, account, container) if user != account: if container not in self._allowed_containers(user, account): raise NotAllowedError @@ -414,7 +415,7 @@ class ModularBackend(BaseBackend): def update_container_policy(self, user, account, container, policy, replace=False): """Update the policy associated with the container.""" - logger.debug("update_container_policy: %s %s %s %s", account, container, policy, replace) + logger.debug("update_container_policy: %s %s %s %s %s", user, account, container, policy, replace) if user != account: raise NotAllowedError path, node = self._lookup_container(account, container) @@ -425,7 +426,7 @@ class ModularBackend(BaseBackend): def put_container(self, user, account, container, policy={}): """Create a new container with the given name.""" - logger.debug("put_container: %s %s %s", account, container, policy) + logger.debug("put_container: %s %s %s %s", user, account, container, policy) if user != account: raise NotAllowedError try: @@ -441,10 +442,10 @@ class ModularBackend(BaseBackend): self._put_policy(node, policy, True) @backend_method - def delete_container(self, user, account, container, until=None): + def delete_container(self, user, account, container, until=None, prefix='', delimiter=None): """Delete/purge the container with the given name.""" - logger.debug("delete_container: %s %s %s", account, container, until) + logger.debug("delete_container: %s %s %s %s %s %s", user, account, container, until, prefix, delimiter) if user != account: raise NotAllowedError path, node = self._lookup_container(account, container) @@ -469,12 +470,55 @@ class ModularBackend(BaseBackend): def _list_objects(self, user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, all_props, public): if user != account and until: raise NotAllowedError + if shared and public: + # get shared first + shared = self._list_object_permissions(user, account, container, prefix, shared=True, public=False) + objects = [] + if shared: + path, node = self._lookup_container(account, container) + shared = self._get_formatted_paths(shared) + objects = self._list_object_properties(node, path, prefix, delimiter, marker, limit, virtual, domain, keys, until, size_range, shared, all_props) + + # get public + objects.extend(self._list_public_object_properties(user, account, container, prefix, all_props)) + + objects.sort(key=lambda x: x[0]) + start, limit = self._list_limits([x[0] for x in objects], marker, limit) + return objects[start:start + limit] + elif public: + objects = self._list_public_object_properties(user, account, container, prefix, all_props) + start, limit = self._list_limits([x[0] for x in objects], marker, limit) + return objects[start:start + limit] + allowed = self._list_object_permissions(user, account, container, prefix, shared, public) - if (shared or public) and not allowed: + if shared and not allowed: return [] path, node = self._lookup_container(account, container) allowed = self._get_formatted_paths(allowed) - return self._list_object_properties(node, path, prefix, delimiter, marker, limit, virtual, domain, keys, until, size_range, allowed, all_props) + objects = self._list_object_properties(node, path, prefix, delimiter, marker, limit, virtual, domain, keys, until, size_range, allowed, all_props) + start, limit = self._list_limits([x[0] for x in objects], marker, limit) + return objects[start:start + limit] + + def _list_public_object_properties(self, user, account, container, prefix, all_props): + public = self._list_object_permissions(user, account, container, prefix, shared=False, public=True) + paths, nodes = self._lookup_objects(public) + path = '/'.join((account, container)) + cont_prefix = path + '/' + paths = [x[len(cont_prefix):] for x in paths] + props = self.node.version_lookup_bulk(nodes, all_props=all_props) + objects = [(path,) + props for path, props in zip(paths, props)] + return objects + + def _list_objects_no_limit(self, user, account, container, prefix, delimiter, virtual, domain, keys, shared, until, size_range, all_props, public): + objects = [] + while True: + marker = objects[-1] if objects else None + limit = 10000 + l = self._list_objects(user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, all_props, public) + objects.extend(l) + if not l or len(l) < limit: + break + return objects def _list_object_permissions(self, user, account, container, prefix, shared, public): allowed = [] @@ -484,13 +528,12 @@ class ModularBackend(BaseBackend): if not allowed: raise NotAllowedError else: - allowed = [] + allowed = set() if shared: - allowed.extend(self.permissions.access_list_shared(path)) + allowed.update(self.permissions.access_list_shared(path)) if public: - allowed.extend([x[0] for x in self.permissions.public_list(path)]) - allowed = list(set(allowed)) - allowed.sort() + allowed.update([x[0] for x in self.permissions.public_list(path)]) + allowed = sorted(allowed) if not allowed: return [] return allowed @@ -499,14 +542,14 @@ class ModularBackend(BaseBackend): def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None, public=False): """Return a list of object (name, version_id) tuples existing under a container.""" - logger.debug("list_objects: %s %s %s %s %s %s %s %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, public) + logger.debug("list_objects: %s %s %s %s %s %s %s %s %s %s %s %s %s %s", user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, public) return self._list_objects(user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, False, public) @backend_method def list_object_meta(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None, public=False): """Return a list of object metadata dicts existing under a container.""" - logger.debug("list_object_meta: %s %s %s %s %s %s %s %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, public) + logger.debug("list_object_meta: %s %s %s %s %s %s %s %s %s %s %s %s %s %s", user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, public) props = self._list_objects(user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, True, public) objects = [] for p in props: @@ -529,14 +572,14 @@ class ModularBackend(BaseBackend): def list_object_permissions(self, user, account, container, prefix=''): """Return a list of paths that enforce permissions under a container.""" - logger.debug("list_object_permissions: %s %s %s", account, container, prefix) + logger.debug("list_object_permissions: %s %s %s %s", user, account, container, prefix) return self._list_object_permissions(user, account, container, prefix, True, False) @backend_method def list_object_public(self, user, account, container, prefix=''): """Return a dict mapping paths to public ids for objects that are public under a container.""" - logger.debug("list_object_public: %s %s %s", account, container, prefix) + logger.debug("list_object_public: %s %s %s %s", user, account, container, prefix) public = {} for path, p in self.permissions.public_list('/'.join((account, container, prefix))): public[path] = p + ULTIMATE_ANSWER @@ -546,7 +589,7 @@ class ModularBackend(BaseBackend): def get_object_meta(self, user, account, container, name, domain, version=None, include_user_defined=True): """Return a dictionary with the object metadata for the domain.""" - logger.debug("get_object_meta: %s %s %s %s %s", account, container, name, domain, version) + logger.debug("get_object_meta: %s %s %s %s %s %s", user, account, container, name, domain, version) self._can_read(user, account, container, name) path, node = self._lookup_object(account, container, name) props = self._get_version(node, version) @@ -580,7 +623,7 @@ class ModularBackend(BaseBackend): def update_object_meta(self, user, account, container, name, domain, meta, replace=False): """Update the metadata associated with the object for the domain and return the new version.""" - logger.debug("update_object_meta: %s %s %s %s %s %s", account, container, name, domain, meta, replace) + logger.debug("update_object_meta: %s %s %s %s %s %s %s", user, account, container, name, domain, meta, replace) self._can_write(user, account, container, name) path, node = self._lookup_object(account, container, name) src_version_id, dest_version_id = self._put_metadata(user, node, domain, meta, replace) @@ -593,7 +636,7 @@ class ModularBackend(BaseBackend): from which the object gets its permissions from, along with a dictionary containing the permissions.""" - logger.debug("get_object_permissions: %s %s %s", account, container, name) + logger.debug("get_object_permissions: %s %s %s %s", user, account, container, name) allowed = 'write' permissions_path = self._get_permissions_path(account, container, name) if user != account: @@ -610,7 +653,7 @@ class ModularBackend(BaseBackend): def update_object_permissions(self, user, account, container, name, permissions): """Update the permissions associated with the object.""" - logger.debug("update_object_permissions: %s %s %s %s", account, container, name, permissions) + logger.debug("update_object_permissions: %s %s %s %s %s", user, account, container, name, permissions) if user != account: raise NotAllowedError path = self._lookup_object(account, container, name)[0] @@ -622,7 +665,7 @@ class ModularBackend(BaseBackend): def get_object_public(self, user, account, container, name): """Return the public id of the object if applicable.""" - logger.debug("get_object_public: %s %s %s", account, container, name) + logger.debug("get_object_public: %s %s %s %s", user, account, container, name) self._can_read(user, account, container, name) path = self._lookup_object(account, container, name)[0] p = self.permissions.public_get(path) @@ -634,7 +677,7 @@ class ModularBackend(BaseBackend): def update_object_public(self, user, account, container, name, public): """Update the public status of the object.""" - logger.debug("update_object_public: %s %s %s %s", account, container, name, public) + logger.debug("update_object_public: %s %s %s %s %s", user, account, container, name, public) self._can_write(user, account, container, name) path = self._lookup_object(account, container, name)[0] if not public: @@ -646,7 +689,7 @@ class ModularBackend(BaseBackend): def get_object_hashmap(self, user, account, container, name, version=None): """Return the object's size and a list with partial hashes.""" - logger.debug("get_object_hashmap: %s %s %s %s", account, container, name, version) + logger.debug("get_object_hashmap: %s %s %s %s %s", user, account, container, name, version) self._can_read(user, account, container, name) path, node = self._lookup_object(account, container, name) props = self._get_version(node, version) @@ -694,7 +737,7 @@ class ModularBackend(BaseBackend): def update_object_hashmap(self, user, account, container, name, size, type, hashmap, checksum, domain, meta={}, replace_meta=False, permissions=None): """Create/update an object with the specified size and partial hashes.""" - logger.debug("update_object_hashmap: %s %s %s %s %s %s %s", account, container, name, size, type, hashmap, checksum) + logger.debug("update_object_hashmap: %s %s %s %s %s %s %s %s", user, account, container, name, size, type, hashmap, checksum) if size == 0: # No such thing as an empty hashmap. hashmap = [self.put_block('')] map = HashMap(self.block_size, self.hash_algorithm) @@ -714,7 +757,7 @@ class ModularBackend(BaseBackend): def update_object_checksum(self, user, account, container, name, version, checksum): """Update an object's checksum.""" - logger.debug("update_object_checksum: %s %s %s %s %s", account, container, name, version, checksum) + logger.debug("update_object_checksum: %s %s %s %s %s %s", user, account, container, name, version, checksum) # Update objects with greater version and same hashmap and size (fix metadata updates). self._can_write(user, account, container, name) path, node = self._lookup_object(account, container, name) @@ -724,7 +767,8 @@ class ModularBackend(BaseBackend): if x[self.SERIAL] >= int(version) and x[self.HASH] == props[self.HASH] and x[self.SIZE] == props[self.SIZE]: self.node.version_put_property(x[self.SERIAL], 'checksum', checksum) - def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, dest_domain=None, dest_meta={}, replace_meta=False, permissions=None, src_version=None, is_move=False): + def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, dest_domain=None, dest_meta={}, replace_meta=False, permissions=None, src_version=None, is_move=False, delimiter=None): + dest_version_ids = [] self._can_read(user, src_account, src_container, src_name) path, node = self._lookup_object(src_account, src_container, src_name) # TODO: Will do another fetch of the properties in duplicate version... @@ -732,32 +776,49 @@ class ModularBackend(BaseBackend): src_version_id = props[self.SERIAL] hash = props[self.HASH] size = props[self.SIZE] - is_copy = not is_move and (src_account, src_container, src_name) != (dest_account, dest_container, dest_name) # New uuid. - dest_version_id = self._update_object_hash(user, dest_account, dest_container, dest_name, size, type, hash, None, dest_domain, dest_meta, replace_meta, permissions, src_node=node, src_version_id=src_version_id, is_copy=is_copy) - return dest_version_id + dest_version_ids.append(self._update_object_hash(user, dest_account, dest_container, dest_name, size, type, hash, None, dest_domain, dest_meta, replace_meta, permissions, src_node=node, src_version_id=src_version_id, is_copy=is_copy)) + if is_move and (src_account, src_container, src_name) != (dest_account, dest_container, dest_name): + self._delete_object(user, src_account, src_container, src_name) + + if delimiter: + prefix = src_name + delimiter if not src_name.endswith(delimiter) else src_name + src_names = self._list_objects_no_limit(user, src_account, src_container, prefix, delimiter=None, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None, all_props=True, public=False) + paths = [elem[0] for elem in src_names] + nodes = [elem[2] for elem in src_names] + # TODO: Will do another fetch of the properties in duplicate version... + props = self._get_versions(nodes) # Check to see if source exists. + + for prop, path, node in zip(props, paths, nodes): + src_version_id = prop[self.SERIAL] + hash = prop[self.HASH] + vtype = prop[self.TYPE] + dest_prefix = dest_name + delimiter if not dest_name.endswith(delimiter) else dest_name + vdest_name = path.replace(prefix, dest_prefix, 1) + dest_version_ids.append(self._update_object_hash(user, dest_account, dest_container, vdest_name, size, vtype, hash, None, dest_domain, dest_meta, replace_meta, permissions, src_node=node, src_version_id=src_version_id, is_copy=is_copy)) + if is_move and (src_account, src_container, src_name) != (dest_account, dest_container, dest_name): + self._delete_object(user, src_account, src_container, path) + return dest_version_ids[0] if len(dest_version_ids) == 1 else dest_version_ids @backend_method - def copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, src_version=None): + def copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, src_version=None, delimiter=None): """Copy an object's data and metadata.""" - logger.debug("copy_object: %s %s %s %s %s %s %s %s %s %s %s %s", src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, src_version) - dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, src_version, False) + logger.debug("copy_object: %s %s %s %s %s %s %s %s %s %s %s %s %s %s", user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, src_version, delimiter) + dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, src_version, False, delimiter) return dest_version_id @backend_method - def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None): + def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, delimiter=None): """Move an object's data and metadata.""" - logger.debug("move_object: %s %s %s %s %s %s %s %s %s %s %s", src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions) + logger.debug("move_object: %s %s %s %s %s %s %s %s %s %s %s %s %s", user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, delimiter) if user != src_account: raise NotAllowedError - dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, None, True) - if (src_account, src_container, src_name) != (dest_account, dest_container, dest_name): - self._delete_object(user, src_account, src_container, src_name) + dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, None, True, delimiter) return dest_version_id - def _delete_object(self, user, account, container, name, until=None): + def _delete_object(self, user, account, container, name, until=None, delimiter=None): if user != account: raise NotAllowedError @@ -791,19 +852,34 @@ class ModularBackend(BaseBackend): self._report_size_change(user, account, -del_size, {'action': 'object delete'}) self._report_object_change(user, account, path, details={'action': 'object delete'}) self.permissions.access_clear(path) + + if delimiter: + prefix = name + delimiter if not name.endswith(delimiter) else name + src_names = self._list_objects_no_limit(user, account, container, prefix, delimiter=None, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None, all_props=True, public=False) + paths = [] + for t in src_names: + path = '/'.join((account, container, t[0])) + node = t[2] + src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, type='', hash=None, checksum='', cluster=CLUSTER_DELETED) + del_size = self._apply_versioning(account, container, src_version_id) + if del_size: + self._report_size_change(user, account, -del_size, {'action': 'object delete'}) + self._report_object_change(user, account, path, details={'action': 'object delete'}) + paths.append(path) + self.permissions.access_clear_bulk(paths) @backend_method - def delete_object(self, user, account, container, name, until=None): + def delete_object(self, user, account, container, name, until=None, prefix='', delimiter=None): """Delete/purge an object.""" - logger.debug("delete_object: %s %s %s %s", account, container, name, until) - self._delete_object(user, account, container, name, until) + logger.debug("delete_object: %s %s %s %s %s %s %s", user, account, container, name, until, prefix, delimiter) + self._delete_object(user, account, container, name, until, delimiter) @backend_method def list_versions(self, user, account, container, name): """Return a list of all (version, version_timestamp) tuples for an object.""" - logger.debug("list_versions: %s %s %s", account, container, name) + logger.debug("list_versions: %s %s %s %s", user, account, container, name) self._can_read(user, account, container, name) path, node = self._lookup_object(account, container, name) versions = self.node.node_get_versions(node) @@ -813,7 +889,7 @@ class ModularBackend(BaseBackend): def get_uuid(self, user, uuid): """Return the (account, container, name) for the UUID given.""" - logger.debug("get_uuid: %s", uuid) + logger.debug("get_uuid: %s %s", user, uuid) info = self.node.latest_uuid(uuid) if info is None: raise NameError @@ -826,7 +902,7 @@ class ModularBackend(BaseBackend): def get_public(self, user, public): """Return the (account, container, name) for the public id given.""" - logger.debug("get_public: %s", public) + logger.debug("get_public: %s %s", user, public) if public is None or public < ULTIMATE_ANSWER: raise NameError path = self.permissions.public_path(public - ULTIMATE_ANSWER) @@ -900,6 +976,12 @@ class ModularBackend(BaseBackend): raise NameError('Object does not exist') return path, node + def _lookup_objects(self, paths): + nodes = self.node.node_lookup_bulk(paths) + if nodes is None: + raise NameError('Object does not exist') + return paths, nodes + def _get_properties(self, node, until=None): """Return properties until the timestamp given.""" @@ -936,6 +1018,21 @@ class ModularBackend(BaseBackend): if props is None or props[self.CLUSTER] == CLUSTER_DELETED: raise IndexError('Version does not exist') return props + + def _get_versions(self, nodes, version=None): + if version is None: + props = self.node.version_lookup_bulk(nodes, inf, CLUSTER_NORMAL) + if not props: + raise NameError('Object does not exist') + else: + try: + version = int(version) + except ValueError: + raise IndexError('Version does not exist') + props = self.node.version_get_properties(version) + if props is None or props[self.CLUSTER] == CLUSTER_DELETED: + raise IndexError('Version does not exist') + return props def _put_version_duplicate(self, user, node, src_node=None, size=None, type=None, hash=None, checksum=None, cluster=CLUSTER_NORMAL, is_copy=False): """Create a new version of the node.""" @@ -1015,10 +1112,8 @@ class ModularBackend(BaseBackend): objects.extend([(p, None) for p in prefixes] if virtual else []) objects.sort(key=lambda x: x[0]) objects = [(x[0][len(cont_prefix):],) + x[1:] for x in objects] + return objects - start, limit = self._list_limits([x[0] for x in objects], marker, limit) - return objects[start:start + limit] - # Reporting functions. def _report_size_change(self, user, account, size, details={}): diff --git a/snf-pithos-tools/pithos/tools/lib/client.py b/snf-pithos-tools/pithos/tools/lib/client.py index 5dec80b..d45f33e 100644 --- a/snf-pithos-tools/pithos/tools/lib/client.py +++ b/snf-pithos-tools/pithos/tools/lib/client.py @@ -460,7 +460,7 @@ class OOS_Client(Client): def _change_obj_location(self, src_container, src_object, dst_container, dst_object, remove=False, meta={}, account=None, - content_type=None, **headers): + content_type=None, delimiter=None, **headers): account = account or self.account path = '/%s/%s/%s' % (account, dst_container, dst_object) headers = {} if not headers else headers @@ -476,16 +476,18 @@ class OOS_Client(Client): headers['content_type'] = content_type else: params['ignore_content_type'] = '' + if delimiter: + params['delimiter'] = delimiter return self.put(path, headers=headers, params=params) def copy_object(self, src_container, src_object, dst_container, dst_object, - meta={}, account=None, content_type=None, **headers): + meta={}, account=None, content_type=None, delimiter=None, **headers): """copies an object""" account = account or self.account return self._change_obj_location(src_container, src_object, dst_container, dst_object, account=account, remove=False, meta=meta, - content_type=content_type, **headers) + content_type=content_type, delimiter=delimiter, **headers) def move_object(self, src_container, src_object, dst_container, dst_object, meta={}, account=None, @@ -760,6 +762,18 @@ class Pithos_Client(OOS_Client): return OOS_Client.create_zero_length_object(self, container, object, **args) + def create_folder(self, container, name, + meta={}, etag=None, + content_encoding=None, + content_disposition=None, + x_object_manifest=None, x_object_sharing=None, + x_object_public=None, account=None): + args = locals().copy() + for elem in ['self', 'container', 'name']: + args.pop(elem) + args['content_type'] = 'application/directory' + return self.create_zero_length_object(container, name, **args) + def create_object(self, container, object, f=stdin, format='text', meta={}, params={}, etag=None, content_type=None, content_encoding=None, content_disposition=None, @@ -870,10 +884,12 @@ class Pithos_Client(OOS_Client): args['x_source_object'] = source return self.update_object(container, object, f=None, **args) - def delete_object(self, container, object, until=None, account=None): + def delete_object(self, container, object, until=None, account=None, delimiter=None): """deletes an object or the object history until the date provided""" account = account or self.account params = {'until':until} if until else {} + if delimiter: + params['delimiter'] = delimiter return OOS_Client.delete_object(self, container, object, params, account) def trash_object(self, container, object): @@ -908,7 +924,7 @@ class Pithos_Client(OOS_Client): def copy_object(self, src_container, src_object, dst_container, dst_object, meta={}, public=False, version=None, account=None, - content_type=None): + content_type=None, delimiter=None): """copies an object""" account = account or self.account headers = {} @@ -918,17 +934,19 @@ class Pithos_Client(OOS_Client): return OOS_Client.copy_object(self, src_container, src_object, dst_container, dst_object, meta=meta, account=account, content_type=content_type, + delimiter=delimiter, **headers) def move_object(self, src_container, src_object, dst_container, dst_object, meta={}, public=False, - account=None, content_type=None): + account=None, content_type=None, delimiter=None): """moves an object""" headers = {} headers['x_object_public'] = public return OOS_Client.move_object(self, src_container, src_object, dst_container, dst_object, meta=meta, account=account, content_type=content_type, + delimiter=delimiter, **headers) def list_shared_by_others(self, limit=None, marker=None, format='text'): diff --git a/snf-pithos-tools/pithos/tools/sh.py b/snf-pithos-tools/pithos/tools/sh.py index 0a18478..e29c83b 100755 --- a/snf-pithos-tools/pithos/tools/sh.py +++ b/snf-pithos-tools/pithos/tools/sh.py @@ -143,6 +143,11 @@ class List(Command): default=None, help='show metadata until that date') parser.add_option('--format', action='store', dest='format', default='%d/%m/%Y', help='format to parse until date') + parser.add_option('--shared', action='store_true', dest='shared', + default=False, help='show only shared') + parser.add_option('--public', action='store_true', dest='public', + default=False, help='show only public') + def execute(self, container=None): if container: @@ -152,7 +157,7 @@ class List(Command): def list_containers(self): attrs = ['limit', 'marker', 'if_modified_since', - 'if_unmodified_since'] + 'if_unmodified_since', 'shared', 'public'] args = self._build_args(attrs) args['format'] = 'json' if self.detail else 'text' @@ -167,7 +172,8 @@ class List(Command): #prepate params params = {} attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path', - 'meta', 'if_modified_since', 'if_unmodified_since'] + 'meta', 'if_modified_since', 'if_unmodified_since', + 'shared', 'public'] args = self._build_args(attrs) args['format'] = 'json' if self.detail else 'text' @@ -256,6 +262,12 @@ class Delete(Command): default=None, help='remove history until that date') parser.add_option('--format', action='store', dest='format', default='%d/%m/%Y', help='format to parse until date') + parser.add_option('--delimiter', action='store', type='str', + dest='delimiter', default=None, + help='mass delete objects with path staring with + delimiter') + parser.add_option('-r', action='store_true', + dest='recursive', default=False, + help='mass delimiter objects with delimiter /') def execute(self, path): container, sep, object = path.partition('/') @@ -265,7 +277,12 @@ class Delete(Command): until = int(_time.mktime(t)) if object: - self.client.delete_object(container, object, until) + kwargs = {} + if self.delimiter: + kwargs['delimiter'] = self.delimiter + elif self.recursive: + kwargs['delimiter'] = '/' + self.client.delete_object(container, object, until, **kwargs) else: self.client.delete_container(container, until) @@ -447,6 +464,12 @@ class CopyObject(Command): parser.add_option('--content-type', action='store', dest='content_type', default=None, help='change object\'s content type') + parser.add_option('--delimiter', action='store', type='str', + dest='delimiter', default=None, + help='mass copy objects with path staring with + delimiter') + parser.add_option('-r', action='store_true', + dest='recursive', default=False, + help='mass copy with delimiter /') def execute(self, src, dst, *args): src_container, sep, src_object = src.partition('/') @@ -463,6 +486,10 @@ class CopyObject(Command): dst_object = dst args = {'content_type':self.content_type} if self.content_type else {} + if self.delimiter: + args['delimiter'] = self.delimiter + elif self.recursive: + args['delimiter'] = '/' self.client.copy_object(src_container, src_object, dst_container, dst_object, meta, self.public, self.version, **args) @@ -576,6 +603,12 @@ class MoveObject(Command): parser.add_option('--content-type', action='store', dest='content_type', default=None, help='change object\'s content type') + parser.add_option('--delimiter', action='store', type='str', + dest='delimiter', default=None, + help='mass move objects with path staring with + delimiter') + parser.add_option('-r', action='store_true', + dest='recursive', default=False, + help='mass move objects with delimiter /') def execute(self, src, dst, *args): src_container, sep, src_object = src.partition('/') @@ -591,6 +624,10 @@ class MoveObject(Command): meta[key] = val args = {'content_type':self.content_type} if self.content_type else {} + if self.delimiter: + args['delimiter'] = self.delimiter + elif self.recursive: + args['delimiter'] = '/' self.client.move_object(src_container, src_object, dst_container, dst_object, meta, self.public, **args) diff --git a/snf-pithos-tools/pithos/tools/test.py b/snf-pithos-tools/pithos/tools/test.py index d3e5dcc..803d9ce 100755 --- a/snf-pithos-tools/pithos/tools/test.py +++ b/snf-pithos-tools/pithos/tools/test.py @@ -538,7 +538,63 @@ class ContainerGet(BaseTestCase): self.obj.append(self.upload_random_data(self.container[0], o)) for o in o_names[8:]: self.obj.append(self.upload_random_data(self.container[1], o)) - + + def test_list_shared(self): + self.client.share_object(self.container[0], self.obj[0]['name'], ('*',)) + objs = self.client.list_objects(self.container[0], shared=True) + self.assertEqual(objs, [self.obj[0]['name']]) + + # create child object + self.upload_random_data(self.container[0], strnextling(self.obj[0]['name'])) + objs = self.client.list_objects(self.container[0], shared=True) + self.assertEqual(objs, [self.obj[0]['name']]) + + # test inheritance + self.client.create_folder(self.container[1], 'folder') + self.client.share_object(self.container[1], 'folder', ('*',)) + self.upload_random_data(self.container[1], 'folder/object') + objs = self.client.list_objects(self.container[1], shared=True) + self.assertEqual(objs, ['folder', 'folder/object']) + + def test_list_public(self): + self.client.publish_object(self.container[0], self.obj[0]['name']) + objs = self.client.list_objects(self.container[0], public=True) + self.assertEqual(objs, [self.obj[0]['name']]) + + # create child object + self.upload_random_data(self.container[0], strnextling(self.obj[0]['name'])) + objs = self.client.list_objects(self.container[0], public=True) + self.assertEqual(objs, [self.obj[0]['name']]) + + # test inheritance + self.client.create_folder(self.container[1], 'folder') + self.client.publish_object(self.container[1], 'folder') + self.upload_random_data(self.container[1], 'folder/object') + objs = self.client.list_objects(self.container[1], public=True) + self.assertEqual(objs, ['folder']) + + def test_list_shared_public(self): + self.client.share_object(self.container[0], self.obj[0]['name'], ('*',)) + self.client.publish_object(self.container[0], self.obj[1]['name']) + objs = self.client.list_objects(self.container[0], shared=True, public=True) + self.assertEqual(objs, [self.obj[0]['name'], self.obj[1]['name']]) + + # create child object + self.upload_random_data(self.container[0], strnextling(self.obj[0]['name'])) + self.upload_random_data(self.container[0], strnextling(self.obj[1]['name'])) + objs = self.client.list_objects(self.container[0], shared=True, public=True) + self.assertEqual(objs, [self.obj[0]['name'], self.obj[1]['name']]) + + # test inheritance + self.client.create_folder(self.container[1], 'folder1') + self.client.share_object(self.container[1], 'folder1', ('*',)) + self.upload_random_data(self.container[1], 'folder1/object') + self.client.create_folder(self.container[1], 'folder2') + self.client.publish_object(self.container[1], 'folder2') + o = self.upload_random_data(self.container[1], 'folder2/object') + objs = self.client.list_objects(self.container[1], shared=True, public=True) + self.assertEqual(objs, ['folder1', 'folder1/object', 'folder2']) + def test_list_objects(self): objects = self.client.list_objects(self.container[0]) l = [elem['name'] for elem in self.obj[:8]] @@ -1216,8 +1272,8 @@ class ObjectPut(BaseTestCase): def _test_maximum_upload_size_exceeds(self): name = o_names[0] meta = {'test':'test1'} - #upload 100MB - length=1024*1024*100 + #upload 5GB + length= 5 * (1024 * 1024 * 1024) + 1 self.assert_raises_fault(400, self.upload_random_data, self.container, name, length, **meta) @@ -1373,7 +1429,23 @@ class ObjectCopy(BaseTestCase): self.assert_raises_fault(404, self.client.copy_object, self.containers[1], self.obj['name'], self.containers[1], 'testcopy', meta) - + + def test_copy_dir(self): + self.client.create_folder(self.containers[0], 'dir') + objects = ('object1', 'subdir/object2', 'dirs') + for name in objects[:-1]: + self.upload_random_data(self.containers[0], 'dir/%s' % name) + self.upload_random_data(self.containers[0], 'dirs') + + self.client.copy_object(self.containers[0], 'dir', self.containers[1], 'dir-backup', delimiter='/') + self.assert_object_exists(self.containers[0], 'dir') + self.assert_object_not_exists(self.containers[1], 'dirs') + for name in objects[:-1]: + meta0 = self.client.retrieve_object_metadata(self.containers[0], 'dir/%s' % name) + meta1 = self.client.retrieve_object_metadata(self.containers[1], 'dir-backup/%s' % name) + t = ('content-length', 'x-object-hash', 'content-type') + (self.assertEqual(meta0[elem], meta1[elem]) for elem in t) + class ObjectMove(BaseTestCase): def setUp(self): BaseTestCase.setUp(self) @@ -1409,6 +1481,26 @@ class ObjectMove(BaseTestCase): #assert src object no more exists self.assert_object_not_exists(self.containers[0], self.obj['name']) + + + def test_move_dir(self): + self.client.create_folder(self.containers[0], 'dir') + objects = ('object1', 'subdir/object2', 'dirs') + meta = {} + for name in objects[:-1]: + self.upload_random_data(self.containers[0], 'dir/%s' % name) + meta[name] = self.client.retrieve_object_metadata(self.containers[0], 'dir/%s' % name) + self.upload_random_data(self.containers[0], 'dirs') + + self.client.move_object(self.containers[0], 'dir', self.containers[1], 'dir-backup', delimiter='/', content_type='application/folder') + self.assert_object_not_exists(self.containers[0], 'dir') + self.assert_object_not_exists(self.containers[1], 'dirs') + for name in objects[:-1]: + self.assert_object_not_exists(self.containers[0], 'dir/%s' % name) + self.assert_object_exists(self.containers[1], 'dir-backup/%s' % name) + meta1 = self.client.retrieve_object_metadata(self.containers[1], 'dir-backup/%s' % name) + t = ('content-length', 'x-object-hash', 'content-type') + (self.assertEqual(meta[name][elem], meta1[elem]) for elem in t) class ObjectPost(BaseTestCase): def setUp(self): @@ -1554,12 +1646,12 @@ class ObjectPost(BaseTestCase): self.assert_raises_fault(400, self.test_update_object, content_length = 1000) - def _test_update_object_invalid_range(self): + def test_update_object_invalid_range(self): with AssertContentInvariant(self.client.retrieve_object, self.containers[0], self.obj[0]['name']): self.assert_raises_fault(416, self.test_update_object, 499, 0, True) - def _test_update_object_invalid_range_and_length(self): + def test_update_object_invalid_range_and_length(self): with AssertContentInvariant(self.client.retrieve_object, self.containers[0], self.obj[0]['name']): self.assert_raises_fault([400, 416], self.test_update_object, 499, 0, True, @@ -1704,6 +1796,19 @@ class ObjectDelete(BaseTestCase): #assert item not found self.assert_raises_fault(404, self.client.delete_object, self.containers[1], self.obj['name']) + + def test_delete_dir(self): + self.client.create_folder(self.containers[0], 'dir') + objects = ('object1', 'subdir/object2', 'dirs') + for name in objects[:-1]: + self.upload_random_data(self.containers[0], 'dir/%s' % name) + self.upload_random_data(self.containers[0], 'dirs') + + self.client.delete_object(self.containers[0], 'dir', delimiter='/') + self.assert_object_not_exists(self.containers[0], 'dir') + self.assert_object_exists(self.containers[0], 'dirs') + for name in objects[:-1]: + self.assert_object_not_exists(self.containers[0], 'dir/%s' % name) class ListSharing(BaseTestCase): def setUp(self): @@ -2273,7 +2378,6 @@ class AssertMappingInvariant(object): for k, v in self.map.items(): if is_date(v): continue - #print '#', k, v, map[k] assert(k in map) assert v == map[k] @@ -2327,6 +2431,26 @@ def is_date(date): return True return False +def strnextling(prefix): + """Return the first unicode string + greater than but not starting with given prefix. + strnextling('hello') -> 'hellp' + """ + if not prefix: + ## all strings start with the null string, + ## therefore we have to approximate strnextling('') + ## with the last unicode character supported by python + ## 0x10ffff for wide (32-bit unicode) python builds + ## 0x00ffff for narrow (16-bit unicode) python builds + ## We will not autodetect. 0xffff is safe enough. + return unichr(0xffff) + s = prefix[:-1] + c = ord(prefix[-1]) + if c >= 0xffff: + raise RuntimeError + s += unichr(c+1) + return s + o_names = ['kate.jpg', 'kate_beckinsale.jpg', 'How To Win Friends And Influence People.pdf',