Clean up, sort out logging.
[pithos] / pithos / api / functions.py
index 39360c9..e852459 100644 (file)
@@ -10,16 +10,20 @@ from django.utils.http import http_date, parse_etags
 try:\r
     from django.utils.http import parse_http_date_safe\r
 except:\r
-    from pithos.api.util import parse_http_date_safe\r
+    from pithos.api.compat import parse_http_date_safe\r
 \r
-from pithos.api.faults import Fault, NotModified, BadRequest, Unauthorized, LengthRequired, PreconditionFailed, RangeNotSatisfiable, UnprocessableEntity\r
-from pithos.api.util import get_object_meta, get_range, api_method\r
-\r
-from pithos.backends.dummy_debug import *\r
+from pithos.api.faults import Fault, NotModified, BadRequest, Unauthorized, ItemNotFound, Conflict, LengthRequired, PreconditionFailed, RangeNotSatisfiable, UnprocessableEntity\r
+from pithos.api.util import get_meta, get_range, api_method\r
+from pithos.backends.dummy import BackEnd\r
 \r
+import os\r
+import datetime\r
 import logging\r
 \r
-logging.basicConfig(level=logging.DEBUG)\r
+from settings import PROJECT_PATH\r
+STORAGE_PATH = os.path.join(PROJECT_PATH, 'data')\r
+\r
+logger = logging.getLogger(__name__)\r
 \r
 @api_method('GET')\r
 def authenticate(request):\r
@@ -35,9 +39,8 @@ def authenticate(request):
         raise BadRequest('Missing auth user or key.')\r
     \r
     response = HttpResponse(status = 204)\r
-    response['X-Auth-Token'] = 'eaaafd18-0fed-4b3a-81b4-663c99ec1cbb'\r
-    # TODO: Do we support redirections?\r
-    #response['X-Storage-Url'] = 'https://storage.grnet.gr/pithos/v1.0/<some reference>'\r
+    response['X-Auth-Token'] = '0000'\r
+    response['X-Storage-Url'] = os.path.join(request.build_absolute_uri(), 'demo')\r
     return response\r
 \r
 def account_demux(request, v_account):\r
@@ -45,6 +48,8 @@ def account_demux(request, v_account):
         return account_meta(request, v_account)\r
     elif request.method == 'GET':\r
         return container_list(request, v_account)\r
+    elif request.method == 'POST':\r
+        return account_update(request, v_account)\r
     else:\r
         return method_not_allowed(request)\r
 \r
@@ -55,6 +60,8 @@ def container_demux(request, v_account, v_container):
         return object_list(request, v_account, v_container)\r
     elif request.method == 'PUT':\r
         return container_create(request, v_account, v_container)\r
+    elif request.method == 'POST':\r
+        return container_update(request, v_account, v_container)\r
     elif request.method == 'DELETE':\r
         return container_delete(request, v_account, v_container)\r
     else:\r
@@ -80,21 +87,43 @@ def object_demux(request, v_account, v_container, v_object):
 def account_meta(request, v_account):\r
     # Normal Response Codes: 204\r
     # Error Response Codes: serviceUnavailable (503),\r
-    #                       itemNotFound (404),\r
     #                       unauthorized (401),\r
     #                       badRequest (400)\r
     \r
-    container_count, bytes_count = get_account_meta(request.user)\r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        info = be.get_account_meta(request.user)\r
+    except NameError:\r
+        info = {'count': 0, 'bytes': 0}\r
     \r
     response = HttpResponse(status = 204)\r
-    response['X-Account-Container-Count'] = container_count\r
-    response['X-Account-Total-Bytes-Used'] = bytes_count\r
+    response['X-Account-Container-Count'] = info['count']\r
+    response['X-Account-Bytes-Used'] = info['bytes']\r
+    for k in [x for x in info.keys() if x.startswith('X-Account-Meta-')]:\r
+        response[k.encode('utf-8')] = info[k].encode('utf-8')\r
+    \r
     return response\r
 \r
+@api_method('POST')\r
+def account_update(request, v_account):\r
+    # Normal Response Codes: 202\r
+    # Error Response Codes: serviceUnavailable (503),\r
+    #                       itemNotFound (404),\r
+    #                       unauthorized (401),\r
+    #                       badRequest (400)\r
+    \r
+    meta = get_meta(request, 'X-Account-Meta-')\r
+    \r
+    be = BackEnd(STORAGE_PATH)\r
+    be.update_account_meta(request.user, meta)\r
+    \r
+    return HttpResponse(status = 202)\r
+\r
 @api_method('GET', format_allowed = True)\r
 def container_list(request, v_account):\r
     # Normal Response Codes: 200, 204\r
     # Error Response Codes: serviceUnavailable (503),\r
+    #                       itemNotFound (404),\r
     #                       unauthorized (401),\r
     #                       badRequest (400)\r
     \r
@@ -104,19 +133,29 @@ def container_list(request, v_account):
         try:\r
             limit = int(limit)\r
         except ValueError:\r
-            limit = None\r
-    \r
-    containers = list_containers(request.user, marker, limit)\r
-    if len(containers) == 0:\r
-        return HttpResponse(status = 204)\r
-    \r
+            limit = 10000\r
+    \r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        containers = be.list_containers(request.user, marker, limit)\r
+    except NameError:\r
+        containers = []\r
+    \r
+    if request.serialization == 'text':\r
+        if len(containers) == 0:\r
+            # The cloudfiles python bindings expect 200 if json/xml.\r
+            return HttpResponse(status = 204)\r
+        return HttpResponse('\n'.join(containers), status = 200)\r
+    \r
+    # TODO: Do this with a backend parameter?\r
+    try:\r
+        containers = [be.get_container_meta(request.user, x) for x in containers]\r
+    except NameError:\r
+        raise ItemNotFound()\r
     if request.serialization == 'xml':\r
         data = render_to_string('containers.xml', {'account': request.user, 'containers': containers})\r
     elif request.serialization  == 'json':\r
         data = json.dumps(containers)\r
-    else:\r
-        data = '\n'.join(x['name'] for x in containers)\r
-    \r
     return HttpResponse(data, status = 200)\r
 \r
 @api_method('HEAD')\r
@@ -127,11 +166,18 @@ def container_meta(request, v_account, v_container):
     #                       unauthorized (401),\r
     #                       badRequest (400)\r
     \r
-    object_count, bytes_count = get_container_meta(request.user, v_container)\r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        info = be.get_container_meta(request.user, v_container)\r
+    except NameError:\r
+        raise ItemNotFound()\r
     \r
     response = HttpResponse(status = 204)\r
-    response['X-Container-Object-Count'] = object_count\r
-    response['X-Container-Bytes-Used'] = bytes_count\r
+    response['X-Container-Object-Count'] = info['count']\r
+    response['X-Container-Bytes-Used'] = info['bytes']\r
+    for k in [x for x in info.keys() if x.startswith('X-Container-Meta-')]:\r
+        response[k.encode('utf-8')] = info[k].encode('utf-8')\r
+    \r
     return response\r
 \r
 @api_method('PUT')\r
@@ -141,25 +187,55 @@ def container_create(request, v_account, v_container):
     #                       itemNotFound (404),\r
     #                       unauthorized (401),\r
     #                       badRequest (400)\r
+    \r
+    meta = get_meta(request, 'X-Container-Meta-')\r
+    \r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        be.create_container(request.user, v_container)\r
+        ret = 201\r
+    except NameError:\r
+        ret = 202\r
+    \r
+    if len(meta) > 0:\r
+        be.update_container_meta(request.user, v_container, meta)\r
+    \r
+    return HttpResponse(status = ret)\r
 \r
-    if create_container(request.user, v_container):\r
-        return HttpResponse(status = 201)\r
-    else:\r
-        return HttpResponse(status = 202)\r
+@api_method('POST')\r
+def container_update(request, v_account, v_container):\r
+    # Normal Response Codes: 202\r
+    # Error Response Codes: serviceUnavailable (503),\r
+    #                       itemNotFound (404),\r
+    #                       unauthorized (401),\r
+    #                       badRequest (400)\r
+    \r
+    meta = get_meta(request, 'X-Container-Meta-')\r
+    \r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        be.update_container_meta(request.user, v_container, meta)\r
+    except NameError:\r
+        raise ItemNotFound()\r
+    \r
+    return HttpResponse(status = 202)\r
 \r
 @api_method('DELETE')\r
 def container_delete(request, v_account, v_container):\r
     # Normal Response Codes: 204\r
     # Error Response Codes: serviceUnavailable (503),\r
+    #                       conflict (409),\r
     #                       itemNotFound (404),\r
     #                       unauthorized (401),\r
     #                       badRequest (400)\r
     \r
-    object_count, bytes_count = get_container_meta(request.user, v_container)\r
-    if object_count > 0:\r
-        return HttpResponse(status = 409)\r
-    \r
-    delete_container(request.user, v_container)\r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        be.delete_container(request.user, v_container)\r
+    except NameError:\r
+        raise ItemNotFound()\r
+    except IndexError:\r
+        raise Conflict()\r
     return HttpResponse(status = 204)\r
 \r
 @api_method('GET', format_allowed = True)\r
@@ -173,8 +249,8 @@ def object_list(request, v_account, v_container):
     path = request.GET.get('path')\r
     prefix = request.GET.get('prefix')\r
     delimiter = request.GET.get('delimiter')\r
-    logging.debug("path: %s", path)\r
     \r
+    # TODO: Check if the cloudfiles python bindings expect the results with the prefix.\r
     # Path overrides prefix and delimiter.\r
     if path:\r
         prefix = path\r
@@ -182,6 +258,8 @@ def object_list(request, v_account, v_container):
     # Naming policy.\r
     if prefix and delimiter:\r
         prefix = prefix + delimiter\r
+    if not prefix:\r
+        prefix = ''\r
     \r
     marker = request.GET.get('marker')\r
     limit = request.GET.get('limit')\r
@@ -189,19 +267,33 @@ def object_list(request, v_account, v_container):
         try:\r
             limit = int(limit)\r
         except ValueError:\r
-            limit = None\r
-    \r
-    objects = list_objects(request.user, v_container, prefix, delimiter, marker, limit)\r
-    if len(objects) == 0:\r
-        return HttpResponse(status = 204)\r
-    \r
+            limit = 10000\r
+    \r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        objects = be.list_objects(request.user, v_container, prefix, delimiter, marker, limit)\r
+    except NameError:\r
+        raise ItemNotFound()\r
+    \r
+    if request.serialization == 'text':\r
+        if len(objects) == 0:\r
+            # The cloudfiles python bindings expect 200 if json/xml.\r
+            return HttpResponse(status = 204)\r
+        return HttpResponse('\n'.join(objects), status = 200)\r
+    \r
+    # TODO: Do this with a backend parameter?\r
+    try:\r
+        objects = [be.get_object_meta(request.user, v_container, x) for x in objects]\r
+    except NameError:\r
+        raise ItemNotFound()\r
+    # Format dates.\r
+    for x in objects:\r
+        if x.has_key('last_modified'):\r
+            x['last_modified'] = datetime.datetime.fromtimestamp(x['last_modified']).isoformat()\r
     if request.serialization == 'xml':\r
         data = render_to_string('objects.xml', {'container': v_container, 'objects': objects})\r
     elif request.serialization  == 'json':\r
         data = json.dumps(objects)\r
-    else:\r
-        data = '\n'.join(x['name'] for x in objects)\r
-    \r
     return HttpResponse(data, status = 200)\r
 \r
 @api_method('HEAD')\r
@@ -211,16 +303,20 @@ def object_meta(request, v_account, v_container, v_object):
     #                       itemNotFound (404),\r
     #                       unauthorized (401),\r
     #                       badRequest (400)\r
-\r
-    info = get_object_meta(request.user, v_container, v_object)\r
+    \r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        info = be.get_object_meta(request.user, v_container, v_object)\r
+    except NameError:\r
+        raise ItemNotFound()\r
     \r
     response = HttpResponse(status = 204)\r
     response['ETag'] = info['hash']\r
     response['Content-Length'] = info['bytes']\r
     response['Content-Type'] = info['content_type']\r
     response['Last-Modified'] = http_date(info['last_modified'])\r
-    for k, v in info['meta'].iteritems():\r
-        response['X-Object-Meta-%s' % k.capitalize()] = v\r
+    for k in [x for x in info.keys() if x.startswith('X-Object-Meta-')]:\r
+        response[k.encode('utf-8')] = info[k].encode('utf-8')\r
     \r
     return response\r
 \r
@@ -235,8 +331,13 @@ def object_read(request, v_account, v_container, v_object):
     #                       badRequest (400),\r
     #                       notModified (304)\r
     \r
-    info = get_object_meta(request.user, v_container, v_object)\r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        info = be.get_object_meta(request.user, v_container, v_object)\r
+    except NameError:\r
+        raise ItemNotFound()\r
     \r
+    # TODO: Check if the cloudfiles python bindings expect hash/content_type/last_modified on range requests.\r
     response = HttpResponse()\r
     response['ETag'] = info['hash']\r
     response['Content-Type'] = info['content_type']\r
@@ -246,30 +347,35 @@ def object_read(request, v_account, v_container, v_object):
     range = get_range(request)\r
     if range is not None:\r
         offset, length = range\r
+        if length:\r
+            if offset + length > info['bytes']:\r
+                raise RangeNotSatisfiable()\r
+        else:\r
+            if offset > info['bytes']:\r
+                raise RangeNotSatisfiable()\r
         if not length:\r
-            length = 0\r
-        if offset + length > info['bytes']:\r
-            raise RangeNotSatisfiable()\r
+            length = -1\r
         \r
         response['Content-Length'] = length        \r
         response.status_code = 206\r
     else:\r
         offset = 0\r
-        length = 0\r
+        length = -1\r
         \r
         response['Content-Length'] = info['bytes']\r
         response.status_code = 200\r
     \r
     # Conditions (according to RFC2616 must be evaluated at the end).\r
+    # TODO: Check etag/date conditions.\r
     if_match = request.META.get('HTTP_IF_MATCH')\r
     if if_match is not None and if_match != '*':\r
         if info['hash'] not in parse_etags(if_match):\r
             raise PreconditionFailed()\r
     \r
     if_none_match = request.META.get('HTTP_IF_NONE_MATCH')\r
-#     if if_none_match is not None:\r
-#         if if_none_match = '*' or info['hash'] in parse_etags(if_none_match):\r
-#             raise NotModified()\r
+    if if_none_match is not None:\r
+        if if_none_match == '*' or info['hash'] in parse_etags(if_none_match):\r
+            raise NotModified()\r
     \r
     if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')\r
     if if_modified_since is not None:\r
@@ -283,7 +389,11 @@ def object_read(request, v_account, v_container, v_object):
     if if_unmodified_since is not None and info['last_modified'] > if_unmodified_since:\r
         raise PreconditionFailed()\r
     \r
-    response.content = get_object_data(request.user, v_container, v_object, offset, length)\r
+    try:\r
+        response.content = be.get_object(request.user, v_container, v_object, offset, length)\r
+    except NameError:\r
+        raise ItemNotFound()\r
+    \r
     return response\r
 \r
 @api_method('PUT')\r
@@ -296,6 +406,8 @@ def object_write(request, v_account, v_container, v_object):
     #                       unauthorized (401),\r
     #                       badRequest (400)\r
     \r
+    be = BackEnd(STORAGE_PATH)\r
+    \r
     copy_from = request.META.get('HTTP_X_COPY_FROM')\r
     if copy_from:\r
         parts = copy_from.split('/')\r
@@ -304,21 +416,27 @@ def object_write(request, v_account, v_container, v_object):
         copy_container = parts[1]\r
         copy_name = '/'.join(parts[2:])\r
         \r
-        info = get_object_meta(request.user, copy_container, copy_name)\r
+        try:\r
+            info = be.get_object_meta(request.user, copy_container, copy_name)\r
+        except NameError:\r
+            raise ItemNotFound()\r
         \r
         content_length = request.META.get('CONTENT_LENGTH')\r
         content_type = request.META.get('CONTENT_TYPE')\r
+        # TODO: Why is this required? Copy this ammount?\r
         if not content_length:\r
             raise LengthRequired()\r
         if content_type:\r
             info['content_type'] = content_type\r
         \r
-        meta = get_object_meta(request)\r
-        for k, v in meta.iteritems():\r
-            info['meta'][k] = v\r
+        meta = get_meta(request, 'X-Object-Meta-')\r
+        info.update(meta)\r
         \r
-        copy_object(request.user, copy_container, copy_name, v_container, v_object)\r
-        update_object_meta(request.user, v_container, v_object, info)\r
+        try:\r
+            be.copy_object(request.user, copy_container, copy_name, v_container, v_object)\r
+            be.update_object_meta(request.user, v_container, v_object, info)\r
+        except NameError:\r
+            raise ItemNotFound()\r
         \r
         response = HttpResponse(status = 201)\r
     else:\r
@@ -326,25 +444,29 @@ def object_write(request, v_account, v_container, v_object):
         content_type = request.META.get('CONTENT_TYPE')\r
         if not content_length or not content_type:\r
             raise LengthRequired()\r
-    \r
-        meta = get_object_meta(request)\r
-        info = {'bytes': content_length, 'content_type': content_type, 'meta': meta}\r
-    \r
+        \r
+        info = {'content_type': content_type}\r
+        meta = get_meta(request, 'X-Object-Meta-')\r
+        info.update(meta)\r
+        \r
+        data = request.raw_post_data\r
+        try:\r
+            be.update_object(request.user, v_container, v_object, data)\r
+            be.update_object_meta(request.user, v_container, v_object, info)\r
+        except NameError:\r
+            raise ItemNotFound()\r
+        \r
+        # TODO: Check before update?\r
+        info = be.get_object_meta(request.user, v_container, v_object)\r
         etag = request.META.get('HTTP_ETAG')\r
         if etag:\r
             etag = parse_etags(etag)[0] # TODO: Unescape properly.\r
-            info['hash'] = etag\r
-    \r
-        data = request.read()\r
-        # TODO: Hash function.\r
-        # etag = hash(data)\r
-        # if info.get('hash') and info['hash'] != etag:\r
-        #     raise UnprocessableEntity()\r
-    \r
-        update_object_data(request.user, v_container, v_name, info, data)\r
-    \r
+            if etag != info['hash']:\r
+                be.delete_object(request.user, v_container, v_object)\r
+                raise UnprocessableEntity()\r
+        \r
         response = HttpResponse(status = 201)\r
-        # response['ETag'] = etag\r
+        response['ETag'] = info['hash']\r
     \r
     return response\r
 \r
@@ -365,19 +487,24 @@ def object_copy(request, v_account, v_container, v_object):
         raise BadRequest('Bad Destination path.')\r
     dest_container = parts[1]\r
     dest_name = '/'.join(parts[2:])\r
-        \r
-    info = get_object_meta(request.user, v_container, v_object)\r
-        \r
+    \r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        info = be.get_object_meta(request.user, v_container, v_object)\r
+    except NameError:\r
+        raise ItemNotFound()\r
+    \r
     content_type = request.META.get('CONTENT_TYPE')\r
     if content_type:\r
         info['content_type'] = content_type\r
-        \r
-    meta = get_object_meta(request)\r
-    for k, v in meta.iteritems():\r
-        info['meta'][k] = v\r
+    meta = get_meta(request, 'X-Object-Meta-')\r
+    info.update(meta)\r
     \r
-    copy_object(request.user, v_container, v_object, dest_container, dest_name)\r
-    update_object_meta(request.user, dest_container, dest_name, info)\r
+    try:\r
+        be.copy_object(request.user, v_container, v_object, dest_container, dest_name)\r
+        be.update_object_meta(request.user, dest_container, dest_name, info)\r
+    except NameError:\r
+        raise ItemNotFound()\r
     \r
     response = HttpResponse(status = 201)\r
 \r
@@ -389,9 +516,14 @@ def object_update(request, v_account, v_container, v_object):
     #                       unauthorized (401),\r
     #                       badRequest (400)\r
     \r
-    meta = get_object_meta(request)\r
+    meta = get_meta(request, 'X-Object-Meta-')\r
+    \r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        be.update_object_meta(request.user, v_container, v_object, meta)\r
+    except NameError:\r
+        raise ItemNotFound()\r
     \r
-    update_object_meta(request.user, v_container, v_object, meta)\r
     return HttpResponse(status = 202)\r
 \r
 @api_method('DELETE')\r
@@ -402,7 +534,11 @@ def object_delete(request, v_account, v_container, v_object):
     #                       unauthorized (401),\r
     #                       badRequest (400)\r
     \r
-    delete_object(request.user, v_container, v_object)\r
+    be = BackEnd(STORAGE_PATH)\r
+    try:\r
+        be.delete_object(request.user, v_container, v_object)\r
+    except NameError:\r
+        raise ItemNotFound()\r
     return HttpResponse(status = 204)\r
 \r
 @api_method()\r