perform case sensitive varchar ordering, handle correctly utf8 columns and decimals

Refs #1458

diff --git a/docs/source/adminguide.rst b/docs/source/adminguide.rst
index e7b4563..3db9875 100644 (file)
--- a/docs/source/adminguide.rst
+++ b/docs/source/adminguide.rst
@@ -164,7 +164,7 @@ Edit ``/etc/mysql/my.cnf`` to allow network connections and restart the server.
 
 Create database and user::
 
-  CREATE DATABASE pithos;
+  CREATE DATABASE pithosdb CHARACTER SET utf8 COLLATE utf8_bin;
   GRANT ALL ON pithos.* TO pithos@localhost IDENTIFIED BY 'password';
   GRANT ALL ON pithos.* TO pithos@'%' IDENTIFIED BY 'password';
 

diff --git a/pithos/api/functions.py b/pithos/api/functions.py
index 947014b..ba5fe54 100644 (file)
--- a/pithos/api/functions.py
+++ b/pithos/api/functions.py
@@ -49,7 +49,7 @@ from pithos.api.util import (rename_meta_key, format_header_key, printable_heade
     update_manifest_meta, update_sharing_meta, update_public_meta, validate_modification_preconditions,
     validate_matching_preconditions, split_container_object_string, copy_or_move_object,
     get_int_parameter, get_content_length, get_content_range, socket_read_iterator,
-    object_data_response, put_object_block, hashmap_hash, api_method)
+    object_data_response, put_object_block, hashmap_hash, api_method, json_encode_decimal)
 from pithos.backends import connect_backend
 from pithos.backends.base import NotAllowedError
 
@@ -542,7 +542,7 @@ def object_list(request, v_account, v_container):
     if request.serialization == 'xml':
         data = render_to_string('objects.xml', {'container': v_container, 'objects': object_meta})
     elif request.serialization  == 'json':
-        data = json.dumps(object_meta)
+        data = json.dumps(object_meta, default=json_encode_decimal)
     response.status_code = 200
     response.content = data
     return response
@@ -619,7 +619,7 @@ def object_read(request, v_account, v_container, v_object):
             d['object'] = v_object
             data = render_to_string('versions.xml', d)
         elif request.serialization  == 'json':
-            data = json.dumps(d)
+            data = json.dumps(d, default=json_encode_decimal)
         
         response = HttpResponse(data, status=200)
         response['Content-Length'] = len(data)

diff --git a/pithos/api/util.py b/pithos/api/util.py
index a00e2ca..0e0aaed 100644 (file)
--- a/pithos/api/util.py
+++ b/pithos/api/util.py
@@ -55,7 +55,7 @@ import logging
 import re
 import hashlib
 import uuid
-
+import decimal
 
 logger = logging.getLogger(__name__)
 
@@ -70,6 +70,11 @@ class UTC(tzinfo):
    def dst(self, dt):
        return timedelta(0)
 
+def json_encode_decimal(obj):
+    if isinstance(obj, decimal.Decimal):
+        return str(obj)
+    raise TypeError(repr(obj) + " is not JSON serializable")
+
 def isoformat(d):
    """Return an ISO8601 date string that includes a timezone."""
 

diff --git a/pithos/backends/lib/sqlalchemy/dbwrapper.py b/pithos/backends/lib/sqlalchemy/dbwrapper.py
index 4091a88..7195660 100644 (file)
--- a/pithos/backends/lib/sqlalchemy/dbwrapper.py
+++ b/pithos/backends/lib/sqlalchemy/dbwrapper.py
@@ -46,6 +46,9 @@ class DBWrapper(object):
                 def connect(self, dbapi_con, con_record):
                     db_cursor = dbapi_con.execute('pragma foreign_keys=ON')
             self.engine = create_engine(db, connect_args={'check_same_thread': False}, poolclass=NullPool, listeners=[ForeignKeysListener()])
+        if db.startswith('mysql://'):
+            db = '%s?charset=utf8&use_unicode=0' %db
+            self.engine = create_engine(db, convert_unicode=True)
         else:
             self.engine = create_engine(db)
         #self.engine.echo = True

diff --git a/pithos/backends/lib/sqlalchemy/node.py b/pithos/backends/lib/sqlalchemy/node.py
index b865382..a19d1c1 100644 (file)
--- a/pithos/backends/lib/sqlalchemy/node.py
+++ b/pithos/backends/lib/sqlalchemy/node.py
@@ -32,7 +32,7 @@
 # or implied, of GRNET S.A.
 
 from time import time
-from sqlalchemy import Table, Integer, BigInteger, Float, Column, String, MetaData, ForeignKey
+from sqlalchemy import Table, Integer, BigInteger, DECIMAL, Column, String, MetaData, ForeignKey
 from sqlalchemy.schema import Index, Sequence
 from sqlalchemy.sql import func, and_, or_, null, select, bindparam
 from sqlalchemy.ext.compiler import compiles
@@ -128,7 +128,7 @@ class Node(DBWorker):
                               primary_key=True))
         columns.append(Column('population', Integer, nullable=False, default=0))
         columns.append(Column('size', BigInteger, nullable=False, default=0))
-        columns.append(Column('mtime', Float))
+        columns.append(Column('mtime', DECIMAL))
         columns.append(Column('cluster', Integer, nullable=False, default=0,
                               primary_key=True))
         self.statistics = Table('statistics', metadata, *columns)
@@ -143,7 +143,7 @@ class Node(DBWorker):
         columns.append(Column('hash', String(255)))
         columns.append(Column('size', BigInteger, nullable=False, default=0))
         columns.append(Column('source', Integer))
-        columns.append(Column('mtime', Float))
+        columns.append(Column('mtime', DECIMAL))
         columns.append(Column('muser', String(255), nullable=False, default=''))
         columns.append(Column('cluster', Integer, nullable=False, default=0))
         self.versions = Table('versions', metadata, *columns)