Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-backend / pithos / backends / modular.py @ 8ed3f04c

History | View | Annotate | Download (62.7 kB)

1 2e662088 Antony Chazapis
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2 2715ade4 Sofia Papagiannaki
#
3 a9b3f29d Antony Chazapis
# Redistribution and use in source and binary forms, with or
4 a9b3f29d Antony Chazapis
# without modification, are permitted provided that the following
5 a9b3f29d Antony Chazapis
# conditions are met:
6 2715ade4 Sofia Papagiannaki
#
7 a9b3f29d Antony Chazapis
#   1. Redistributions of source code must retain the above
8 a9b3f29d Antony Chazapis
#      copyright notice, this list of conditions and the following
9 a9b3f29d Antony Chazapis
#      disclaimer.
10 2715ade4 Sofia Papagiannaki
#
11 7ff57991 Antony Chazapis
#   2. Redistributions in binary form must reproduce the above
12 7ff57991 Antony Chazapis
#      copyright notice, this list of conditions and the following
13 a9b3f29d Antony Chazapis
#      disclaimer in the documentation and/or other materials
14 a9b3f29d Antony Chazapis
#      provided with the distribution.
15 2715ade4 Sofia Papagiannaki
#
16 a9b3f29d Antony Chazapis
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17 a9b3f29d Antony Chazapis
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 a9b3f29d Antony Chazapis
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 a9b3f29d Antony Chazapis
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20 a9b3f29d Antony Chazapis
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 a9b3f29d Antony Chazapis
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 a9b3f29d Antony Chazapis
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23 a9b3f29d Antony Chazapis
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 a9b3f29d Antony Chazapis
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 a9b3f29d Antony Chazapis
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26 a9b3f29d Antony Chazapis
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 a9b3f29d Antony Chazapis
# POSSIBILITY OF SUCH DAMAGE.
28 2715ade4 Sofia Papagiannaki
#
29 a9b3f29d Antony Chazapis
# The views and conclusions contained in the software and
30 a9b3f29d Antony Chazapis
# documentation are those of the authors and should not be
31 a9b3f29d Antony Chazapis
# interpreted as representing official policies, either expressed
32 a9b3f29d Antony Chazapis
# or implied, of GRNET S.A.
33 a9b3f29d Antony Chazapis
34 2c5363a0 Antony Chazapis
import sys
35 a9b3f29d Antony Chazapis
import os
36 a9b3f29d Antony Chazapis
import time
37 37bee317 Antony Chazapis
import uuid as uuidlib
38 a9b3f29d Antony Chazapis
import logging
39 6e147ecc Antony Chazapis
import hashlib
40 a9b3f29d Antony Chazapis
import binascii
41 a9b3f29d Antony Chazapis
42 b3119c3e Georgios D. Tsoukalas
from kamaki.clients.quotaholder import QuotaholderClient
43 0307b47f Georgios D. Tsoukalas
44 7efc9f86 Sofia Papagiannaki
from base import DEFAULT_QUOTA, DEFAULT_VERSIONING, NotAllowedError, QuotaError, BaseBackend, \
45 7efc9f86 Sofia Papagiannaki
    AccountExists, ContainerExists, AccountNotEmpty, ContainerNotEmpty, ItemNotExists, VersionNotExists
46 a9b3f29d Antony Chazapis
47 6e147ecc Antony Chazapis
# Stripped-down version of the HashMap class found in tools.
48 2715ade4 Sofia Papagiannaki
49 2715ade4 Sofia Papagiannaki
50 6e147ecc Antony Chazapis
class HashMap(list):
51 6e147ecc Antony Chazapis
52 6e147ecc Antony Chazapis
    def __init__(self, blocksize, blockhash):
53 6e147ecc Antony Chazapis
        super(HashMap, self).__init__()
54 6e147ecc Antony Chazapis
        self.blocksize = blocksize
55 6e147ecc Antony Chazapis
        self.blockhash = blockhash
56 6e147ecc Antony Chazapis
57 6e147ecc Antony Chazapis
    def _hash_raw(self, v):
58 6e147ecc Antony Chazapis
        h = hashlib.new(self.blockhash)
59 6e147ecc Antony Chazapis
        h.update(v)
60 6e147ecc Antony Chazapis
        return h.digest()
61 6e147ecc Antony Chazapis
62 6e147ecc Antony Chazapis
    def hash(self):
63 6e147ecc Antony Chazapis
        if len(self) == 0:
64 6e147ecc Antony Chazapis
            return self._hash_raw('')
65 6e147ecc Antony Chazapis
        if len(self) == 1:
66 6e147ecc Antony Chazapis
            return self.__getitem__(0)
67 6e147ecc Antony Chazapis
68 6e147ecc Antony Chazapis
        h = list(self)
69 6e147ecc Antony Chazapis
        s = 2
70 6e147ecc Antony Chazapis
        while s < len(h):
71 6e147ecc Antony Chazapis
            s = s * 2
72 6e147ecc Antony Chazapis
        h += [('\x00' * len(h[0]))] * (s - len(h))
73 6e147ecc Antony Chazapis
        while len(h) > 1:
74 6e147ecc Antony Chazapis
            h = [self._hash_raw(h[x] + h[x + 1]) for x in range(0, len(h), 2)]
75 6e147ecc Antony Chazapis
        return h[0]
76 5a96180b Antony Chazapis
77 228de81b Antony Chazapis
# Default modules and settings.
78 228de81b Antony Chazapis
DEFAULT_DB_MODULE = 'pithos.backends.lib.sqlalchemy'
79 228de81b Antony Chazapis
DEFAULT_DB_CONNECTION = 'sqlite:///backend.db'
80 228de81b Antony Chazapis
DEFAULT_BLOCK_MODULE = 'pithos.backends.lib.hashfiler'
81 228de81b Antony Chazapis
DEFAULT_BLOCK_PATH = 'data/'
82 f3b65e8f Antony Chazapis
DEFAULT_BLOCK_UMASK = 0o022
83 fa9cae7e Antony Chazapis
#DEFAULT_QUEUE_MODULE = 'pithos.backends.lib.rabbitmq'
84 7f1f0464 Georgios D. Tsoukalas
DEFAULT_BLOCK_PARAMS = { 'mappool': None, 'blockpool': None }
85 f4fbb0fa Sofia Papagiannaki
#DEFAULT_QUEUE_HOSTS = '[amqp://guest:guest@localhost:5672]'
86 f4fbb0fa Sofia Papagiannaki
#DEFAULT_QUEUE_EXCHANGE = 'pithos'
87 fa9cae7e Antony Chazapis
88 8d9a3fbd Antony Chazapis
QUEUE_MESSAGE_KEY_PREFIX = 'pithos.%s'
89 39ef6f41 Antony Chazapis
QUEUE_CLIENT_ID = 'pithos'
90 73673127 Antony Chazapis
QUEUE_INSTANCE_ID = '1'
91 228de81b Antony Chazapis
92 2715ade4 Sofia Papagiannaki
(CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED) = range(3)
93 44ad5860 Antony Chazapis
94 44ad5860 Antony Chazapis
inf = float('inf')
95 44ad5860 Antony Chazapis
96 bb4eafc6 Antony Chazapis
ULTIMATE_ANSWER = 42
97 bb4eafc6 Antony Chazapis
98 a9b3f29d Antony Chazapis
99 a9b3f29d Antony Chazapis
logger = logging.getLogger(__name__)
100 a9b3f29d Antony Chazapis
101 1c2fc0ff Antony Chazapis
102 a9b3f29d Antony Chazapis
def backend_method(func=None, autocommit=1):
103 a9b3f29d Antony Chazapis
    if func is None:
104 a9b3f29d Antony Chazapis
        def fn(func):
105 a9b3f29d Antony Chazapis
            return backend_method(func, autocommit)
106 a9b3f29d Antony Chazapis
        return fn
107 a9b3f29d Antony Chazapis
108 a9b3f29d Antony Chazapis
    if not autocommit:
109 a9b3f29d Antony Chazapis
        return func
110 2715ade4 Sofia Papagiannaki
111 a9b3f29d Antony Chazapis
    def fn(self, *args, **kw):
112 2c5363a0 Antony Chazapis
        self.wrapper.execute()
113 ecd01afd root
        serials = []
114 ecd01afd root
        self.serials = serials
115 85352a77 root
        self.messages = []
116 8313c681 root
117 a9b3f29d Antony Chazapis
        try:
118 a9b3f29d Antony Chazapis
            ret = func(self, *args, **kw)
119 39ef6f41 Antony Chazapis
            for m in self.messages:
120 39ef6f41 Antony Chazapis
                self.queue.send(*m)
121 5578064f root
            if serials:
122 5578064f root
                self.quotaholder.accept_commission(
123 5578064f root
                            context     =   {},
124 5578064f root
                            clientkey   =   'pithos',
125 5578064f root
                            serials     =   serials)
126 a74ba506 Sofia Papagiannaki
            self.wrapper.commit()
127 a9b3f29d Antony Chazapis
            return ret
128 a9b3f29d Antony Chazapis
        except:
129 73fbe301 Sofia Papagiannaki
            if serials:
130 73fbe301 Sofia Papagiannaki
                self.quotaholder.reject_commission(
131 73fbe301 Sofia Papagiannaki
                            context     =   {},
132 73fbe301 Sofia Papagiannaki
                            clientkey   =   'pithos',
133 73fbe301 Sofia Papagiannaki
                            serials     =   serials)
134 2c5363a0 Antony Chazapis
            self.wrapper.rollback()
135 a9b3f29d Antony Chazapis
            raise
136 a9b3f29d Antony Chazapis
    return fn
137 a9b3f29d Antony Chazapis
138 a9b3f29d Antony Chazapis
139 a9b3f29d Antony Chazapis
class ModularBackend(BaseBackend):
140 a9b3f29d Antony Chazapis
    """A modular backend.
141 2715ade4 Sofia Papagiannaki

142 e9363f82 Antony Chazapis
    Uses modules for SQL functions and storage.
143 a9b3f29d Antony Chazapis
    """
144 2715ade4 Sofia Papagiannaki
145 46286f5f Antony Chazapis
    def __init__(self, db_module=None, db_connection=None,
146 f3b65e8f Antony Chazapis
                 block_module=None, block_path=None, block_umask=None,
147 3173699c Sofia Papagiannaki
                 queue_module=None, queue_hosts=None, queue_exchange=None,
148 3173699c Sofia Papagiannaki
                 quotaholder_url=None, quotaholder_token=None,
149 7f1f0464 Georgios D. Tsoukalas
                 free_versioning=True, block_params=None):
150 228de81b Antony Chazapis
        db_module = db_module or DEFAULT_DB_MODULE
151 228de81b Antony Chazapis
        db_connection = db_connection or DEFAULT_DB_CONNECTION
152 228de81b Antony Chazapis
        block_module = block_module or DEFAULT_BLOCK_MODULE
153 228de81b Antony Chazapis
        block_path = block_path or DEFAULT_BLOCK_PATH
154 f3b65e8f Antony Chazapis
        block_umask = block_umask or DEFAULT_BLOCK_UMASK
155 7f1f0464 Georgios D. Tsoukalas
        block_params = block_params or DEFAULT_BLOCK_PARAMS
156 46286f5f Antony Chazapis
        #queue_module = queue_module or DEFAULT_QUEUE_MODULE
157 7f1f0464 Georgios D. Tsoukalas
158 7f1f0464 Georgios D. Tsoukalas
        self.default_policy = {'quota': DEFAULT_QUOTA, 'versioning': DEFAULT_VERSIONING}
159 f4fbb0fa Sofia Papagiannaki
        #queue_hosts = queue_hosts or DEFAULT_QUEUE_HOSTS
160 f4fbb0fa Sofia Papagiannaki
        #queue_exchange = queue_exchange or DEFAULT_QUEUE_EXCHANGE
161 7f1f0464 Georgios D. Tsoukalas
162 a9b3f29d Antony Chazapis
        self.hash_algorithm = 'sha256'
163 2715ade4 Sofia Papagiannaki
        self.block_size = 4 * 1024 * 1024  # 4MB
164 b1dadd0e Sofia Papagiannaki
        self.free_versioning = free_versioning
165 2715ade4 Sofia Papagiannaki
166 2715ade4 Sofia Papagiannaki
        self.default_policy = {'quota': DEFAULT_QUOTA,
167 2715ade4 Sofia Papagiannaki
                               'versioning': DEFAULT_VERSIONING}
168 2715ade4 Sofia Papagiannaki
169 46286f5f Antony Chazapis
        def load_module(m):
170 46286f5f Antony Chazapis
            __import__(m)
171 46286f5f Antony Chazapis
            return sys.modules[m]
172 2715ade4 Sofia Papagiannaki
173 46286f5f Antony Chazapis
        self.db_module = load_module(db_module)
174 46286f5f Antony Chazapis
        self.wrapper = self.db_module.DBWrapper(db_connection)
175 e9363f82 Antony Chazapis
        params = {'wrapper': self.wrapper}
176 e9363f82 Antony Chazapis
        self.permissions = self.db_module.Permissions(**params)
177 2c690fe9 Sofia Papagiannaki
        self.config = self.db_module.Config(**params)
178 cb787cc4 Sofia Papagiannaki
        self.quotaholder_serials = self.db_module.QuotaholderSerial(**params)
179 e9363f82 Antony Chazapis
        for x in ['READ', 'WRITE']:
180 e9363f82 Antony Chazapis
            setattr(self, x, getattr(self.db_module, x))
181 e9363f82 Antony Chazapis
        self.node = self.db_module.Node(**params)
182 33b4e4a6 Antony Chazapis
        for x in ['ROOTNODE', 'SERIAL', 'HASH', 'SIZE', 'TYPE', 'MTIME', 'MUSER', 'UUID', 'CHECKSUM', 'CLUSTER', 'MATCH_PREFIX', 'MATCH_EXACT']:
183 e9363f82 Antony Chazapis
            setattr(self, x, getattr(self.db_module, x))
184 2715ade4 Sofia Papagiannaki
185 46286f5f Antony Chazapis
        self.block_module = load_module(block_module)
186 7f1f0464 Georgios D. Tsoukalas
        self.block_params = block_params
187 7ca7bb08 Antony Chazapis
        params = {'path': block_path,
188 7ca7bb08 Antony Chazapis
                  'block_size': self.block_size,
189 f3b65e8f Antony Chazapis
                  'hash_algorithm': self.hash_algorithm,
190 f3b65e8f Antony Chazapis
                  'umask': block_umask}
191 7f1f0464 Georgios D. Tsoukalas
        params.update(self.block_params)
192 7ca7bb08 Antony Chazapis
        self.store = self.block_module.Store(**params)
193 46286f5f Antony Chazapis
194 f4fbb0fa Sofia Papagiannaki
        if queue_module and queue_hosts:
195 46286f5f Antony Chazapis
            self.queue_module = load_module(queue_module)
196 f4fbb0fa Sofia Papagiannaki
            params = {'hosts': queue_hosts,
197 7f1f0464 Georgios D. Tsoukalas
                      'exchange': queue_exchange,
198 fa9cae7e Antony Chazapis
                      'client_id': QUEUE_CLIENT_ID}
199 46286f5f Antony Chazapis
            self.queue = self.queue_module.Queue(**params)
200 46286f5f Antony Chazapis
        else:
201 46286f5f Antony Chazapis
            class NoQueue:
202 1a239dc1 Sofia Papagiannaki
                def send(self, *args):
203 46286f5f Antony Chazapis
                    pass
204 2715ade4 Sofia Papagiannaki
205 b9a8feec root
                def close(self):
206 b9a8feec root
                    pass
207 2715ade4 Sofia Papagiannaki
208 46286f5f Antony Chazapis
            self.queue = NoQueue()
209 2715ade4 Sofia Papagiannaki
210 7ed99da8 root
        self.quotaholder_url = quotaholder_url
211 3173699c Sofia Papagiannaki
        self.quotaholder_token = quotaholder_token
212 3173699c Sofia Papagiannaki
        self.quotaholder = QuotaholderClient(quotaholder_url, quotaholder_token)
213 7ed99da8 root
        self.serials = []
214 a7f7699d root
        self.messages = []
215 0307b47f Georgios D. Tsoukalas
216 d14fe290 Antony Chazapis
    def close(self):
217 d14fe290 Antony Chazapis
        self.wrapper.close()
218 b9a8feec root
        self.queue.close()
219 2715ade4 Sofia Papagiannaki
220 c846fad1 Sofia Papagiannaki
    @property
221 c846fad1 Sofia Papagiannaki
    def using_external_quotaholder(self):
222 c846fad1 Sofia Papagiannaki
        if self.quotaholder_url:
223 c846fad1 Sofia Papagiannaki
            return True
224 c846fad1 Sofia Papagiannaki
        return False
225 c846fad1 Sofia Papagiannaki
226 a9b3f29d Antony Chazapis
    @backend_method
227 a9b3f29d Antony Chazapis
    def list_accounts(self, user, marker=None, limit=10000):
228 a9b3f29d Antony Chazapis
        """Return a list of accounts the user can access."""
229 2715ade4 Sofia Papagiannaki
230 02c4d2ba Antony Chazapis
        logger.debug("list_accounts: %s %s %s", user, marker, limit)
231 a9b3f29d Antony Chazapis
        allowed = self._allowed_accounts(user)
232 a9b3f29d Antony Chazapis
        start, limit = self._list_limits(allowed, marker, limit)
233 a9b3f29d Antony Chazapis
        return allowed[start:start + limit]
234 2715ade4 Sofia Papagiannaki
235 a9b3f29d Antony Chazapis
    @backend_method
236 c846fad1 Sofia Papagiannaki
    def get_account_meta(
237 c846fad1 Sofia Papagiannaki
            self, user, account, domain, until=None, include_user_defined=True,
238 c846fad1 Sofia Papagiannaki
            external_quota=None):
239 cb69c154 Antony Chazapis
        """Return a dictionary with the account metadata for the domain."""
240 2715ade4 Sofia Papagiannaki
241 2715ade4 Sofia Papagiannaki
        logger.debug(
242 2715ade4 Sofia Papagiannaki
            "get_account_meta: %s %s %s %s", user, account, domain, until)
243 c915d3bf Antony Chazapis
        path, node = self._lookup_account(account, user == account)
244 a9b3f29d Antony Chazapis
        if user != account:
245 44ad5860 Antony Chazapis
            if until or node is None or account not in self._allowed_accounts(user):
246 a9b3f29d Antony Chazapis
                raise NotAllowedError
247 a9b3f29d Antony Chazapis
        try:
248 44ad5860 Antony Chazapis
            props = self._get_properties(node, until)
249 2c5363a0 Antony Chazapis
            mtime = props[self.MTIME]
250 a9b3f29d Antony Chazapis
        except NameError:
251 62f915a1 Antony Chazapis
            props = None
252 a9b3f29d Antony Chazapis
            mtime = until
253 62f915a1 Antony Chazapis
        count, bytes, tstamp = self._get_statistics(node, until)
254 62f915a1 Antony Chazapis
        tstamp = max(tstamp, mtime)
255 a9b3f29d Antony Chazapis
        if until is None:
256 a9b3f29d Antony Chazapis
            modified = tstamp
257 a9b3f29d Antony Chazapis
        else:
258 2715ade4 Sofia Papagiannaki
            modified = self._get_statistics(
259 2715ade4 Sofia Papagiannaki
                node)[2]  # Overall last modification.
260 62f915a1 Antony Chazapis
            modified = max(modified, mtime)
261 2715ade4 Sofia Papagiannaki
262 a9b3f29d Antony Chazapis
        if user != account:
263 a9b3f29d Antony Chazapis
            meta = {'name': account}
264 a9b3f29d Antony Chazapis
        else:
265 44ad5860 Antony Chazapis
            meta = {}
266 82482e2c Antony Chazapis
            if props is not None and include_user_defined:
267 2715ade4 Sofia Papagiannaki
                meta.update(
268 2715ade4 Sofia Papagiannaki
                    dict(self.node.attribute_get(props[self.SERIAL], domain)))
269 a9b3f29d Antony Chazapis
            if until is not None:
270 a9b3f29d Antony Chazapis
                meta.update({'until_timestamp': tstamp})
271 44ad5860 Antony Chazapis
            meta.update({'name': account, 'count': count, 'bytes': bytes})
272 0ed09c7c Sofia Papagiannaki
            if self.using_external_quotaholder:
273 478f763e Sofia Papagiannaki
                external_quota = external_quota or {}
274 0ed09c7c Sofia Papagiannaki
                meta['bytes'] = external_quota.get('currValue', 0)
275 a9b3f29d Antony Chazapis
        meta.update({'modified': modified})
276 a9b3f29d Antony Chazapis
        return meta
277 2715ade4 Sofia Papagiannaki
278 a9b3f29d Antony Chazapis
    @backend_method
279 cb69c154 Antony Chazapis
    def update_account_meta(self, user, account, domain, meta, replace=False):
280 cb69c154 Antony Chazapis
        """Update the metadata associated with the account for the domain."""
281 2715ade4 Sofia Papagiannaki
282 2715ade4 Sofia Papagiannaki
        logger.debug("update_account_meta: %s %s %s %s %s", user,
283 2715ade4 Sofia Papagiannaki
                     account, domain, meta, replace)
284 a9b3f29d Antony Chazapis
        if user != account:
285 a9b3f29d Antony Chazapis
            raise NotAllowedError
286 c915d3bf Antony Chazapis
        path, node = self._lookup_account(account, True)
287 4819d34f Antony Chazapis
        self._put_metadata(user, node, domain, meta, replace)
288 2715ade4 Sofia Papagiannaki
289 a9b3f29d Antony Chazapis
    @backend_method
290 a9b3f29d Antony Chazapis
    def get_account_groups(self, user, account):
291 a9b3f29d Antony Chazapis
        """Return a dictionary with the user groups defined for this account."""
292 2715ade4 Sofia Papagiannaki
293 fe2db49d Sofia Papagiannaki
        logger.debug("get_account_groups: %s %s", user, account)
294 a9b3f29d Antony Chazapis
        if user != account:
295 a9b3f29d Antony Chazapis
            if account not in self._allowed_accounts(user):
296 a9b3f29d Antony Chazapis
                raise NotAllowedError
297 a9b3f29d Antony Chazapis
            return {}
298 44ad5860 Antony Chazapis
        self._lookup_account(account, True)
299 0f9d752c Antony Chazapis
        return self.permissions.group_dict(account)
300 2715ade4 Sofia Papagiannaki
301 a9b3f29d Antony Chazapis
    @backend_method
302 a9b3f29d Antony Chazapis
    def update_account_groups(self, user, account, groups, replace=False):
303 a9b3f29d Antony Chazapis
        """Update the groups associated with the account."""
304 2715ade4 Sofia Papagiannaki
305 2715ade4 Sofia Papagiannaki
        logger.debug("update_account_groups: %s %s %s %s", user,
306 2715ade4 Sofia Papagiannaki
                     account, groups, replace)
307 a9b3f29d Antony Chazapis
        if user != account:
308 a9b3f29d Antony Chazapis
            raise NotAllowedError
309 44ad5860 Antony Chazapis
        self._lookup_account(account, True)
310 a9b3f29d Antony Chazapis
        self._check_groups(groups)
311 0f9d752c Antony Chazapis
        if replace:
312 0f9d752c Antony Chazapis
            self.permissions.group_destroy(account)
313 0f9d752c Antony Chazapis
        for k, v in groups.iteritems():
314 2715ade4 Sofia Papagiannaki
            if not replace:  # If not already deleted.
315 0f9d752c Antony Chazapis
                self.permissions.group_delete(account, k)
316 0f9d752c Antony Chazapis
            if v:
317 0f9d752c Antony Chazapis
                self.permissions.group_addmany(account, k, v)
318 2715ade4 Sofia Papagiannaki
319 a9b3f29d Antony Chazapis
    @backend_method
320 c846fad1 Sofia Papagiannaki
    def get_account_policy(self, user, account, external_quota=None):
321 b2832c6a Antony Chazapis
        """Return a dictionary with the account policy."""
322 2715ade4 Sofia Papagiannaki
323 fe2db49d Sofia Papagiannaki
        logger.debug("get_account_policy: %s %s", user, account)
324 b2832c6a Antony Chazapis
        if user != account:
325 647a5f48 Antony Chazapis
            if account not in self._allowed_accounts(user):
326 647a5f48 Antony Chazapis
                raise NotAllowedError
327 647a5f48 Antony Chazapis
            return {}
328 b2832c6a Antony Chazapis
        path, node = self._lookup_account(account, True)
329 c846fad1 Sofia Papagiannaki
        policy = self._get_policy(node)
330 07fcdb96 Sofia Papagiannaki
        if self.using_external_quotaholder:
331 478f763e Sofia Papagiannaki
            external_quota = external_quota or {}
332 07fcdb96 Sofia Papagiannaki
            policy['quota'] = external_quota.get('maxValue', 0)
333 c846fad1 Sofia Papagiannaki
        return policy
334 2715ade4 Sofia Papagiannaki
335 b2832c6a Antony Chazapis
    @backend_method
336 b2832c6a Antony Chazapis
    def update_account_policy(self, user, account, policy, replace=False):
337 b2832c6a Antony Chazapis
        """Update the policy associated with the account."""
338 2715ade4 Sofia Papagiannaki
339 2715ade4 Sofia Papagiannaki
        logger.debug("update_account_policy: %s %s %s %s", user,
340 2715ade4 Sofia Papagiannaki
                     account, policy, replace)
341 b2832c6a Antony Chazapis
        if user != account:
342 b2832c6a Antony Chazapis
            raise NotAllowedError
343 b2832c6a Antony Chazapis
        path, node = self._lookup_account(account, True)
344 b2832c6a Antony Chazapis
        self._check_policy(policy)
345 b2832c6a Antony Chazapis
        self._put_policy(node, policy, replace)
346 2715ade4 Sofia Papagiannaki
347 b2832c6a Antony Chazapis
    @backend_method
348 8693b873 Sofia Papagiannaki
    def put_account(self, user, account, policy={}):
349 a9b3f29d Antony Chazapis
        """Create a new account with the given name."""
350 2715ade4 Sofia Papagiannaki
351 fe2db49d Sofia Papagiannaki
        logger.debug("put_account: %s %s %s", user, account, policy)
352 a9b3f29d Antony Chazapis
        if user != account:
353 a9b3f29d Antony Chazapis
            raise NotAllowedError
354 44ad5860 Antony Chazapis
        node = self.node.node_lookup(account)
355 44ad5860 Antony Chazapis
        if node is not None:
356 7efc9f86 Sofia Papagiannaki
            raise AccountExists('Account already exists')
357 b2832c6a Antony Chazapis
        if policy:
358 b2832c6a Antony Chazapis
            self._check_policy(policy)
359 b2832c6a Antony Chazapis
        node = self._put_path(user, self.ROOTNODE, account)
360 b2832c6a Antony Chazapis
        self._put_policy(node, policy, True)
361 2715ade4 Sofia Papagiannaki
362 a9b3f29d Antony Chazapis
    @backend_method
363 a9b3f29d Antony Chazapis
    def delete_account(self, user, account):
364 a9b3f29d Antony Chazapis
        """Delete the account with the given name."""
365 2715ade4 Sofia Papagiannaki
366 fe2db49d Sofia Papagiannaki
        logger.debug("delete_account: %s %s", user, account)
367 a9b3f29d Antony Chazapis
        if user != account:
368 a9b3f29d Antony Chazapis
            raise NotAllowedError
369 c915d3bf Antony Chazapis
        node = self.node.node_lookup(account)
370 c915d3bf Antony Chazapis
        if node is None:
371 c915d3bf Antony Chazapis
            return
372 c915d3bf Antony Chazapis
        if not self.node.node_remove(node):
373 7efc9f86 Sofia Papagiannaki
            raise AccountNotEmpty('Account is not empty')
374 0f9d752c Antony Chazapis
        self.permissions.group_destroy(account)
375 2715ade4 Sofia Papagiannaki
376 62f915a1 Antony Chazapis
    @backend_method
377 90ee1eb3 Sofia Papagiannaki
    def list_containers(self, user, account, marker=None, limit=10000, shared=False, until=None, public=False):
378 62f915a1 Antony Chazapis
        """Return a list of containers existing under an account."""
379 2715ade4 Sofia Papagiannaki
380 2715ade4 Sofia Papagiannaki
        logger.debug("list_containers: %s %s %s %s %s %s %s", user,
381 2715ade4 Sofia Papagiannaki
                     account, marker, limit, shared, until, public)
382 62f915a1 Antony Chazapis
        if user != account:
383 62f915a1 Antony Chazapis
            if until or account not in self._allowed_accounts(user):
384 62f915a1 Antony Chazapis
                raise NotAllowedError
385 62f915a1 Antony Chazapis
            allowed = self._allowed_containers(user, account)
386 62f915a1 Antony Chazapis
            start, limit = self._list_limits(allowed, marker, limit)
387 62f915a1 Antony Chazapis
            return allowed[start:start + limit]
388 90ee1eb3 Sofia Papagiannaki
        if shared or public:
389 56ac7c81 Sofia Papagiannaki
            allowed = set()
390 90ee1eb3 Sofia Papagiannaki
            if shared:
391 56ac7c81 Sofia Papagiannaki
                allowed.update([x.split('/', 2)[1] for x in self.permissions.access_list_shared(account)])
392 90ee1eb3 Sofia Papagiannaki
            if public:
393 56ac7c81 Sofia Papagiannaki
                allowed.update([x[0].split('/', 2)[1] for x in self.permissions.public_list(account)])
394 56ac7c81 Sofia Papagiannaki
            allowed = sorted(allowed)
395 62f915a1 Antony Chazapis
            start, limit = self._list_limits(allowed, marker, limit)
396 62f915a1 Antony Chazapis
            return allowed[start:start + limit]
397 62f915a1 Antony Chazapis
        node = self.node.node_lookup(account)
398 2715ade4 Sofia Papagiannaki
        containers = [x[0] for x in self._list_object_properties(
399 2715ade4 Sofia Papagiannaki
            node, account, '', '/', marker, limit, False, None, [], until)]
400 2715ade4 Sofia Papagiannaki
        start, limit = self._list_limits(
401 2715ade4 Sofia Papagiannaki
            [x[0] for x in containers], marker, limit)
402 cf4a7a7b Sofia Papagiannaki
        return containers[start:start + limit]
403 2715ade4 Sofia Papagiannaki
404 371d907a Antony Chazapis
    @backend_method
405 371d907a Antony Chazapis
    def list_container_meta(self, user, account, container, domain, until=None):
406 371d907a Antony Chazapis
        """Return a list with all the container's object meta keys for the domain."""
407 2715ade4 Sofia Papagiannaki
408 2715ade4 Sofia Papagiannaki
        logger.debug("list_container_meta: %s %s %s %s %s", user,
409 2715ade4 Sofia Papagiannaki
                     account, container, domain, until)
410 371d907a Antony Chazapis
        allowed = []
411 371d907a Antony Chazapis
        if user != account:
412 371d907a Antony Chazapis
            if until:
413 371d907a Antony Chazapis
                raise NotAllowedError
414 2715ade4 Sofia Papagiannaki
            allowed = self.permissions.access_list_paths(
415 2715ade4 Sofia Papagiannaki
                user, '/'.join((account, container)))
416 371d907a Antony Chazapis
            if not allowed:
417 371d907a Antony Chazapis
                raise NotAllowedError
418 371d907a Antony Chazapis
        path, node = self._lookup_container(account, container)
419 371d907a Antony Chazapis
        before = until if until is not None else inf
420 371d907a Antony Chazapis
        allowed = self._get_formatted_paths(allowed)
421 371d907a Antony Chazapis
        return self.node.latest_attribute_keys(node, domain, before, CLUSTER_DELETED, allowed)
422 2715ade4 Sofia Papagiannaki
423 a9b3f29d Antony Chazapis
    @backend_method
424 82482e2c Antony Chazapis
    def get_container_meta(self, user, account, container, domain, until=None, include_user_defined=True):
425 cb69c154 Antony Chazapis
        """Return a dictionary with the container metadata for the domain."""
426 2715ade4 Sofia Papagiannaki
427 2715ade4 Sofia Papagiannaki
        logger.debug("get_container_meta: %s %s %s %s %s", user,
428 2715ade4 Sofia Papagiannaki
                     account, container, domain, until)
429 a9b3f29d Antony Chazapis
        if user != account:
430 a9b3f29d Antony Chazapis
            if until or container not in self._allowed_containers(user, account):
431 a9b3f29d Antony Chazapis
                raise NotAllowedError
432 c915d3bf Antony Chazapis
        path, node = self._lookup_container(account, container)
433 c915d3bf Antony Chazapis
        props = self._get_properties(node, until)
434 2c5363a0 Antony Chazapis
        mtime = props[self.MTIME]
435 62f915a1 Antony Chazapis
        count, bytes, tstamp = self._get_statistics(node, until)
436 62f915a1 Antony Chazapis
        tstamp = max(tstamp, mtime)
437 a9b3f29d Antony Chazapis
        if until is None:
438 a9b3f29d Antony Chazapis
            modified = tstamp
439 a9b3f29d Antony Chazapis
        else:
440 2715ade4 Sofia Papagiannaki
            modified = self._get_statistics(
441 2715ade4 Sofia Papagiannaki
                node)[2]  # Overall last modification.
442 62f915a1 Antony Chazapis
            modified = max(modified, mtime)
443 2715ade4 Sofia Papagiannaki
444 a9b3f29d Antony Chazapis
        if user != account:
445 c915d3bf Antony Chazapis
            meta = {'name': container}
446 a9b3f29d Antony Chazapis
        else:
447 82482e2c Antony Chazapis
            meta = {}
448 82482e2c Antony Chazapis
            if include_user_defined:
449 2715ade4 Sofia Papagiannaki
                meta.update(
450 2715ade4 Sofia Papagiannaki
                    dict(self.node.attribute_get(props[self.SERIAL], domain)))
451 a9b3f29d Antony Chazapis
            if until is not None:
452 a9b3f29d Antony Chazapis
                meta.update({'until_timestamp': tstamp})
453 c915d3bf Antony Chazapis
            meta.update({'name': container, 'count': count, 'bytes': bytes})
454 c915d3bf Antony Chazapis
        meta.update({'modified': modified})
455 a9b3f29d Antony Chazapis
        return meta
456 2715ade4 Sofia Papagiannaki
457 a9b3f29d Antony Chazapis
    @backend_method
458 cb69c154 Antony Chazapis
    def update_container_meta(self, user, account, container, domain, meta, replace=False):
459 cb69c154 Antony Chazapis
        """Update the metadata associated with the container for the domain."""
460 2715ade4 Sofia Papagiannaki
461 2715ade4 Sofia Papagiannaki
        logger.debug("update_container_meta: %s %s %s %s %s %s",
462 2715ade4 Sofia Papagiannaki
                     user, account, container, domain, meta, replace)
463 a9b3f29d Antony Chazapis
        if user != account:
464 a9b3f29d Antony Chazapis
            raise NotAllowedError
465 c915d3bf Antony Chazapis
        path, node = self._lookup_container(account, container)
466 2715ade4 Sofia Papagiannaki
        src_version_id, dest_version_id = self._put_metadata(
467 2715ade4 Sofia Papagiannaki
            user, node, domain, meta, replace)
468 f9ea264b Antony Chazapis
        if src_version_id is not None:
469 f9ea264b Antony Chazapis
            versioning = self._get_policy(node)['versioning']
470 f9ea264b Antony Chazapis
            if versioning != 'auto':
471 f9ea264b Antony Chazapis
                self.node.version_remove(src_version_id)
472 2715ade4 Sofia Papagiannaki
473 a9b3f29d Antony Chazapis
    @backend_method
474 a9b3f29d Antony Chazapis
    def get_container_policy(self, user, account, container):
475 a9b3f29d Antony Chazapis
        """Return a dictionary with the container policy."""
476 2715ade4 Sofia Papagiannaki
477 2715ade4 Sofia Papagiannaki
        logger.debug(
478 2715ade4 Sofia Papagiannaki
            "get_container_policy: %s %s %s", user, account, container)
479 a9b3f29d Antony Chazapis
        if user != account:
480 a9b3f29d Antony Chazapis
            if container not in self._allowed_containers(user, account):
481 a9b3f29d Antony Chazapis
                raise NotAllowedError
482 a9b3f29d Antony Chazapis
            return {}
483 5e7485da Antony Chazapis
        path, node = self._lookup_container(account, container)
484 b9064632 Antony Chazapis
        return self._get_policy(node)
485 2715ade4 Sofia Papagiannaki
486 a9b3f29d Antony Chazapis
    @backend_method
487 a9b3f29d Antony Chazapis
    def update_container_policy(self, user, account, container, policy, replace=False):
488 b2832c6a Antony Chazapis
        """Update the policy associated with the container."""
489 2715ade4 Sofia Papagiannaki
490 2715ade4 Sofia Papagiannaki
        logger.debug("update_container_policy: %s %s %s %s %s",
491 2715ade4 Sofia Papagiannaki
                     user, account, container, policy, replace)
492 a9b3f29d Antony Chazapis
        if user != account:
493 a9b3f29d Antony Chazapis
            raise NotAllowedError
494 5e7485da Antony Chazapis
        path, node = self._lookup_container(account, container)
495 a9b3f29d Antony Chazapis
        self._check_policy(policy)
496 b2832c6a Antony Chazapis
        self._put_policy(node, policy, replace)
497 2715ade4 Sofia Papagiannaki
498 a9b3f29d Antony Chazapis
    @backend_method
499 8693b873 Sofia Papagiannaki
    def put_container(self, user, account, container, policy={}):
500 a9b3f29d Antony Chazapis
        """Create a new container with the given name."""
501 2715ade4 Sofia Papagiannaki
502 2715ade4 Sofia Papagiannaki
        logger.debug(
503 2715ade4 Sofia Papagiannaki
            "put_container: %s %s %s %s", user, account, container, policy)
504 a9b3f29d Antony Chazapis
        if user != account:
505 a9b3f29d Antony Chazapis
            raise NotAllowedError
506 a9b3f29d Antony Chazapis
        try:
507 c915d3bf Antony Chazapis
            path, node = self._lookup_container(account, container)
508 a9b3f29d Antony Chazapis
        except NameError:
509 a9b3f29d Antony Chazapis
            pass
510 a9b3f29d Antony Chazapis
        else:
511 7efc9f86 Sofia Papagiannaki
            raise ContainerExists('Container already exists')
512 a9b3f29d Antony Chazapis
        if policy:
513 a9b3f29d Antony Chazapis
            self._check_policy(policy)
514 a9b3f29d Antony Chazapis
        path = '/'.join((account, container))
515 2715ade4 Sofia Papagiannaki
        node = self._put_path(
516 2715ade4 Sofia Papagiannaki
            user, self._lookup_account(account, True)[1], path)
517 b2832c6a Antony Chazapis
        self._put_policy(node, policy, True)
518 2715ade4 Sofia Papagiannaki
519 a9b3f29d Antony Chazapis
    @backend_method
520 fe2db49d Sofia Papagiannaki
    def delete_container(self, user, account, container, until=None, prefix='', delimiter=None):
521 a9b3f29d Antony Chazapis
        """Delete/purge the container with the given name."""
522 2715ade4 Sofia Papagiannaki
523 2715ade4 Sofia Papagiannaki
        logger.debug("delete_container: %s %s %s %s %s %s", user,
524 2715ade4 Sofia Papagiannaki
                     account, container, until, prefix, delimiter)
525 a9b3f29d Antony Chazapis
        if user != account:
526 a9b3f29d Antony Chazapis
            raise NotAllowedError
527 c915d3bf Antony Chazapis
        path, node = self._lookup_container(account, container)
528 2715ade4 Sofia Papagiannaki
529 a9b3f29d Antony Chazapis
        if until is not None:
530 388ea25f Sofia Papagiannaki
            hashes, size, serials = self.node.node_purge_children(
531 2715ade4 Sofia Papagiannaki
                node, until, CLUSTER_HISTORY)
532 04230536 Antony Chazapis
            for h in hashes:
533 04230536 Antony Chazapis
                self.store.map_delete(h)
534 c915d3bf Antony Chazapis
            self.node.node_purge_children(node, until, CLUSTER_DELETED)
535 388ea25f Sofia Papagiannaki
            self._report_size_change(user, account, -size,
536 7ed99da8 root
                                     {'action':'container purge', 'path': path,
537 b1dadd0e Sofia Papagiannaki
                                      'versions': ','.join(str(i) for i in serials)})
538 a9b3f29d Antony Chazapis
            return
539 2715ade4 Sofia Papagiannaki
540 e46b2bcf Sofia Papagiannaki
        if not delimiter:
541 e46b2bcf Sofia Papagiannaki
            if self._get_statistics(node)[0] > 0:
542 e46b2bcf Sofia Papagiannaki
                raise ContainerNotEmpty('Container is not empty')
543 388ea25f Sofia Papagiannaki
            hashes, size, serials = self.node.node_purge_children(
544 2715ade4 Sofia Papagiannaki
                node, inf, CLUSTER_HISTORY)
545 e46b2bcf Sofia Papagiannaki
            for h in hashes:
546 e46b2bcf Sofia Papagiannaki
                self.store.map_delete(h)
547 e46b2bcf Sofia Papagiannaki
            self.node.node_purge_children(node, inf, CLUSTER_DELETED)
548 e46b2bcf Sofia Papagiannaki
            self.node.node_remove(node)
549 388ea25f Sofia Papagiannaki
            self._report_size_change(user, account, -size,
550 7ed99da8 root
                                     {'action': 'container delete',
551 7ed99da8 root
                                      'path': path,
552 b1dadd0e Sofia Papagiannaki
                                      'versions': ','.join(str(i) for i in serials)})
553 e46b2bcf Sofia Papagiannaki
        else:
554 b1dadd0e Sofia Papagiannaki
            # remove only contents
555 e46b2bcf Sofia Papagiannaki
            src_names = self._list_objects_no_limit(user, account, container, prefix='', delimiter=None, virtual=False, domain=None, keys=[], shared=False, until=None, size_range=None, all_props=True, public=False)
556 e46b2bcf Sofia Papagiannaki
            paths = []
557 e46b2bcf Sofia Papagiannaki
            for t in src_names:
558 e46b2bcf Sofia Papagiannaki
                path = '/'.join((account, container, t[0]))
559 e46b2bcf Sofia Papagiannaki
                node = t[2]
560 e46b2bcf Sofia Papagiannaki
                src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, type='', hash=None, checksum='', cluster=CLUSTER_DELETED)
561 2715ade4 Sofia Papagiannaki
                del_size = self._apply_versioning(
562 2715ade4 Sofia Papagiannaki
                    account, container, src_version_id)
563 b1dadd0e Sofia Papagiannaki
                self._report_size_change(
564 7f1f0464 Georgios D. Tsoukalas
                        user, account, -del_size, {
565 7f1f0464 Georgios D. Tsoukalas
                                'action': 'object delete',
566 7f1f0464 Georgios D. Tsoukalas
                                'path': path,
567 7f1f0464 Georgios D. Tsoukalas
                        'versions': ','.join([str(dest_version_id)])
568 b1dadd0e Sofia Papagiannaki
                     }
569 b1dadd0e Sofia Papagiannaki
                )
570 2715ade4 Sofia Papagiannaki
                self._report_object_change(
571 2715ade4 Sofia Papagiannaki
                    user, account, path, details={'action': 'object delete'})
572 e46b2bcf Sofia Papagiannaki
                paths.append(path)
573 e46b2bcf Sofia Papagiannaki
            self.permissions.access_clear_bulk(paths)
574 2715ade4 Sofia Papagiannaki
575 90ee1eb3 Sofia Papagiannaki
    def _list_objects(self, user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, all_props, public):
576 15a96c3e Antony Chazapis
        if user != account and until:
577 15a96c3e Antony Chazapis
            raise NotAllowedError
578 cf4a7a7b Sofia Papagiannaki
        if shared and public:
579 cf4a7a7b Sofia Papagiannaki
            # get shared first
580 2715ade4 Sofia Papagiannaki
            shared = self._list_object_permissions(
581 2715ade4 Sofia Papagiannaki
                user, account, container, prefix, shared=True, public=False)
582 31e1acd3 Sofia Papagiannaki
            objects = set()
583 cf4a7a7b Sofia Papagiannaki
            if shared:
584 cf4a7a7b Sofia Papagiannaki
                path, node = self._lookup_container(account, container)
585 cf4a7a7b Sofia Papagiannaki
                shared = self._get_formatted_paths(shared)
586 31e1acd3 Sofia Papagiannaki
                objects |= set(self._list_object_properties(node, path, prefix, delimiter, marker, limit, virtual, domain, keys, until, size_range, shared, all_props))
587 2715ade4 Sofia Papagiannaki
588 cf4a7a7b Sofia Papagiannaki
            # get public
589 2715ade4 Sofia Papagiannaki
            objects |= set(self._list_public_object_properties(
590 2715ade4 Sofia Papagiannaki
                user, account, container, prefix, all_props))
591 31e1acd3 Sofia Papagiannaki
            objects = list(objects)
592 2715ade4 Sofia Papagiannaki
593 cf4a7a7b Sofia Papagiannaki
            objects.sort(key=lambda x: x[0])
594 2715ade4 Sofia Papagiannaki
            start, limit = self._list_limits(
595 2715ade4 Sofia Papagiannaki
                [x[0] for x in objects], marker, limit)
596 cf4a7a7b Sofia Papagiannaki
            return objects[start:start + limit]
597 cf4a7a7b Sofia Papagiannaki
        elif public:
598 2715ade4 Sofia Papagiannaki
            objects = self._list_public_object_properties(
599 2715ade4 Sofia Papagiannaki
                user, account, container, prefix, all_props)
600 2715ade4 Sofia Papagiannaki
            start, limit = self._list_limits(
601 2715ade4 Sofia Papagiannaki
                [x[0] for x in objects], marker, limit)
602 cf4a7a7b Sofia Papagiannaki
            return objects[start:start + limit]
603 2715ade4 Sofia Papagiannaki
604 2715ade4 Sofia Papagiannaki
        allowed = self._list_object_permissions(
605 2715ade4 Sofia Papagiannaki
            user, account, container, prefix, shared, public)
606 cf4a7a7b Sofia Papagiannaki
        if shared and not allowed:
607 fcd37c40 Antony Chazapis
            return []
608 15a96c3e Antony Chazapis
        path, node = self._lookup_container(account, container)
609 15a96c3e Antony Chazapis
        allowed = self._get_formatted_paths(allowed)
610 cf4a7a7b Sofia Papagiannaki
        objects = self._list_object_properties(node, path, prefix, delimiter, marker, limit, virtual, domain, keys, until, size_range, allowed, all_props)
611 2715ade4 Sofia Papagiannaki
        start, limit = self._list_limits(
612 2715ade4 Sofia Papagiannaki
            [x[0] for x in objects], marker, limit)
613 cf4a7a7b Sofia Papagiannaki
        return objects[start:start + limit]
614 2715ade4 Sofia Papagiannaki
615 cf4a7a7b Sofia Papagiannaki
    def _list_public_object_properties(self, user, account, container, prefix, all_props):
616 2715ade4 Sofia Papagiannaki
        public = self._list_object_permissions(
617 2715ade4 Sofia Papagiannaki
            user, account, container, prefix, shared=False, public=True)
618 cf4a7a7b Sofia Papagiannaki
        paths, nodes = self._lookup_objects(public)
619 cf4a7a7b Sofia Papagiannaki
        path = '/'.join((account, container))
620 cf4a7a7b Sofia Papagiannaki
        cont_prefix = path + '/'
621 cf4a7a7b Sofia Papagiannaki
        paths = [x[len(cont_prefix):] for x in paths]
622 cf4a7a7b Sofia Papagiannaki
        props = self.node.version_lookup_bulk(nodes, all_props=all_props)
623 cf4a7a7b Sofia Papagiannaki
        objects = [(path,) + props for path, props in zip(paths, props)]
624 cf4a7a7b Sofia Papagiannaki
        return objects
625 2715ade4 Sofia Papagiannaki
626 4d15c94e Sofia Papagiannaki
    def _list_objects_no_limit(self, user, account, container, prefix, delimiter, virtual, domain, keys, shared, until, size_range, all_props, public):
627 4d15c94e Sofia Papagiannaki
        objects = []
628 4d15c94e Sofia Papagiannaki
        while True:
629 4d15c94e Sofia Papagiannaki
            marker = objects[-1] if objects else None
630 4d15c94e Sofia Papagiannaki
            limit = 10000
631 4d15c94e Sofia Papagiannaki
            l = self._list_objects(user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, all_props, public)
632 4d15c94e Sofia Papagiannaki
            objects.extend(l)
633 4d15c94e Sofia Papagiannaki
            if not l or len(l) < limit:
634 4d15c94e Sofia Papagiannaki
                break
635 4d15c94e Sofia Papagiannaki
        return objects
636 2715ade4 Sofia Papagiannaki
637 90ee1eb3 Sofia Papagiannaki
    def _list_object_permissions(self, user, account, container, prefix, shared, public):
638 62f915a1 Antony Chazapis
        allowed = []
639 fcd37c40 Antony Chazapis
        path = '/'.join((account, container, prefix)).rstrip('/')
640 62f915a1 Antony Chazapis
        if user != account:
641 fcd37c40 Antony Chazapis
            allowed = self.permissions.access_list_paths(user, path)
642 62f915a1 Antony Chazapis
            if not allowed:
643 62f915a1 Antony Chazapis
                raise NotAllowedError
644 62f915a1 Antony Chazapis
        else:
645 56ac7c81 Sofia Papagiannaki
            allowed = set()
646 62f915a1 Antony Chazapis
            if shared:
647 56ac7c81 Sofia Papagiannaki
                allowed.update(self.permissions.access_list_shared(path))
648 c53c4def Sofia Papagiannaki
            if public:
649 2715ade4 Sofia Papagiannaki
                allowed.update(
650 2715ade4 Sofia Papagiannaki
                    [x[0] for x in self.permissions.public_list(path)])
651 56ac7c81 Sofia Papagiannaki
            allowed = sorted(allowed)
652 c53c4def Sofia Papagiannaki
            if not allowed:
653 c53c4def Sofia Papagiannaki
                return []
654 15a96c3e Antony Chazapis
        return allowed
655 2715ade4 Sofia Papagiannaki
656 62f915a1 Antony Chazapis
    @backend_method
657 90ee1eb3 Sofia Papagiannaki
    def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None, public=False):
658 371d907a Antony Chazapis
        """Return a list of object (name, version_id) tuples existing under a container."""
659 2715ade4 Sofia Papagiannaki
660 fe2db49d Sofia Papagiannaki
        logger.debug("list_objects: %s %s %s %s %s %s %s %s %s %s %s %s %s %s", user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, public)
661 90ee1eb3 Sofia Papagiannaki
        return self._list_objects(user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, False, public)
662 2715ade4 Sofia Papagiannaki
663 371d907a Antony Chazapis
    @backend_method
664 90ee1eb3 Sofia Papagiannaki
    def list_object_meta(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None, public=False):
665 371d907a Antony Chazapis
        """Return a list of object metadata dicts existing under a container."""
666 2715ade4 Sofia Papagiannaki
667 fe2db49d Sofia Papagiannaki
        logger.debug("list_object_meta: %s %s %s %s %s %s %s %s %s %s %s %s %s %s", user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, public)
668 90ee1eb3 Sofia Papagiannaki
        props = self._list_objects(user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, True, public)
669 371d907a Antony Chazapis
        objects = []
670 371d907a Antony Chazapis
        for p in props:
671 371d907a Antony Chazapis
            if len(p) == 2:
672 371d907a Antony Chazapis
                objects.append({'subdir': p[0]})
673 371d907a Antony Chazapis
            else:
674 371d907a Antony Chazapis
                objects.append({'name': p[0],
675 371d907a Antony Chazapis
                                'bytes': p[self.SIZE + 1],
676 371d907a Antony Chazapis
                                'type': p[self.TYPE + 1],
677 371d907a Antony Chazapis
                                'hash': p[self.HASH + 1],
678 371d907a Antony Chazapis
                                'version': p[self.SERIAL + 1],
679 371d907a Antony Chazapis
                                'version_timestamp': p[self.MTIME + 1],
680 371d907a Antony Chazapis
                                'modified': p[self.MTIME + 1] if until is None else None,
681 371d907a Antony Chazapis
                                'modified_by': p[self.MUSER + 1],
682 371d907a Antony Chazapis
                                'uuid': p[self.UUID + 1],
683 371d907a Antony Chazapis
                                'checksum': p[self.CHECKSUM + 1]})
684 371d907a Antony Chazapis
        return objects
685 2715ade4 Sofia Papagiannaki
686 a9b3f29d Antony Chazapis
    @backend_method
687 15a96c3e Antony Chazapis
    def list_object_permissions(self, user, account, container, prefix=''):
688 15a96c3e Antony Chazapis
        """Return a list of paths that enforce permissions under a container."""
689 2715ade4 Sofia Papagiannaki
690 2715ade4 Sofia Papagiannaki
        logger.debug("list_object_permissions: %s %s %s %s", user,
691 2715ade4 Sofia Papagiannaki
                     account, container, prefix)
692 90ee1eb3 Sofia Papagiannaki
        return self._list_object_permissions(user, account, container, prefix, True, False)
693 2715ade4 Sofia Papagiannaki
694 15a96c3e Antony Chazapis
    @backend_method
695 15a96c3e Antony Chazapis
    def list_object_public(self, user, account, container, prefix=''):
696 15a96c3e Antony Chazapis
        """Return a dict mapping paths to public ids for objects that are public under a container."""
697 2715ade4 Sofia Papagiannaki
698 2715ade4 Sofia Papagiannaki
        logger.debug("list_object_public: %s %s %s %s", user,
699 2715ade4 Sofia Papagiannaki
                     account, container, prefix)
700 15a96c3e Antony Chazapis
        public = {}
701 15a96c3e Antony Chazapis
        for path, p in self.permissions.public_list('/'.join((account, container, prefix))):
702 15a96c3e Antony Chazapis
            public[path] = p + ULTIMATE_ANSWER
703 15a96c3e Antony Chazapis
        return public
704 2715ade4 Sofia Papagiannaki
705 15a96c3e Antony Chazapis
    @backend_method
706 82482e2c Antony Chazapis
    def get_object_meta(self, user, account, container, name, domain, version=None, include_user_defined=True):
707 cb69c154 Antony Chazapis
        """Return a dictionary with the object metadata for the domain."""
708 2715ade4 Sofia Papagiannaki
709 2715ade4 Sofia Papagiannaki
        logger.debug("get_object_meta: %s %s %s %s %s %s", user,
710 2715ade4 Sofia Papagiannaki
                     account, container, name, domain, version)
711 a9b3f29d Antony Chazapis
        self._can_read(user, account, container, name)
712 c915d3bf Antony Chazapis
        path, node = self._lookup_object(account, container, name)
713 c915d3bf Antony Chazapis
        props = self._get_version(node, version)
714 a9b3f29d Antony Chazapis
        if version is None:
715 2c5363a0 Antony Chazapis
            modified = props[self.MTIME]
716 a9b3f29d Antony Chazapis
        else:
717 97d45f69 Antony Chazapis
            try:
718 2715ade4 Sofia Papagiannaki
                modified = self._get_version(
719 2715ade4 Sofia Papagiannaki
                    node)[self.MTIME]  # Overall last modification.
720 2715ade4 Sofia Papagiannaki
            except NameError:  # Object may be deleted.
721 2715ade4 Sofia Papagiannaki
                del_props = self.node.version_lookup(
722 2715ade4 Sofia Papagiannaki
                    node, inf, CLUSTER_DELETED)
723 97d45f69 Antony Chazapis
                if del_props is None:
724 7efc9f86 Sofia Papagiannaki
                    raise ItemNotExists('Object does not exist')
725 97d45f69 Antony Chazapis
                modified = del_props[self.MTIME]
726 2715ade4 Sofia Papagiannaki
727 82482e2c Antony Chazapis
        meta = {}
728 82482e2c Antony Chazapis
        if include_user_defined:
729 2715ade4 Sofia Papagiannaki
            meta.update(
730 2715ade4 Sofia Papagiannaki
                dict(self.node.attribute_get(props[self.SERIAL], domain)))
731 33b4e4a6 Antony Chazapis
        meta.update({'name': name,
732 33b4e4a6 Antony Chazapis
                     'bytes': props[self.SIZE],
733 33b4e4a6 Antony Chazapis
                     'type': props[self.TYPE],
734 371d907a Antony Chazapis
                     'hash': props[self.HASH],
735 33b4e4a6 Antony Chazapis
                     'version': props[self.SERIAL],
736 33b4e4a6 Antony Chazapis
                     'version_timestamp': props[self.MTIME],
737 33b4e4a6 Antony Chazapis
                     'modified': modified,
738 33b4e4a6 Antony Chazapis
                     'modified_by': props[self.MUSER],
739 33b4e4a6 Antony Chazapis
                     'uuid': props[self.UUID],
740 33b4e4a6 Antony Chazapis
                     'checksum': props[self.CHECKSUM]})
741 a9b3f29d Antony Chazapis
        return meta
742 2715ade4 Sofia Papagiannaki
743 a9b3f29d Antony Chazapis
    @backend_method
744 cb69c154 Antony Chazapis
    def update_object_meta(self, user, account, container, name, domain, meta, replace=False):
745 cb69c154 Antony Chazapis
        """Update the metadata associated with the object for the domain and return the new version."""
746 2715ade4 Sofia Papagiannaki
747 2715ade4 Sofia Papagiannaki
        logger.debug("update_object_meta: %s %s %s %s %s %s %s",
748 2715ade4 Sofia Papagiannaki
                     user, account, container, name, domain, meta, replace)
749 a9b3f29d Antony Chazapis
        self._can_write(user, account, container, name)
750 c915d3bf Antony Chazapis
        path, node = self._lookup_object(account, container, name)
751 2715ade4 Sofia Papagiannaki
        src_version_id, dest_version_id = self._put_metadata(
752 2715ade4 Sofia Papagiannaki
            user, node, domain, meta, replace)
753 5cc484e1 Antony Chazapis
        self._apply_versioning(account, container, src_version_id)
754 5cc484e1 Antony Chazapis
        return dest_version_id
755 2715ade4 Sofia Papagiannaki
756 a9b3f29d Antony Chazapis
    @backend_method
757 a9b3f29d Antony Chazapis
    def get_object_permissions(self, user, account, container, name):
758 067cf1fc Antony Chazapis
        """Return the action allowed on the object, the path
759 067cf1fc Antony Chazapis
        from which the object gets its permissions from,
760 a9b3f29d Antony Chazapis
        along with a dictionary containing the permissions."""
761 2715ade4 Sofia Papagiannaki
762 2715ade4 Sofia Papagiannaki
        logger.debug("get_object_permissions: %s %s %s %s", user,
763 2715ade4 Sofia Papagiannaki
                     account, container, name)
764 067cf1fc Antony Chazapis
        allowed = 'write'
765 92da0e5a Antony Chazapis
        permissions_path = self._get_permissions_path(account, container, name)
766 067cf1fc Antony Chazapis
        if user != account:
767 92da0e5a Antony Chazapis
            if self.permissions.access_check(permissions_path, self.WRITE, user):
768 067cf1fc Antony Chazapis
                allowed = 'write'
769 92da0e5a Antony Chazapis
            elif self.permissions.access_check(permissions_path, self.READ, user):
770 067cf1fc Antony Chazapis
                allowed = 'read'
771 067cf1fc Antony Chazapis
            else:
772 067cf1fc Antony Chazapis
                raise NotAllowedError
773 92da0e5a Antony Chazapis
        self._lookup_object(account, container, name)
774 92da0e5a Antony Chazapis
        return (allowed, permissions_path, self.permissions.access_get(permissions_path))
775 2715ade4 Sofia Papagiannaki
776 a9b3f29d Antony Chazapis
    @backend_method
777 a9b3f29d Antony Chazapis
    def update_object_permissions(self, user, account, container, name, permissions):
778 a9b3f29d Antony Chazapis
        """Update the permissions associated with the object."""
779 2715ade4 Sofia Papagiannaki
780 2715ade4 Sofia Papagiannaki
        logger.debug("update_object_permissions: %s %s %s %s %s",
781 2715ade4 Sofia Papagiannaki
                     user, account, container, name, permissions)
782 a9b3f29d Antony Chazapis
        if user != account:
783 a9b3f29d Antony Chazapis
            raise NotAllowedError
784 c915d3bf Antony Chazapis
        path = self._lookup_object(account, container, name)[0]
785 6f4bce7b Antony Chazapis
        self._check_permissions(path, permissions)
786 0f9d752c Antony Chazapis
        self.permissions.access_set(path, permissions)
787 2715ade4 Sofia Papagiannaki
        self._report_sharing_change(user, account, path, {'members':
788 2715ade4 Sofia Papagiannaki
                                    self.permissions.access_members(path)})
789 2715ade4 Sofia Papagiannaki
790 a9b3f29d Antony Chazapis
    @backend_method
791 a9b3f29d Antony Chazapis
    def get_object_public(self, user, account, container, name):
792 bb4eafc6 Antony Chazapis
        """Return the public id of the object if applicable."""
793 2715ade4 Sofia Papagiannaki
794 2715ade4 Sofia Papagiannaki
        logger.debug(
795 2715ade4 Sofia Papagiannaki
            "get_object_public: %s %s %s %s", user, account, container, name)
796 a9b3f29d Antony Chazapis
        self._can_read(user, account, container, name)
797 c915d3bf Antony Chazapis
        path = self._lookup_object(account, container, name)[0]
798 bb4eafc6 Antony Chazapis
        p = self.permissions.public_get(path)
799 bb4eafc6 Antony Chazapis
        if p is not None:
800 bb4eafc6 Antony Chazapis
            p += ULTIMATE_ANSWER
801 bb4eafc6 Antony Chazapis
        return p
802 2715ade4 Sofia Papagiannaki
803 a9b3f29d Antony Chazapis
    @backend_method
804 a9b3f29d Antony Chazapis
    def update_object_public(self, user, account, container, name, public):
805 a9b3f29d Antony Chazapis
        """Update the public status of the object."""
806 2715ade4 Sofia Papagiannaki
807 2715ade4 Sofia Papagiannaki
        logger.debug("update_object_public: %s %s %s %s %s", user,
808 2715ade4 Sofia Papagiannaki
                     account, container, name, public)
809 a9b3f29d Antony Chazapis
        self._can_write(user, account, container, name)
810 c915d3bf Antony Chazapis
        path = self._lookup_object(account, container, name)[0]
811 0f9d752c Antony Chazapis
        if not public:
812 0f9d752c Antony Chazapis
            self.permissions.public_unset(path)
813 0f9d752c Antony Chazapis
        else:
814 0f9d752c Antony Chazapis
            self.permissions.public_set(path)
815 2715ade4 Sofia Papagiannaki
816 a9b3f29d Antony Chazapis
    @backend_method
817 a9b3f29d Antony Chazapis
    def get_object_hashmap(self, user, account, container, name, version=None):
818 a9b3f29d Antony Chazapis
        """Return the object's size and a list with partial hashes."""
819 2715ade4 Sofia Papagiannaki
820 2715ade4 Sofia Papagiannaki
        logger.debug("get_object_hashmap: %s %s %s %s %s", user,
821 2715ade4 Sofia Papagiannaki
                     account, container, name, version)
822 a9b3f29d Antony Chazapis
        self._can_read(user, account, container, name)
823 c915d3bf Antony Chazapis
        path, node = self._lookup_object(account, container, name)
824 c915d3bf Antony Chazapis
        props = self._get_version(node, version)
825 7ca7bb08 Antony Chazapis
        hashmap = self.store.map_get(binascii.unhexlify(props[self.HASH]))
826 2c5363a0 Antony Chazapis
        return props[self.SIZE], [binascii.hexlify(x) for x in hashmap]
827 2715ade4 Sofia Papagiannaki
828 f9ea264b Antony Chazapis
    def _update_object_hash(self, user, account, container, name, size, type, hash, checksum, domain, meta, replace_meta, permissions, src_node=None, src_version_id=None, is_copy=False):
829 b9064632 Antony Chazapis
        if permissions is not None and user != account:
830 b9064632 Antony Chazapis
            raise NotAllowedError
831 b9064632 Antony Chazapis
        self._can_write(user, account, container, name)
832 b9064632 Antony Chazapis
        if permissions is not None:
833 b9064632 Antony Chazapis
            path = '/'.join((account, container, name))
834 b9064632 Antony Chazapis
            self._check_permissions(path, permissions)
835 2715ade4 Sofia Papagiannaki
836 b9064632 Antony Chazapis
        account_path, account_node = self._lookup_account(account, True)
837 2715ade4 Sofia Papagiannaki
        container_path, container_node = self._lookup_container(
838 2715ade4 Sofia Papagiannaki
            account, container)
839 2715ade4 Sofia Papagiannaki
        path, node = self._put_object_node(
840 2715ade4 Sofia Papagiannaki
            container_path, container_node, name)
841 33b4e4a6 Antony Chazapis
        pre_version_id, dest_version_id = self._put_version_duplicate(user, node, src_node=src_node, size=size, type=type, hash=hash, checksum=checksum, is_copy=is_copy)
842 2715ade4 Sofia Papagiannaki
843 f9ea264b Antony Chazapis
        # Handle meta.
844 f9ea264b Antony Chazapis
        if src_version_id is None:
845 f9ea264b Antony Chazapis
            src_version_id = pre_version_id
846 2715ade4 Sofia Papagiannaki
        self._put_metadata_duplicate(
847 2715ade4 Sofia Papagiannaki
            src_version_id, dest_version_id, domain, meta, replace_meta)
848 2715ade4 Sofia Papagiannaki
849 813e42e5 Antony Chazapis
        del_size = self._apply_versioning(account, container, pre_version_id)
850 813e42e5 Antony Chazapis
        size_delta = size - del_size
851 e20a751d Sofia Papagiannaki
        if not self.using_external_quotaholder: # Check account quota.
852 d3655326 Sofia Papagiannaki
            if size_delta > 0:
853 d3655326 Sofia Papagiannaki
                account_quota = long(self._get_policy(account_node)['quota'])
854 d3655326 Sofia Papagiannaki
                account_usage = self._get_statistics(account_node)[1] + size_delta
855 d3655326 Sofia Papagiannaki
                if (account_quota > 0 and account_usage > account_quota):
856 ccfd4e44 Sofia Papagiannaki
                    raise QuotaError('account quota exceeded: limit: %s, usage: %s' % (
857 d3655326 Sofia Papagiannaki
                        account_quota, account_usage
858 d3655326 Sofia Papagiannaki
                    ))
859 e20a751d Sofia Papagiannaki
860 e20a751d Sofia Papagiannaki
        # Check container quota.
861 e20a751d Sofia Papagiannaki
        container_quota = long(self._get_policy(container_node)['quota'])
862 e20a751d Sofia Papagiannaki
        container_usage = self._get_statistics(container_node)[1] + size_delta
863 e20a751d Sofia Papagiannaki
        if (container_quota > 0 and container_usage > container_quota):
864 e20a751d Sofia Papagiannaki
            # This must be executed in a transaction, so the version is
865 e20a751d Sofia Papagiannaki
            # never created if it fails.
866 ccfd4e44 Sofia Papagiannaki
            raise QuotaError('container quota exceeded: limit: %s, usage: %s' % (
867 e20a751d Sofia Papagiannaki
                container_quota, container_usage
868 e20a751d Sofia Papagiannaki
            ))
869 e20a751d Sofia Papagiannaki
870 388ea25f Sofia Papagiannaki
        self._report_size_change(user, account, size_delta,
871 7ed99da8 root
                                 {'action': 'object update', 'path': path,
872 b1dadd0e Sofia Papagiannaki
                                  'versions': ','.join([str(dest_version_id)])})
873 b9064632 Antony Chazapis
        if permissions is not None:
874 b9064632 Antony Chazapis
            self.permissions.access_set(path, permissions)
875 2715ade4 Sofia Papagiannaki
            self._report_sharing_change(user, account, path, {'members': self.permissions.access_members(path)})
876 2715ade4 Sofia Papagiannaki
877 39ef6f41 Antony Chazapis
        self._report_object_change(user, account, path, details={'version': dest_version_id, 'action': 'object update'})
878 f9ea264b Antony Chazapis
        return dest_version_id
879 2715ade4 Sofia Papagiannaki
880 a9b3f29d Antony Chazapis
    @backend_method
881 33b4e4a6 Antony Chazapis
    def update_object_hashmap(self, user, account, container, name, size, type, hashmap, checksum, domain, meta={}, replace_meta=False, permissions=None):
882 a9b3f29d Antony Chazapis
        """Create/update an object with the specified size and partial hashes."""
883 2715ade4 Sofia Papagiannaki
884 2715ade4 Sofia Papagiannaki
        logger.debug("update_object_hashmap: %s %s %s %s %s %s %s %s", user,
885 2715ade4 Sofia Papagiannaki
                     account, container, name, size, type, hashmap, checksum)
886 2715ade4 Sofia Papagiannaki
        if size == 0:  # No such thing as an empty hashmap.
887 6d64339e Antony Chazapis
            hashmap = [self.put_block('')]
888 1c2fc0ff Antony Chazapis
        map = HashMap(self.block_size, self.hash_algorithm)
889 1c2fc0ff Antony Chazapis
        map.extend([binascii.unhexlify(x) for x in hashmap])
890 7ca7bb08 Antony Chazapis
        missing = self.store.block_search(map)
891 a9b3f29d Antony Chazapis
        if missing:
892 a9b3f29d Antony Chazapis
            ie = IndexError()
893 dd71f493 Antony Chazapis
            ie.data = [binascii.hexlify(x) for x in missing]
894 a9b3f29d Antony Chazapis
            raise ie
895 2715ade4 Sofia Papagiannaki
896 1c2fc0ff Antony Chazapis
        hash = map.hash()
897 f9ea264b Antony Chazapis
        dest_version_id = self._update_object_hash(user, account, container, name, size, type, binascii.hexlify(hash), checksum, domain, meta, replace_meta, permissions)
898 7ca7bb08 Antony Chazapis
        self.store.map_put(hash, map)
899 02c4d2ba Antony Chazapis
        return dest_version_id
900 2715ade4 Sofia Papagiannaki
901 33b4e4a6 Antony Chazapis
    @backend_method
902 33b4e4a6 Antony Chazapis
    def update_object_checksum(self, user, account, container, name, version, checksum):
903 33b4e4a6 Antony Chazapis
        """Update an object's checksum."""
904 2715ade4 Sofia Papagiannaki
905 2715ade4 Sofia Papagiannaki
        logger.debug("update_object_checksum: %s %s %s %s %s %s",
906 2715ade4 Sofia Papagiannaki
                     user, account, container, name, version, checksum)
907 33b4e4a6 Antony Chazapis
        # Update objects with greater version and same hashmap and size (fix metadata updates).
908 33b4e4a6 Antony Chazapis
        self._can_write(user, account, container, name)
909 33b4e4a6 Antony Chazapis
        path, node = self._lookup_object(account, container, name)
910 33b4e4a6 Antony Chazapis
        props = self._get_version(node, version)
911 33b4e4a6 Antony Chazapis
        versions = self.node.node_get_versions(node)
912 33b4e4a6 Antony Chazapis
        for x in versions:
913 33b4e4a6 Antony Chazapis
            if x[self.SERIAL] >= int(version) and x[self.HASH] == props[self.HASH] and x[self.SIZE] == props[self.SIZE]:
914 2715ade4 Sofia Papagiannaki
                self.node.version_put_property(
915 2715ade4 Sofia Papagiannaki
                    x[self.SERIAL], 'checksum', checksum)
916 2715ade4 Sofia Papagiannaki
917 4d15c94e Sofia Papagiannaki
    def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, dest_domain=None, dest_meta={}, replace_meta=False, permissions=None, src_version=None, is_move=False, delimiter=None):
918 4d15c94e Sofia Papagiannaki
        dest_version_ids = []
919 79bb41b7 Antony Chazapis
        self._can_read(user, src_account, src_container, src_name)
920 b9064632 Antony Chazapis
        path, node = self._lookup_object(src_account, src_container, src_name)
921 1730b3bf chazapis
        # TODO: Will do another fetch of the properties in duplicate version...
922 2715ade4 Sofia Papagiannaki
        props = self._get_version(
923 2715ade4 Sofia Papagiannaki
            node, src_version)  # Check to see if source exists.
924 b9064632 Antony Chazapis
        src_version_id = props[self.SERIAL]
925 b9064632 Antony Chazapis
        hash = props[self.HASH]
926 b9064632 Antony Chazapis
        size = props[self.SIZE]
927 2715ade4 Sofia Papagiannaki
        is_copy = not is_move and (src_account, src_container, src_name) != (
928 2715ade4 Sofia Papagiannaki
            dest_account, dest_container, dest_name)  # New uuid.
929 4d15c94e Sofia Papagiannaki
        dest_version_ids.append(self._update_object_hash(user, dest_account, dest_container, dest_name, size, type, hash, None, dest_domain, dest_meta, replace_meta, permissions, src_node=node, src_version_id=src_version_id, is_copy=is_copy))
930 3f767854 Sofia Papagiannaki
        if is_move and (src_account, src_container, src_name) != (dest_account, dest_container, dest_name):
931 2715ade4 Sofia Papagiannaki
            self._delete_object(user, src_account, src_container, src_name)
932 2715ade4 Sofia Papagiannaki
933 4d15c94e Sofia Papagiannaki
        if delimiter:
934 2715ade4 Sofia Papagiannaki
            prefix = src_name + \
935 2715ade4 Sofia Papagiannaki
                delimiter if not src_name.endswith(delimiter) else src_name
936 e46b2bcf Sofia Papagiannaki
            src_names = self._list_objects_no_limit(user, src_account, src_container, prefix, delimiter=None, virtual=False, domain=None, keys=[], shared=False, until=None, size_range=None, all_props=True, public=False)
937 2715ade4 Sofia Papagiannaki
            src_names.sort(key=lambda x: x[2])  # order by nodes
938 4d15c94e Sofia Papagiannaki
            paths = [elem[0] for elem in src_names]
939 4d15c94e Sofia Papagiannaki
            nodes = [elem[2] for elem in src_names]
940 4d15c94e Sofia Papagiannaki
            # TODO: Will do another fetch of the properties in duplicate version...
941 2715ade4 Sofia Papagiannaki
            props = self._get_versions(nodes)  # Check to see if source exists.
942 2715ade4 Sofia Papagiannaki
943 4d15c94e Sofia Papagiannaki
            for prop, path, node in zip(props, paths, nodes):
944 4d15c94e Sofia Papagiannaki
                src_version_id = prop[self.SERIAL]
945 4d15c94e Sofia Papagiannaki
                hash = prop[self.HASH]
946 4d15c94e Sofia Papagiannaki
                vtype = prop[self.TYPE]
947 07867f70 Sofia Papagiannaki
                size = prop[self.SIZE]
948 2715ade4 Sofia Papagiannaki
                dest_prefix = dest_name + delimiter if not dest_name.endswith(
949 2715ade4 Sofia Papagiannaki
                    delimiter) else dest_name
950 4d15c94e Sofia Papagiannaki
                vdest_name = path.replace(prefix, dest_prefix, 1)
951 e46b2bcf Sofia Papagiannaki
                dest_version_ids.append(self._update_object_hash(user, dest_account, dest_container, vdest_name, size, vtype, hash, None, dest_domain, meta={}, replace_meta=False, permissions=None, src_node=node, src_version_id=src_version_id, is_copy=is_copy))
952 3f767854 Sofia Papagiannaki
                if is_move and (src_account, src_container, src_name) != (dest_account, dest_container, dest_name):
953 2715ade4 Sofia Papagiannaki
                    self._delete_object(user, src_account, src_container, path)
954 4d15c94e Sofia Papagiannaki
        return dest_version_ids[0] if len(dest_version_ids) == 1 else dest_version_ids
955 2715ade4 Sofia Papagiannaki
956 4d15c94e Sofia Papagiannaki
    @backend_method
957 4d15c94e Sofia Papagiannaki
    def copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, src_version=None, delimiter=None):
958 dff7b6f1 Sofia Papagiannaki
        """Copy an object's data and metadata."""
959 2715ade4 Sofia Papagiannaki
960 4d15c94e Sofia Papagiannaki
        logger.debug("copy_object: %s %s %s %s %s %s %s %s %s %s %s %s %s %s", user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, src_version, delimiter)
961 4d15c94e Sofia Papagiannaki
        dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, src_version, False, delimiter)
962 46286f5f Antony Chazapis
        return dest_version_id
963 2715ade4 Sofia Papagiannaki
964 dff7b6f1 Sofia Papagiannaki
    @backend_method
965 4d15c94e Sofia Papagiannaki
    def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta={}, replace_meta=False, permissions=None, delimiter=None):
966 a9b3f29d Antony Chazapis
        """Move an object's data and metadata."""
967 2715ade4 Sofia Papagiannaki
968 4d15c94e Sofia Papagiannaki
        logger.debug("move_object: %s %s %s %s %s %s %s %s %s %s %s %s %s", user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, delimiter)
969 79bb41b7 Antony Chazapis
        if user != src_account:
970 79bb41b7 Antony Chazapis
            raise NotAllowedError
971 4d15c94e Sofia Papagiannaki
        dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, None, True, delimiter)
972 02c4d2ba Antony Chazapis
        return dest_version_id
973 2715ade4 Sofia Papagiannaki
974 4d15c94e Sofia Papagiannaki
    def _delete_object(self, user, account, container, name, until=None, delimiter=None):
975 a9b3f29d Antony Chazapis
        if user != account:
976 a9b3f29d Antony Chazapis
            raise NotAllowedError
977 2715ade4 Sofia Papagiannaki
978 a9b3f29d Antony Chazapis
        if until is not None:
979 a9b3f29d Antony Chazapis
            path = '/'.join((account, container, name))
980 c915d3bf Antony Chazapis
            node = self.node.node_lookup(path)
981 c915d3bf Antony Chazapis
            if node is None:
982 c915d3bf Antony Chazapis
                return
983 813e42e5 Antony Chazapis
            hashes = []
984 813e42e5 Antony Chazapis
            size = 0
985 388ea25f Sofia Papagiannaki
            serials = []
986 388ea25f Sofia Papagiannaki
            h, s, v = self.node.node_purge(node, until, CLUSTER_NORMAL)
987 813e42e5 Antony Chazapis
            hashes += h
988 813e42e5 Antony Chazapis
            size += s
989 388ea25f Sofia Papagiannaki
            serials += v
990 388ea25f Sofia Papagiannaki
            h, s, v = self.node.node_purge(node, until, CLUSTER_HISTORY)
991 813e42e5 Antony Chazapis
            hashes += h
992 813e42e5 Antony Chazapis
            size += s
993 388ea25f Sofia Papagiannaki
            serials += v
994 04230536 Antony Chazapis
            for h in hashes:
995 04230536 Antony Chazapis
                self.store.map_delete(h)
996 4a1c29ea Antony Chazapis
            self.node.node_purge(node, until, CLUSTER_DELETED)
997 a9b3f29d Antony Chazapis
            try:
998 c915d3bf Antony Chazapis
                props = self._get_version(node)
999 a9b3f29d Antony Chazapis
            except NameError:
1000 0f9d752c Antony Chazapis
                self.permissions.access_clear(path)
1001 388ea25f Sofia Papagiannaki
            self._report_size_change(user, account, -size,
1002 7ed99da8 root
                                    {'action': 'object purge', 'path': path,
1003 b1dadd0e Sofia Papagiannaki
                                     'versions': ','.join(str(i) for i in serials)})
1004 a9b3f29d Antony Chazapis
            return
1005 2715ade4 Sofia Papagiannaki
1006 c915d3bf Antony Chazapis
        path, node = self._lookup_object(account, container, name)
1007 33b4e4a6 Antony Chazapis
        src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, type='', hash=None, checksum='', cluster=CLUSTER_DELETED)
1008 813e42e5 Antony Chazapis
        del_size = self._apply_versioning(account, container, src_version_id)
1009 b1dadd0e Sofia Papagiannaki
        self._report_size_change(user, account, -del_size,
1010 b1dadd0e Sofia Papagiannaki
                                 {'action': 'object delete', 'path': path,
1011 b1dadd0e Sofia Papagiannaki
                                  'versions': ','.join([str(dest_version_id)])})
1012 2715ade4 Sofia Papagiannaki
        self._report_object_change(
1013 2715ade4 Sofia Papagiannaki
            user, account, path, details={'action': 'object delete'})
1014 0f9d752c Antony Chazapis
        self.permissions.access_clear(path)
1015 2715ade4 Sofia Papagiannaki
1016 4d15c94e Sofia Papagiannaki
        if delimiter:
1017 4d15c94e Sofia Papagiannaki
            prefix = name + delimiter if not name.endswith(delimiter) else name
1018 e46b2bcf Sofia Papagiannaki
            src_names = self._list_objects_no_limit(user, account, container, prefix, delimiter=None, virtual=False, domain=None, keys=[], shared=False, until=None, size_range=None, all_props=True, public=False)
1019 4d15c94e Sofia Papagiannaki
            paths = []
1020 4d15c94e Sofia Papagiannaki
            for t in src_names:
1021 2715ade4 Sofia Papagiannaki
                path = '/'.join((account, container, t[0]))
1022 2715ade4 Sofia Papagiannaki
                node = t[2]
1023 4d15c94e Sofia Papagiannaki
                src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, type='', hash=None, checksum='', cluster=CLUSTER_DELETED)
1024 2715ade4 Sofia Papagiannaki
                del_size = self._apply_versioning(
1025 2715ade4 Sofia Papagiannaki
                    account, container, src_version_id)
1026 b1dadd0e Sofia Papagiannaki
                self._report_size_change(user, account, -del_size,
1027 b1dadd0e Sofia Papagiannaki
                                         {'action': 'object delete',
1028 b1dadd0e Sofia Papagiannaki
                                          'path': path,
1029 b1dadd0e Sofia Papagiannaki
                                          'versions': ','.join([str(dest_version_id)])})
1030 2715ade4 Sofia Papagiannaki
                self._report_object_change(
1031 2715ade4 Sofia Papagiannaki
                    user, account, path, details={'action': 'object delete'})
1032 4d15c94e Sofia Papagiannaki
                paths.append(path)
1033 4d15c94e Sofia Papagiannaki
            self.permissions.access_clear_bulk(paths)
1034 2715ade4 Sofia Papagiannaki
1035 4d15c94e Sofia Papagiannaki
    @backend_method
1036 4d15c94e Sofia Papagiannaki
    def delete_object(self, user, account, container, name, until=None, prefix='', delimiter=None):
1037 dff7b6f1 Sofia Papagiannaki
        """Delete/purge an object."""
1038 2715ade4 Sofia Papagiannaki
1039 2715ade4 Sofia Papagiannaki
        logger.debug("delete_object: %s %s %s %s %s %s %s", user,
1040 2715ade4 Sofia Papagiannaki
                     account, container, name, until, prefix, delimiter)
1041 4d15c94e Sofia Papagiannaki
        self._delete_object(user, account, container, name, until, delimiter)
1042 2715ade4 Sofia Papagiannaki
1043 dff7b6f1 Sofia Papagiannaki
    @backend_method
1044 62f915a1 Antony Chazapis
    def list_versions(self, user, account, container, name):
1045 62f915a1 Antony Chazapis
        """Return a list of all (version, version_timestamp) tuples for an object."""
1046 2715ade4 Sofia Papagiannaki
1047 2715ade4 Sofia Papagiannaki
        logger.debug(
1048 2715ade4 Sofia Papagiannaki
            "list_versions: %s %s %s %s", user, account, container, name)
1049 62f915a1 Antony Chazapis
        self._can_read(user, account, container, name)
1050 60b8a083 Antony Chazapis
        path, node = self._lookup_object(account, container, name)
1051 97d45f69 Antony Chazapis
        versions = self.node.node_get_versions(node)
1052 97d45f69 Antony Chazapis
        return [[x[self.SERIAL], x[self.MTIME]] for x in versions if x[self.CLUSTER] != CLUSTER_DELETED]
1053 2715ade4 Sofia Papagiannaki
1054 bb4eafc6 Antony Chazapis
    @backend_method
1055 37bee317 Antony Chazapis
    def get_uuid(self, user, uuid):
1056 37bee317 Antony Chazapis
        """Return the (account, container, name) for the UUID given."""
1057 2715ade4 Sofia Papagiannaki
1058 fe2db49d Sofia Papagiannaki
        logger.debug("get_uuid: %s %s", user, uuid)
1059 37bee317 Antony Chazapis
        info = self.node.latest_uuid(uuid)
1060 37bee317 Antony Chazapis
        if info is None:
1061 37bee317 Antony Chazapis
            raise NameError
1062 37bee317 Antony Chazapis
        path, serial = info
1063 37bee317 Antony Chazapis
        account, container, name = path.split('/', 2)
1064 37bee317 Antony Chazapis
        self._can_read(user, account, container, name)
1065 37bee317 Antony Chazapis
        return (account, container, name)
1066 2715ade4 Sofia Papagiannaki
1067 37bee317 Antony Chazapis
    @backend_method
1068 bb4eafc6 Antony Chazapis
    def get_public(self, user, public):
1069 bb4eafc6 Antony Chazapis
        """Return the (account, container, name) for the public id given."""
1070 2715ade4 Sofia Papagiannaki
1071 fe2db49d Sofia Papagiannaki
        logger.debug("get_public: %s %s", user, public)
1072 bb4eafc6 Antony Chazapis
        if public is None or public < ULTIMATE_ANSWER:
1073 bb4eafc6 Antony Chazapis
            raise NameError
1074 bb4eafc6 Antony Chazapis
        path = self.permissions.public_path(public - ULTIMATE_ANSWER)
1075 37bee317 Antony Chazapis
        if path is None:
1076 37bee317 Antony Chazapis
            raise NameError
1077 bb4eafc6 Antony Chazapis
        account, container, name = path.split('/', 2)
1078 bb4eafc6 Antony Chazapis
        self._can_read(user, account, container, name)
1079 bb4eafc6 Antony Chazapis
        return (account, container, name)
1080 2715ade4 Sofia Papagiannaki
1081 a9b3f29d Antony Chazapis
    @backend_method(autocommit=0)
1082 a9b3f29d Antony Chazapis
    def get_block(self, hash):
1083 a9b3f29d Antony Chazapis
        """Return a block's data."""
1084 2715ade4 Sofia Papagiannaki
1085 a9b3f29d Antony Chazapis
        logger.debug("get_block: %s", hash)
1086 7ca7bb08 Antony Chazapis
        block = self.store.block_get(binascii.unhexlify(hash))
1087 7ca7bb08 Antony Chazapis
        if not block:
1088 7efc9f86 Sofia Papagiannaki
            raise ItemNotExists('Block does not exist')
1089 7ca7bb08 Antony Chazapis
        return block
1090 2715ade4 Sofia Papagiannaki
1091 a9b3f29d Antony Chazapis
    @backend_method(autocommit=0)
1092 a9b3f29d Antony Chazapis
    def put_block(self, data):
1093 60b8a083 Antony Chazapis
        """Store a block and return the hash."""
1094 2715ade4 Sofia Papagiannaki
1095 a9b3f29d Antony Chazapis
        logger.debug("put_block: %s", len(data))
1096 7ca7bb08 Antony Chazapis
        return binascii.hexlify(self.store.block_put(data))
1097 2715ade4 Sofia Papagiannaki
1098 a9b3f29d Antony Chazapis
    @backend_method(autocommit=0)
1099 a9b3f29d Antony Chazapis
    def update_block(self, hash, data, offset=0):
1100 a9b3f29d Antony Chazapis
        """Update a known block and return the hash."""
1101 2715ade4 Sofia Papagiannaki
1102 a9b3f29d Antony Chazapis
        logger.debug("update_block: %s %s %s", hash, len(data), offset)
1103 a9b3f29d Antony Chazapis
        if offset == 0 and len(data) == self.block_size:
1104 a9b3f29d Antony Chazapis
            return self.put_block(data)
1105 7ca7bb08 Antony Chazapis
        h = self.store.block_update(binascii.unhexlify(hash), offset, data)
1106 a9b3f29d Antony Chazapis
        return binascii.hexlify(h)
1107 2715ade4 Sofia Papagiannaki
1108 44ad5860 Antony Chazapis
    # Path functions.
1109 2715ade4 Sofia Papagiannaki
1110 37bee317 Antony Chazapis
    def _generate_uuid(self):
1111 37bee317 Antony Chazapis
        return str(uuidlib.uuid4())
1112 2715ade4 Sofia Papagiannaki
1113 b9064632 Antony Chazapis
    def _put_object_node(self, path, parent, name):
1114 c915d3bf Antony Chazapis
        path = '/'.join((path, name))
1115 c915d3bf Antony Chazapis
        node = self.node.node_lookup(path)
1116 c915d3bf Antony Chazapis
        if node is None:
1117 c915d3bf Antony Chazapis
            node = self.node.node_create(parent, path)
1118 c915d3bf Antony Chazapis
        return path, node
1119 2715ade4 Sofia Papagiannaki
1120 62f915a1 Antony Chazapis
    def _put_path(self, user, parent, path):
1121 62f915a1 Antony Chazapis
        node = self.node.node_create(parent, path)
1122 2715ade4 Sofia Papagiannaki
        self.node.version_create(node, None, 0, '', None, user,
1123 2715ade4 Sofia Papagiannaki
                                 self._generate_uuid(), '', CLUSTER_NORMAL)
1124 62f915a1 Antony Chazapis
        return node
1125 2715ade4 Sofia Papagiannaki
1126 44ad5860 Antony Chazapis
    def _lookup_account(self, account, create=True):
1127 44ad5860 Antony Chazapis
        node = self.node.node_lookup(account)
1128 44ad5860 Antony Chazapis
        if node is None and create:
1129 2715ade4 Sofia Papagiannaki
            node = self._put_path(
1130 2715ade4 Sofia Papagiannaki
                account, self.ROOTNODE, account)  # User is account.
1131 c915d3bf Antony Chazapis
        return account, node
1132 2715ade4 Sofia Papagiannaki
1133 44ad5860 Antony Chazapis
    def _lookup_container(self, account, container):
1134 c915d3bf Antony Chazapis
        path = '/'.join((account, container))
1135 c915d3bf Antony Chazapis
        node = self.node.node_lookup(path)
1136 44ad5860 Antony Chazapis
        if node is None:
1137 7efc9f86 Sofia Papagiannaki
            raise ItemNotExists('Container does not exist')
1138 c915d3bf Antony Chazapis
        return path, node
1139 2715ade4 Sofia Papagiannaki
1140 44ad5860 Antony Chazapis
    def _lookup_object(self, account, container, name):
1141 c915d3bf Antony Chazapis
        path = '/'.join((account, container, name))
1142 c915d3bf Antony Chazapis
        node = self.node.node_lookup(path)
1143 44ad5860 Antony Chazapis
        if node is None:
1144 7efc9f86 Sofia Papagiannaki
            raise ItemNotExists('Object does not exist')
1145 c915d3bf Antony Chazapis
        return path, node
1146 2715ade4 Sofia Papagiannaki
1147 cf4a7a7b Sofia Papagiannaki
    def _lookup_objects(self, paths):
1148 7efc9f86 Sofia Papagiannaki
        nodes = self.node.node_lookup_bulk(paths)
1149 cf4a7a7b Sofia Papagiannaki
        return paths, nodes
1150 2715ade4 Sofia Papagiannaki
1151 44ad5860 Antony Chazapis
    def _get_properties(self, node, until=None):
1152 44ad5860 Antony Chazapis
        """Return properties until the timestamp given."""
1153 2715ade4 Sofia Papagiannaki
1154 44ad5860 Antony Chazapis
        before = until if until is not None else inf
1155 44ad5860 Antony Chazapis
        props = self.node.version_lookup(node, before, CLUSTER_NORMAL)
1156 44ad5860 Antony Chazapis
        if props is None and until is not None:
1157 44ad5860 Antony Chazapis
            props = self.node.version_lookup(node, before, CLUSTER_HISTORY)
1158 44ad5860 Antony Chazapis
        if props is None:
1159 7efc9f86 Sofia Papagiannaki
            raise ItemNotExists('Path does not exist')
1160 44ad5860 Antony Chazapis
        return props
1161 2715ade4 Sofia Papagiannaki
1162 62f915a1 Antony Chazapis
    def _get_statistics(self, node, until=None):
1163 62f915a1 Antony Chazapis
        """Return count, sum of size and latest timestamp of everything under node."""
1164 2715ade4 Sofia Papagiannaki
1165 44ad5860 Antony Chazapis
        if until is None:
1166 62f915a1 Antony Chazapis
            stats = self.node.statistics_get(node, CLUSTER_NORMAL)
1167 44ad5860 Antony Chazapis
        else:
1168 62f915a1 Antony Chazapis
            stats = self.node.statistics_latest(node, until, CLUSTER_DELETED)
1169 62f915a1 Antony Chazapis
        if stats is None:
1170 62f915a1 Antony Chazapis
            stats = (0, 0, 0)
1171 62f915a1 Antony Chazapis
        return stats
1172 2715ade4 Sofia Papagiannaki
1173 44ad5860 Antony Chazapis
    def _get_version(self, node, version=None):
1174 44ad5860 Antony Chazapis
        if version is None:
1175 44ad5860 Antony Chazapis
            props = self.node.version_lookup(node, inf, CLUSTER_NORMAL)
1176 44ad5860 Antony Chazapis
            if props is None:
1177 7efc9f86 Sofia Papagiannaki
                raise ItemNotExists('Object does not exist')
1178 44ad5860 Antony Chazapis
        else:
1179 07afd277 Antony Chazapis
            try:
1180 07afd277 Antony Chazapis
                version = int(version)
1181 07afd277 Antony Chazapis
            except ValueError:
1182 7efc9f86 Sofia Papagiannaki
                raise VersionNotExists('Version does not exist')
1183 44ad5860 Antony Chazapis
            props = self.node.version_get_properties(version)
1184 2c5363a0 Antony Chazapis
            if props is None or props[self.CLUSTER] == CLUSTER_DELETED:
1185 7efc9f86 Sofia Papagiannaki
                raise VersionNotExists('Version does not exist')
1186 44ad5860 Antony Chazapis
        return props
1187 4d15c94e Sofia Papagiannaki
1188 7efc9f86 Sofia Papagiannaki
    def _get_versions(self, nodes):
1189 7efc9f86 Sofia Papagiannaki
        return self.node.version_lookup_bulk(nodes, inf, CLUSTER_NORMAL)
1190 2715ade4 Sofia Papagiannaki
1191 33b4e4a6 Antony Chazapis
    def _put_version_duplicate(self, user, node, src_node=None, size=None, type=None, hash=None, checksum=None, cluster=CLUSTER_NORMAL, is_copy=False):
1192 b9064632 Antony Chazapis
        """Create a new version of the node."""
1193 2715ade4 Sofia Papagiannaki
1194 2715ade4 Sofia Papagiannaki
        props = self.node.version_lookup(
1195 2715ade4 Sofia Papagiannaki
            node if src_node is None else src_node, inf, CLUSTER_NORMAL)
1196 b9064632 Antony Chazapis
        if props is not None:
1197 b9064632 Antony Chazapis
            src_version_id = props[self.SERIAL]
1198 b9064632 Antony Chazapis
            src_hash = props[self.HASH]
1199 b9064632 Antony Chazapis
            src_size = props[self.SIZE]
1200 66ce2ca5 Antony Chazapis
            src_type = props[self.TYPE]
1201 33b4e4a6 Antony Chazapis
            src_checksum = props[self.CHECKSUM]
1202 44ad5860 Antony Chazapis
        else:
1203 b9064632 Antony Chazapis
            src_version_id = None
1204 b9064632 Antony Chazapis
            src_hash = None
1205 b9064632 Antony Chazapis
            src_size = 0
1206 66ce2ca5 Antony Chazapis
            src_type = ''
1207 33b4e4a6 Antony Chazapis
            src_checksum = ''
1208 2715ade4 Sofia Papagiannaki
        if size is None:  # Set metadata.
1209 2715ade4 Sofia Papagiannaki
            hash = src_hash  # This way hash can be set to None (account or container).
1210 b9064632 Antony Chazapis
            size = src_size
1211 66ce2ca5 Antony Chazapis
        if type is None:
1212 66ce2ca5 Antony Chazapis
            type = src_type
1213 33b4e4a6 Antony Chazapis
        if checksum is None:
1214 33b4e4a6 Antony Chazapis
            checksum = src_checksum
1215 2715ade4 Sofia Papagiannaki
        uuid = self._generate_uuid(
1216 2715ade4 Sofia Papagiannaki
        ) if (is_copy or src_version_id is None) else props[self.UUID]
1217 2715ade4 Sofia Papagiannaki
1218 1730b3bf chazapis
        if src_node is None:
1219 1730b3bf chazapis
            pre_version_id = src_version_id
1220 1730b3bf chazapis
        else:
1221 1730b3bf chazapis
            pre_version_id = None
1222 1730b3bf chazapis
            props = self.node.version_lookup(node, inf, CLUSTER_NORMAL)
1223 1730b3bf chazapis
            if props is not None:
1224 1730b3bf chazapis
                pre_version_id = props[self.SERIAL]
1225 1730b3bf chazapis
        if pre_version_id is not None:
1226 1730b3bf chazapis
            self.node.version_recluster(pre_version_id, CLUSTER_HISTORY)
1227 2715ade4 Sofia Papagiannaki
1228 33b4e4a6 Antony Chazapis
        dest_version_id, mtime = self.node.version_create(node, hash, size, type, src_version_id, user, uuid, checksum, cluster)
1229 1730b3bf chazapis
        return pre_version_id, dest_version_id
1230 2715ade4 Sofia Papagiannaki
1231 4819d34f Antony Chazapis
    def _put_metadata_duplicate(self, src_version_id, dest_version_id, domain, meta, replace=False):
1232 4819d34f Antony Chazapis
        if src_version_id is not None:
1233 4819d34f Antony Chazapis
            self.node.attribute_copy(src_version_id, dest_version_id)
1234 4819d34f Antony Chazapis
        if not replace:
1235 2715ade4 Sofia Papagiannaki
            self.node.attribute_del(dest_version_id, domain, (
1236 2715ade4 Sofia Papagiannaki
                k for k, v in meta.iteritems() if v == ''))
1237 2715ade4 Sofia Papagiannaki
            self.node.attribute_set(dest_version_id, domain, (
1238 2715ade4 Sofia Papagiannaki
                (k, v) for k, v in meta.iteritems() if v != ''))
1239 4819d34f Antony Chazapis
        else:
1240 4819d34f Antony Chazapis
            self.node.attribute_del(dest_version_id, domain)
1241 2715ade4 Sofia Papagiannaki
            self.node.attribute_set(dest_version_id, domain, ((
1242 2715ade4 Sofia Papagiannaki
                k, v) for k, v in meta.iteritems()))
1243 2715ade4 Sofia Papagiannaki
1244 4819d34f Antony Chazapis
    def _put_metadata(self, user, node, domain, meta, replace=False):
1245 44ad5860 Antony Chazapis
        """Create a new version and store metadata."""
1246 2715ade4 Sofia Papagiannaki
1247 2715ade4 Sofia Papagiannaki
        src_version_id, dest_version_id = self._put_version_duplicate(
1248 2715ade4 Sofia Papagiannaki
            user, node)
1249 2715ade4 Sofia Papagiannaki
        self._put_metadata_duplicate(
1250 2715ade4 Sofia Papagiannaki
            src_version_id, dest_version_id, domain, meta, replace)
1251 5cc484e1 Antony Chazapis
        return src_version_id, dest_version_id
1252 2715ade4 Sofia Papagiannaki
1253 60b8a083 Antony Chazapis
    def _list_limits(self, listing, marker, limit):
1254 60b8a083 Antony Chazapis
        start = 0
1255 60b8a083 Antony Chazapis
        if marker:
1256 60b8a083 Antony Chazapis
            try:
1257 60b8a083 Antony Chazapis
                start = listing.index(marker) + 1
1258 60b8a083 Antony Chazapis
            except ValueError:
1259 60b8a083 Antony Chazapis
                pass
1260 60b8a083 Antony Chazapis
        if not limit or limit > 10000:
1261 60b8a083 Antony Chazapis
            limit = 10000
1262 60b8a083 Antony Chazapis
        return start, limit
1263 2715ade4 Sofia Papagiannaki
1264 371d907a Antony Chazapis
    def _list_object_properties(self, parent, path, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], until=None, size_range=None, allowed=[], all_props=False):
1265 60b8a083 Antony Chazapis
        cont_prefix = path + '/'
1266 60b8a083 Antony Chazapis
        prefix = cont_prefix + prefix
1267 60b8a083 Antony Chazapis
        start = cont_prefix + marker if marker else None
1268 60b8a083 Antony Chazapis
        before = until if until is not None else inf
1269 4819d34f Antony Chazapis
        filterq = keys if domain else []
1270 7ff57991 Antony Chazapis
        sizeq = size_range
1271 2715ade4 Sofia Papagiannaki
1272 371d907a Antony Chazapis
        objects, prefixes = self.node.latest_version_list(parent, prefix, delimiter, start, limit, before, CLUSTER_DELETED, allowed, domain, filterq, sizeq, all_props)
1273 60b8a083 Antony Chazapis
        objects.extend([(p, None) for p in prefixes] if virtual else [])
1274 43be9afd Sofia Papagiannaki
        objects.sort(key=lambda x: x[0])
1275 371d907a Antony Chazapis
        objects = [(x[0][len(cont_prefix):],) + x[1:] for x in objects]
1276 cf4a7a7b Sofia Papagiannaki
        return objects
1277 2715ade4 Sofia Papagiannaki
1278 813e42e5 Antony Chazapis
    # Reporting functions.
1279 2715ade4 Sofia Papagiannaki
1280 813e42e5 Antony Chazapis
    def _report_size_change(self, user, account, size, details={}):
1281 8ed3f04c Sofia Papagiannaki
        if size == 0:
1282 8ed3f04c Sofia Papagiannaki
            return
1283 8ed3f04c Sofia Papagiannaki
1284 813e42e5 Antony Chazapis
        account_node = self._lookup_account(account, True)[1]
1285 813e42e5 Antony Chazapis
        total = self._get_statistics(account_node)[1]
1286 813e42e5 Antony Chazapis
        details.update({'user': user, 'total': total})
1287 2715ade4 Sofia Papagiannaki
        logger.debug(
1288 2715ade4 Sofia Papagiannaki
            "_report_size_change: %s %s %s %s", user, account, size, details)
1289 6eb0de11 Sofia Papagiannaki
        self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('resource.diskspace',),
1290 7ed99da8 root
                              account, QUEUE_INSTANCE_ID, 'diskspace',
1291 7ed99da8 root
                              float(size), details))
1292 7ed99da8 root
1293 c846fad1 Sofia Papagiannaki
        if not self.using_external_quotaholder:
1294 73fbe301 Sofia Papagiannaki
            return
1295 7f1f0464 Georgios D. Tsoukalas
1296 ccfd4e44 Sofia Papagiannaki
        try:
1297 ccfd4e44 Sofia Papagiannaki
            serial = self.quotaholder.issue_commission(
1298 ccfd4e44 Sofia Papagiannaki
                    context     =   {},
1299 ccfd4e44 Sofia Papagiannaki
                    target      =   account,
1300 ccfd4e44 Sofia Papagiannaki
                    key         =   '1',
1301 ccfd4e44 Sofia Papagiannaki
                    clientkey   =   'pithos',
1302 ccfd4e44 Sofia Papagiannaki
                    ownerkey    =   '',
1303 ccfd4e44 Sofia Papagiannaki
                    name        =   details['path'] if 'path' in details else '',
1304 ccfd4e44 Sofia Papagiannaki
                    provisions  =   (('pithos+', 'pithos+.diskspace', size),)
1305 ccfd4e44 Sofia Papagiannaki
            )
1306 ccfd4e44 Sofia Papagiannaki
        except BaseException, e:
1307 ccfd4e44 Sofia Papagiannaki
            raise QuotaError(e)
1308 ccfd4e44 Sofia Papagiannaki
        else:
1309 ccfd4e44 Sofia Papagiannaki
            self.serials.append(serial)
1310 0307b47f Georgios D. Tsoukalas
1311 39ef6f41 Antony Chazapis
    def _report_object_change(self, user, account, path, details={}):
1312 b82d3277 Sofia Papagiannaki
        details.update({'user': user})
1313 2715ade4 Sofia Papagiannaki
        logger.debug("_report_object_change: %s %s %s %s", user,
1314 2715ade4 Sofia Papagiannaki
                     account, path, details)
1315 f4fbb0fa Sofia Papagiannaki
        self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('object',),
1316 b1dadd0e Sofia Papagiannaki
                              account, QUEUE_INSTANCE_ID, 'object', path, details))
1317 2715ade4 Sofia Papagiannaki
1318 a74ba506 Sofia Papagiannaki
    def _report_sharing_change(self, user, account, path, details={}):
1319 2715ade4 Sofia Papagiannaki
        logger.debug("_report_permissions_change: %s %s %s %s",
1320 2715ade4 Sofia Papagiannaki
                     user, account, path, details)
1321 a74ba506 Sofia Papagiannaki
        details.update({'user': user})
1322 f4fbb0fa Sofia Papagiannaki
        self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('sharing',),
1323 b1dadd0e Sofia Papagiannaki
                              account, QUEUE_INSTANCE_ID, 'sharing', path, details))
1324 2715ade4 Sofia Papagiannaki
1325 60b8a083 Antony Chazapis
    # Policy functions.
1326 2715ade4 Sofia Papagiannaki
1327 60b8a083 Antony Chazapis
    def _check_policy(self, policy):
1328 60b8a083 Antony Chazapis
        for k in policy.keys():
1329 60b8a083 Antony Chazapis
            if policy[k] == '':
1330 60b8a083 Antony Chazapis
                policy[k] = self.default_policy.get(k)
1331 60b8a083 Antony Chazapis
        for k, v in policy.iteritems():
1332 60b8a083 Antony Chazapis
            if k == 'quota':
1333 2715ade4 Sofia Papagiannaki
                q = int(v)  # May raise ValueError.
1334 60b8a083 Antony Chazapis
                if q < 0:
1335 60b8a083 Antony Chazapis
                    raise ValueError
1336 60b8a083 Antony Chazapis
            elif k == 'versioning':
1337 5cc484e1 Antony Chazapis
                if v not in ['auto', 'none']:
1338 60b8a083 Antony Chazapis
                    raise ValueError
1339 60b8a083 Antony Chazapis
            else:
1340 60b8a083 Antony Chazapis
                raise ValueError
1341 2715ade4 Sofia Papagiannaki
1342 b2832c6a Antony Chazapis
    def _put_policy(self, node, policy, replace):
1343 b2832c6a Antony Chazapis
        if replace:
1344 b2832c6a Antony Chazapis
            for k, v in self.default_policy.iteritems():
1345 b2832c6a Antony Chazapis
                if k not in policy:
1346 b2832c6a Antony Chazapis
                    policy[k] = v
1347 b2832c6a Antony Chazapis
        self.node.policy_set(node, policy)
1348 2715ade4 Sofia Papagiannaki
1349 b9064632 Antony Chazapis
    def _get_policy(self, node):
1350 b9064632 Antony Chazapis
        policy = self.default_policy.copy()
1351 b9064632 Antony Chazapis
        policy.update(self.node.policy_get(node))
1352 b9064632 Antony Chazapis
        return policy
1353 2715ade4 Sofia Papagiannaki
1354 5cc484e1 Antony Chazapis
    def _apply_versioning(self, account, container, version_id):
1355 813e42e5 Antony Chazapis
        """Delete the provided version if such is the policy.
1356 813e42e5 Antony Chazapis
           Return size of object removed.
1357 813e42e5 Antony Chazapis
        """
1358 2715ade4 Sofia Papagiannaki
1359 5cc484e1 Antony Chazapis
        if version_id is None:
1360 813e42e5 Antony Chazapis
            return 0
1361 5cc484e1 Antony Chazapis
        path, node = self._lookup_container(account, container)
1362 5cc484e1 Antony Chazapis
        versioning = self._get_policy(node)['versioning']
1363 1dd34bdd Sofia Papagiannaki
        if versioning != 'auto':
1364 813e42e5 Antony Chazapis
            hash, size = self.node.version_remove(version_id)
1365 5161c672 Antony Chazapis
            self.store.map_delete(hash)
1366 813e42e5 Antony Chazapis
            return size
1367 1dd34bdd Sofia Papagiannaki
        elif self.free_versioning:
1368 fff37615 Sofia Papagiannaki
            return self.node.version_get_properties(
1369 fff37615 Sofia Papagiannaki
                version_id, keys=('size',))[0]
1370 813e42e5 Antony Chazapis
        return 0
1371 2715ade4 Sofia Papagiannaki
1372 a9b3f29d Antony Chazapis
    # Access control functions.
1373 2715ade4 Sofia Papagiannaki
1374 a9b3f29d Antony Chazapis
    def _check_groups(self, groups):
1375 0f9d752c Antony Chazapis
        # raise ValueError('Bad characters in groups')
1376 a9b3f29d Antony Chazapis
        pass
1377 2715ade4 Sofia Papagiannaki
1378 a9b3f29d Antony Chazapis
    def _check_permissions(self, path, permissions):
1379 0f9d752c Antony Chazapis
        # raise ValueError('Bad characters in permissions')
1380 5e068361 Antony Chazapis
        pass
1381 2715ade4 Sofia Papagiannaki
1382 92da0e5a Antony Chazapis
    def _get_formatted_paths(self, paths):
1383 92da0e5a Antony Chazapis
        formatted = []
1384 92da0e5a Antony Chazapis
        for p in paths:
1385 92da0e5a Antony Chazapis
            node = self.node.node_lookup(p)
1386 a8b82467 Sofia Papagiannaki
            props = None
1387 92da0e5a Antony Chazapis
            if node is not None:
1388 92da0e5a Antony Chazapis
                props = self.node.version_lookup(node, inf, CLUSTER_NORMAL)
1389 92da0e5a Antony Chazapis
            if props is not None:
1390 692485cc Antony Chazapis
                if props[self.TYPE].split(';', 1)[0].strip() in ('application/directory', 'application/folder'):
1391 d57eaad4 Antony Chazapis
                    formatted.append((p.rstrip('/') + '/', self.MATCH_PREFIX))
1392 d57eaad4 Antony Chazapis
                formatted.append((p, self.MATCH_EXACT))
1393 92da0e5a Antony Chazapis
        return formatted
1394 2715ade4 Sofia Papagiannaki
1395 5e068361 Antony Chazapis
    def _get_permissions_path(self, account, container, name):
1396 5e068361 Antony Chazapis
        path = '/'.join((account, container, name))
1397 5e068361 Antony Chazapis
        permission_paths = self.permissions.access_inherit(path)
1398 5e068361 Antony Chazapis
        permission_paths.sort()
1399 5e068361 Antony Chazapis
        permission_paths.reverse()
1400 5e068361 Antony Chazapis
        for p in permission_paths:
1401 5e068361 Antony Chazapis
            if p == path:
1402 5e068361 Antony Chazapis
                return p
1403 5e068361 Antony Chazapis
            else:
1404 71dbc012 Antony Chazapis
                if p.count('/') < 2:
1405 71dbc012 Antony Chazapis
                    continue
1406 92da0e5a Antony Chazapis
                node = self.node.node_lookup(p)
1407 d62295ba Sofia Papagiannaki
                props = None
1408 92da0e5a Antony Chazapis
                if node is not None:
1409 92da0e5a Antony Chazapis
                    props = self.node.version_lookup(node, inf, CLUSTER_NORMAL)
1410 92da0e5a Antony Chazapis
                if props is not None:
1411 692485cc Antony Chazapis
                    if props[self.TYPE].split(';', 1)[0].strip() in ('application/directory', 'application/folder'):
1412 5e068361 Antony Chazapis
                        return p
1413 5e068361 Antony Chazapis
        return None
1414 2715ade4 Sofia Papagiannaki
1415 6f4bce7b Antony Chazapis
    def _can_read(self, user, account, container, name):
1416 a9b3f29d Antony Chazapis
        if user == account:
1417 a9b3f29d Antony Chazapis
            return True
1418 a9b3f29d Antony Chazapis
        path = '/'.join((account, container, name))
1419 aeb2b64f Antony Chazapis
        if self.permissions.public_get(path) is not None:
1420 71dbc012 Antony Chazapis
            return True
1421 5e068361 Antony Chazapis
        path = self._get_permissions_path(account, container, name)
1422 71dbc012 Antony Chazapis
        if not path:
1423 71dbc012 Antony Chazapis
            raise NotAllowedError
1424 2c5363a0 Antony Chazapis
        if not self.permissions.access_check(path, self.READ, user) and not self.permissions.access_check(path, self.WRITE, user):
1425 a9b3f29d Antony Chazapis
            raise NotAllowedError
1426 2715ade4 Sofia Papagiannaki
1427 a9b3f29d Antony Chazapis
    def _can_write(self, user, account, container, name):
1428 6f4bce7b Antony Chazapis
        if user == account:
1429 6f4bce7b Antony Chazapis
            return True
1430 6f4bce7b Antony Chazapis
        path = '/'.join((account, container, name))
1431 71dbc012 Antony Chazapis
        path = self._get_permissions_path(account, container, name)
1432 71dbc012 Antony Chazapis
        if not path:
1433 71dbc012 Antony Chazapis
            raise NotAllowedError
1434 2c5363a0 Antony Chazapis
        if not self.permissions.access_check(path, self.WRITE, user):
1435 a9b3f29d Antony Chazapis
            raise NotAllowedError
1436 2715ade4 Sofia Papagiannaki
1437 a9b3f29d Antony Chazapis
    def _allowed_accounts(self, user):
1438 a9b3f29d Antony Chazapis
        allow = set()
1439 0f9d752c Antony Chazapis
        for path in self.permissions.access_list_paths(user):
1440 a9b3f29d Antony Chazapis
            allow.add(path.split('/', 1)[0])
1441 a9b3f29d Antony Chazapis
        return sorted(allow)
1442 2715ade4 Sofia Papagiannaki
1443 a9b3f29d Antony Chazapis
    def _allowed_containers(self, user, account):
1444 a9b3f29d Antony Chazapis
        allow = set()
1445 0f9d752c Antony Chazapis
        for path in self.permissions.access_list_paths(user, account):
1446 a9b3f29d Antony Chazapis
            allow.add(path.split('/', 2)[1])
1447 a9b3f29d Antony Chazapis
        return sorted(allow)