Revision f4b61e0c

b/README
17 17
Running the server
18 18
------------------
19 19

  
20
Enter the pithos dir and run:
20
Enter the pithos dir, copy settings.py.dist to settings.py and change the SECRET_KEY inside.
21

  
22
Then run:
21 23
    python manage.py syncdb
22 24
    python manage.py loaddata aai/fixtures/auth_test_data.json (to load sample users)
23 25
    python manage.py runserver
b/docs/source/adminguide.rst
16 16
  cd /
17 17
  git clone https://code.grnet.gr/git/pithos
18 18

  
19
Setup the files (choose where to store data in ``settings.py``)::
19
Setup the files (choose where to store data in ``settings.py`` and change ``SECRET_KEY``)::
20 20

  
21 21
  cd /pithos/pithos
22 22
  cp settings.py.dist settings.py
b/docs/source/backends.rst
2 2
========
3 3

  
4 4
.. toctree::
5
   :maxdepth: 1
5
   :maxdepth: 3
6 6
   
7
   base_backend
8
   simple_backend
7
BaseBackend
8
-----------
9

  
10
.. autoclass:: pithos.backends.base.BaseBackend
11

  
12
SimpleBackend
13
-------------
14

  
15
.. autoclass:: pithos.backends.simple.SimpleBackend
16
   :show-inheritance:
17
   :members:
18
   :inherited-members:
19

  
20
ModularBackend
21
--------------
22

  
23
.. autoclass:: pithos.backends.modular.ModularBackend
24
   :show-inheritance:
25
   :members:
26
   :inherited-members:
27

  
28
Node
29
~~~~
30

  
31
.. automodule:: pithos.backends.lib.node
32
   :show-inheritance:
33
   :members:
34
   :undoc-members:
35

  
36
Policy
37
~~~~~~
38

  
39
.. automodule:: pithos.backends.lib.policy
40
   :show-inheritance:
41
   :members:
42
   :undoc-members:
43

  
44
Permissions
45
~~~~~~~~~~~
46

  
47
.. automodule:: pithos.backends.lib.permissions
48
   :show-inheritance:
49
   :members:
50
   :undoc-members:
51

  
52
Hashfiler
53
~~~~~~~~~
54

  
55
.. automodule:: pithos.backends.lib.hashfiler
56
   :show-inheritance:
57
   :members:
58
   :undoc-members:
/dev/null
1
BaseBackend
2
===========
3

  
4
.. automodule:: pithos.backends.base
/dev/null
1
Client Library
2
==============
3

  
4
.. automodule:: pithos.lib.client
b/docs/source/clientlib.rst
1
Client Library
2
==============
3

  
4
.. automodule:: tools.lib.client
5
   :show-inheritance:
6
   :members:
7
   :undoc-members:
/dev/null
1
Clients
2
========
3

  
4
.. toctree::
5
   :maxdepth: 1
6
   
7
   client-lib
b/docs/source/index.rst
4 4
Contents:
5 5

  
6 6
.. toctree::
7
   :maxdepth: 2
7
   :maxdepth: 3
8 8
   
9 9
   devguide
10 10
   adminguide
11 11
   backends
12
   clients
12
   clientlib
13 13

  
14 14
Indices and tables
15 15
==================
/dev/null
1
SimpleBackend
2
=============
3

  
4
.. automodule:: pithos.backends.simple
b/pithos/backends/base.py
46 46
    
47 47
    The following variables should be available:
48 48
        'hash_algorithm': Suggested is 'sha256'
49
        
49 50
        'block_size': Suggested is 4MB
50 51
    """
51 52
    
......
54 55
        
55 56
        Parameters:
56 57
            'marker': Start list from the next item after 'marker'
58
            
57 59
            'limit': Number of containers to return
58 60
        """
59 61
        return []
......
63 65
        
64 66
        The keys returned are all user-defined, except:
65 67
            'name': The account name
68
            
66 69
            'count': The number of containers (or 0)
70
            
67 71
            'bytes': The total data size (or 0)
72
            
68 73
            'modified': Last modification timestamp (overall)
74
            
69 75
            'until_timestamp': Last modification until the timestamp provided
70 76
        
71 77
        Raises:
......
78 84
        
79 85
        Parameters:
80 86
            'meta': Dictionary with metadata to update
87
            
81 88
            'replace': Replace instead of update
82 89
        
83 90
        Raises:
......
98 105
        
99 106
        Raises:
100 107
            NotAllowedError: Operation not permitted
108
            
101 109
            ValueError: Invalid data in groups
102 110
        """
103 111
        return
......
115 123
        
116 124
        Raises:
117 125
            NotAllowedError: Operation not permitted
126
            
118 127
            IndexError: Account is not empty
119 128
        """
120 129
        return
......
124 133
        
125 134
        Parameters:
126 135
            'marker': Start list from the next item after 'marker'
136
            
127 137
            'limit': Number of containers to return
138
            
128 139
            'shared': Only list containers with permissions set
140
            
129 141
        
130 142
        Raises:
131 143
            NotAllowedError: Operation not permitted
......
137 149
        
138 150
        The keys returned are all user-defined, except:
139 151
            'name': The container name
152
            
140 153
            'count': The number of objects
154
            
141 155
            'bytes': The total data size
156
            
142 157
            'modified': Last modification timestamp (overall)
158
            
143 159
            'until_timestamp': Last modification until the timestamp provided
144 160
        
145 161
        Raises:
146 162
            NotAllowedError: Operation not permitted
163
            
147 164
            NameError: Container does not exist
148 165
        """
149 166
        return {}
......
153 170
        
154 171
        Parameters:
155 172
            'meta': Dictionary with metadata to update
173
            
156 174
            'replace': Replace instead of update
157 175
        
158 176
        Raises:
159 177
            NotAllowedError: Operation not permitted
178
            
160 179
            NameError: Container does not exist
161 180
        """
162 181
        return
......
166 185
        
167 186
        The keys returned are:
168 187
            'quota': The maximum bytes allowed (default is 0 - unlimited)
188
            
169 189
            'versioning': Can be 'auto', 'manual' or 'none' (default is 'manual')
170 190
        
171 191
        Raises:
172 192
            NotAllowedError: Operation not permitted
193
            
173 194
            NameError: Container does not exist
174 195
        """
175 196
        return {}
......
179 200
        
180 201
        Raises:
181 202
            NotAllowedError: Operation not permitted
203
            
182 204
            NameError: Container does not exist
205
            
183 206
            ValueError: Invalid policy defined
184 207
        """
185 208
        return
......
189 212
        
190 213
        Raises:
191 214
            NotAllowedError: Operation not permitted
215
            
192 216
            NameError: Container already exists
217
            
193 218
            ValueError: Invalid policy defined
194 219
        """
195 220
        return
......
199 224
        
200 225
        Raises:
201 226
            NotAllowedError: Operation not permitted
227
            
202 228
            NameError: Container does not exist
229
            
203 230
            IndexError: Container is not empty
204 231
        """
205 232
        return
......
209 236
        
210 237
        Parameters:
211 238
            'prefix': List objects starting with 'prefix'
239
            
212 240
            'delimiter': Return unique names before 'delimiter' and after 'prefix'
241
            
213 242
            'marker': Start list from the next item after 'marker'
243
            
214 244
            'limit': Number of objects to return
215
            'virtual': If not set, the result will only include names starting\
216
                       with 'prefix' and ending without a 'delimiter' or with\
217
                       the first occurance of the 'delimiter' after 'prefix'.\
218
                       If set, the result will include all names after 'prefix',\
245
            
246
            'virtual': If not set, the result will only include names starting
247
                       with 'prefix' and ending without a 'delimiter' or with
248
                       the first occurance of the 'delimiter' after 'prefix'.
249
                       If set, the result will include all names after 'prefix',
219 250
                       up to and including the 'delimiter' if it is found
251
            
220 252
            'keys': Include objects that have meta with the keys in the list
253
            
221 254
            'shared': Only list objects with permissions set
222 255
        
223 256
        Raises:
224 257
            NotAllowedError: Operation not permitted
258
            
225 259
            NameError: Container does not exist
226 260
        """
227 261
        return []
......
231 265
        
232 266
        Raises:
233 267
            NotAllowedError: Operation not permitted
268
            
234 269
            NameError: Container does not exist
235 270
        """
236 271
        return []
......
240 275
        
241 276
        The keys returned are all user-defined, except:
242 277
            'name': The object name
278
            
243 279
            'bytes': The total data size
280
            
244 281
            'modified': Last modification timestamp (overall)
282
            
245 283
            'modified_by': The user that committed the object (version requested)
284
            
246 285
            'version': The version identifier
286
            
247 287
            'version_timestamp': The version's modification timestamp
248 288
        
249 289
        Raises:
250 290
            NotAllowedError: Operation not permitted
291
            
251 292
            NameError: Container/object does not exist
293
            
252 294
            IndexError: Version does not exist
253 295
        """
254 296
        return {}
......
258 300
        
259 301
        Parameters:
260 302
            'meta': Dictionary with metadata to update
303
            
261 304
            'replace': Replace instead of update
262 305
        
263 306
        Raises:
264 307
            NotAllowedError: Operation not permitted
308
            
265 309
            NameError: Container/object does not exist
266 310
        """
267 311
        return
268 312
    
269 313
    def get_object_permissions(self, user, account, container, name):
270
        """Return the path from which this object gets its permissions from,\
314
        """Return the path from which this object gets its permissions from,
271 315
        along with a dictionary containing the permissions.
272 316
        
273 317
        The keys are:
274 318
            'read': The object is readable by the users/groups in the list
319
            
275 320
            'write': The object is writable by the users/groups in the list
276 321
        
277 322
        Raises:
278 323
            NotAllowedError: Operation not permitted
324
            
279 325
            NameError: Container/object does not exist
280 326
        """
281 327
        return {}
......
288 334
        
289 335
        Raises:
290 336
            NotAllowedError: Operation not permitted
337
            
291 338
            NameError: Container/object does not exist
339
            
292 340
            ValueError: Invalid users/groups in permissions
293
            AttributeError: Can not set permissions, as this object\
294
                is already shared/private by another object higher\
295
                in the hierarchy, or setting permissions here will\
341
            
342
            AttributeError: Can not set permissions, as this object
343
                is already shared/private by another object higher
344
                in the hierarchy, or setting permissions here will
296 345
                invalidate other permissions deeper in the hierarchy
297 346
        """
298 347
        return
......
302 351
        
303 352
        Raises:
304 353
            NotAllowedError: Operation not permitted
354
            
305 355
            NameError: Container/object does not exist
306 356
        """
307 357
        return None
......
314 364
        
315 365
        Raises:
316 366
            NotAllowedError: Operation not permitted
367
            
317 368
            NameError: Container/object does not exist
318 369
        """
319 370
        return
......
323 374
        
324 375
        Raises:
325 376
            NotAllowedError: Operation not permitted
377
            
326 378
            NameError: Container/object does not exist
379
            
327 380
            IndexError: Version does not exist
328 381
        """
329 382
        return 0, []
......
333 386
        
334 387
        Parameters:
335 388
            'dest_meta': Dictionary with metadata to change
389
            
336 390
            'replace_meta': Replace metadata instead of update
391
            
337 392
            'permissions': Updated object permissions
338 393
        
339 394
        Raises:
340 395
            NotAllowedError: Operation not permitted
396
            
341 397
            NameError: Container does not exist
398
            
342 399
            ValueError: Invalid users/groups in permissions
400
            
343 401
            AttributeError: Can not set permissions
344 402
        """
345 403
        return
......
349 407
        
350 408
        Parameters:
351 409
            'dest_meta': Dictionary with metadata to change from source to destination
410
            
352 411
            'replace_meta': Replace metadata instead of update
412
            
353 413
            'permissions': New object permissions
414
            
354 415
            'src_version': Copy from the version provided
355 416
        
356 417
        Raises:
357 418
            NotAllowedError: Operation not permitted
419
            
358 420
            NameError: Container/object does not exist
421
            
359 422
            IndexError: Version does not exist
423
            
360 424
            ValueError: Invalid users/groups in permissions
425
            
361 426
            AttributeError: Can not set permissions
362 427
        """
363 428
        return
......
367 432
        
368 433
        Parameters:
369 434
            'dest_meta': Dictionary with metadata to change from source to destination
435
            
370 436
            'replace_meta': Replace metadata instead of update
437
            
371 438
            'permissions': New object permissions
372 439
        
373 440
        Raises:
374 441
            NotAllowedError: Operation not permitted
442
            
375 443
            NameError: Container/object does not exist
444
            
376 445
            ValueError: Invalid users/groups in permissions
446
            
377 447
            AttributeError: Can not set permissions
378 448
        """
379 449
        return
......
383 453
        
384 454
        Raises:
385 455
            NotAllowedError: Operation not permitted
456
            
386 457
            NameError: Container/object does not exist
387 458
        """
388 459
        return
b/pithos/backends/lib/hashfiler/__init__.py
34 34
from blocker import Blocker
35 35
from mapper import Mapper
36 36

  
37
__all__ = ["Blocker", "Mapper"]
38

  
b/pithos/backends/lib/hashfiler/blocker.py
74 74
        self.hashlen = len(emptyhash)
75 75
        self.emptyhash = emptyhash
76 76

  
77
    def get_rear_block(self, blkhash, create=0):
77
    def _get_rear_block(self, blkhash, create=0):
78 78
        filename = hexlify(blkhash)
79 79
        dir = join(self.blockpath, filename[0:2], filename[2:4], filename[4:6])
80 80
        if not exists(dir):
......
82 82
        name = join(dir, filename)
83 83
        return ContextFile(name, create)
84 84

  
85
    def check_rear_block(self, blkhash):
85
    def _check_rear_block(self, blkhash):
86 86
        filename = hexlify(blkhash)
87 87
        dir = join(self.blockpath, filename[0:2], filename[2:4], filename[4:6])
88 88
        name = join(dir, filename)
......
101 101
        missing = []
102 102
        append = missing.append
103 103
        for i, h in enumerate(hashes):
104
            if not self.check_rear_block(h):
104
            if not self._check_rear_block(h):
105 105
                append(i)
106 106
        return missing
107 107

  
......
113 113
        block = None
114 114

  
115 115
        for h in hashes:
116
            with self.get_rear_block(h, 0) as rbl:
116
            with self._get_rear_block(h, 0) as rbl:
117 117
                if not rbl:
118 118
                    break
119 119
                for block in rbl.sync_read_chunks(blocksize, 1, 0):
......
135 135
        mf = None
136 136
        missing = self.block_ping(hashlist)
137 137
        for i in missing:
138
            with self.get_rear_block(hashlist[i], 1) as rbl:
138
            with self._get_rear_block(hashlist[i], 1) as rbl:
139 139
                 rbl.sync_write(blocklist[i]) #XXX: verify?
140 140

  
141 141
        return hashlist, missing
b/pithos/backends/lib/hashfiler/mapper.py
57 57
                raise ValueError("Variable mappath '%s' is not a directory" % (mappath,))
58 58
        self.mappath = mappath
59 59

  
60
    def get_rear_map(self, name, create=0):
60
    def _get_rear_map(self, name, create=0):
61 61
        name = join(self.mappath, hex(int(name)))
62 62
        return ContextFile(name, create)
63 63

  
64
    def delete_rear_map(self, name):
64
    def _delete_rear_map(self, name):
65 65
        name = join(self.mappath, hex(int(name)))
66 66
        try:
67 67
            unlink(name)
......
79 79
        namelen = self.namelen
80 80
        hashes = ()
81 81

  
82
        with self.get_rear_map(name, 0) as rmap:
82
        with self._get_rear_map(name, 0) as rmap:
83 83
            if rmap:
84 84
                hashes = list(rmap.sync_read_chunks(namelen, nr, blkoff))
85 85
        return hashes
......
87 87
    def map_stor(self, name, hashes=(), blkoff=0, create=1):
88 88
        """Store hashes in the given hashes map, replacing the old ones."""
89 89
        namelen = self.namelen
90
        with self.get_rear_map(name, 1) as rmap:
90
        with self._get_rear_map(name, 1) as rmap:
91 91
            rmap.sync_write_chunks(namelen, blkoff, hashes, None)
92 92

  
93 93
#     def map_copy(self, src, dst):
94 94
#         """Copy a hashes map to another one, replacing it."""
95
#         with self.get_rear_map(src, 0) as rmap:
95
#         with self._get_rear_map(src, 0) as rmap:
96 96
#             if rmap:
97 97
#                 rmap.copy_to(dst)
98 98

  
99 99
    def map_remv(self, name):
100 100
        """Remove a hashes map. Returns true if the map was found and removed."""
101
        return self.delete_rear_map(name)
101
        return self._delete_rear_map(name)
102 102

  
b/pithos/backends/simple.py
609 609
    
610 610
    @backend_method(autocommit=0)
611 611
    def put_block(self, data):
612
        """Create a block and return the hash."""
612
        """Store a block and return the hash."""
613 613
        
614 614
        logger.debug("put_block: %s", len(data))
615 615
        hashes, absent = self.blocker.block_stor((data,))

Also available in: Unified diff