Revision f4b61e0c
b/README | ||
---|---|---|
17 | 17 |
Running the server |
18 | 18 |
------------------ |
19 | 19 |
|
20 |
Enter the pithos dir and run: |
|
20 |
Enter the pithos dir, copy settings.py.dist to settings.py and change the SECRET_KEY inside. |
|
21 |
|
|
22 |
Then run: |
|
21 | 23 |
python manage.py syncdb |
22 | 24 |
python manage.py loaddata aai/fixtures/auth_test_data.json (to load sample users) |
23 | 25 |
python manage.py runserver |
b/docs/source/adminguide.rst | ||
---|---|---|
16 | 16 |
cd / |
17 | 17 |
git clone https://code.grnet.gr/git/pithos |
18 | 18 |
|
19 |
Setup the files (choose where to store data in ``settings.py``):: |
|
19 |
Setup the files (choose where to store data in ``settings.py`` and change ``SECRET_KEY``)::
|
|
20 | 20 |
|
21 | 21 |
cd /pithos/pithos |
22 | 22 |
cp settings.py.dist settings.py |
b/docs/source/backends.rst | ||
---|---|---|
2 | 2 |
======== |
3 | 3 |
|
4 | 4 |
.. toctree:: |
5 |
:maxdepth: 1
|
|
5 |
:maxdepth: 3
|
|
6 | 6 |
|
7 |
base_backend |
|
8 |
simple_backend |
|
7 |
BaseBackend |
|
8 |
----------- |
|
9 |
|
|
10 |
.. autoclass:: pithos.backends.base.BaseBackend |
|
11 |
|
|
12 |
SimpleBackend |
|
13 |
------------- |
|
14 |
|
|
15 |
.. autoclass:: pithos.backends.simple.SimpleBackend |
|
16 |
:show-inheritance: |
|
17 |
:members: |
|
18 |
:inherited-members: |
|
19 |
|
|
20 |
ModularBackend |
|
21 |
-------------- |
|
22 |
|
|
23 |
.. autoclass:: pithos.backends.modular.ModularBackend |
|
24 |
:show-inheritance: |
|
25 |
:members: |
|
26 |
:inherited-members: |
|
27 |
|
|
28 |
Node |
|
29 |
~~~~ |
|
30 |
|
|
31 |
.. automodule:: pithos.backends.lib.node |
|
32 |
:show-inheritance: |
|
33 |
:members: |
|
34 |
:undoc-members: |
|
35 |
|
|
36 |
Policy |
|
37 |
~~~~~~ |
|
38 |
|
|
39 |
.. automodule:: pithos.backends.lib.policy |
|
40 |
:show-inheritance: |
|
41 |
:members: |
|
42 |
:undoc-members: |
|
43 |
|
|
44 |
Permissions |
|
45 |
~~~~~~~~~~~ |
|
46 |
|
|
47 |
.. automodule:: pithos.backends.lib.permissions |
|
48 |
:show-inheritance: |
|
49 |
:members: |
|
50 |
:undoc-members: |
|
51 |
|
|
52 |
Hashfiler |
|
53 |
~~~~~~~~~ |
|
54 |
|
|
55 |
.. automodule:: pithos.backends.lib.hashfiler |
|
56 |
:show-inheritance: |
|
57 |
:members: |
|
58 |
:undoc-members: |
/dev/null | ||
---|---|---|
1 |
BaseBackend |
|
2 |
=========== |
|
3 |
|
|
4 |
.. automodule:: pithos.backends.base |
/dev/null | ||
---|---|---|
1 |
Client Library |
|
2 |
============== |
|
3 |
|
|
4 |
.. automodule:: pithos.lib.client |
b/docs/source/clientlib.rst | ||
---|---|---|
1 |
Client Library |
|
2 |
============== |
|
3 |
|
|
4 |
.. automodule:: tools.lib.client |
|
5 |
:show-inheritance: |
|
6 |
:members: |
|
7 |
:undoc-members: |
/dev/null | ||
---|---|---|
1 |
Clients |
|
2 |
======== |
|
3 |
|
|
4 |
.. toctree:: |
|
5 |
:maxdepth: 1 |
|
6 |
|
|
7 |
client-lib |
b/docs/source/index.rst | ||
---|---|---|
4 | 4 |
Contents: |
5 | 5 |
|
6 | 6 |
.. toctree:: |
7 |
:maxdepth: 2
|
|
7 |
:maxdepth: 3
|
|
8 | 8 |
|
9 | 9 |
devguide |
10 | 10 |
adminguide |
11 | 11 |
backends |
12 |
clients
|
|
12 |
clientlib
|
|
13 | 13 |
|
14 | 14 |
Indices and tables |
15 | 15 |
================== |
/dev/null | ||
---|---|---|
1 |
SimpleBackend |
|
2 |
============= |
|
3 |
|
|
4 |
.. automodule:: pithos.backends.simple |
b/pithos/backends/base.py | ||
---|---|---|
46 | 46 |
|
47 | 47 |
The following variables should be available: |
48 | 48 |
'hash_algorithm': Suggested is 'sha256' |
49 |
|
|
49 | 50 |
'block_size': Suggested is 4MB |
50 | 51 |
""" |
51 | 52 |
|
... | ... | |
54 | 55 |
|
55 | 56 |
Parameters: |
56 | 57 |
'marker': Start list from the next item after 'marker' |
58 |
|
|
57 | 59 |
'limit': Number of containers to return |
58 | 60 |
""" |
59 | 61 |
return [] |
... | ... | |
63 | 65 |
|
64 | 66 |
The keys returned are all user-defined, except: |
65 | 67 |
'name': The account name |
68 |
|
|
66 | 69 |
'count': The number of containers (or 0) |
70 |
|
|
67 | 71 |
'bytes': The total data size (or 0) |
72 |
|
|
68 | 73 |
'modified': Last modification timestamp (overall) |
74 |
|
|
69 | 75 |
'until_timestamp': Last modification until the timestamp provided |
70 | 76 |
|
71 | 77 |
Raises: |
... | ... | |
78 | 84 |
|
79 | 85 |
Parameters: |
80 | 86 |
'meta': Dictionary with metadata to update |
87 |
|
|
81 | 88 |
'replace': Replace instead of update |
82 | 89 |
|
83 | 90 |
Raises: |
... | ... | |
98 | 105 |
|
99 | 106 |
Raises: |
100 | 107 |
NotAllowedError: Operation not permitted |
108 |
|
|
101 | 109 |
ValueError: Invalid data in groups |
102 | 110 |
""" |
103 | 111 |
return |
... | ... | |
115 | 123 |
|
116 | 124 |
Raises: |
117 | 125 |
NotAllowedError: Operation not permitted |
126 |
|
|
118 | 127 |
IndexError: Account is not empty |
119 | 128 |
""" |
120 | 129 |
return |
... | ... | |
124 | 133 |
|
125 | 134 |
Parameters: |
126 | 135 |
'marker': Start list from the next item after 'marker' |
136 |
|
|
127 | 137 |
'limit': Number of containers to return |
138 |
|
|
128 | 139 |
'shared': Only list containers with permissions set |
140 |
|
|
129 | 141 |
|
130 | 142 |
Raises: |
131 | 143 |
NotAllowedError: Operation not permitted |
... | ... | |
137 | 149 |
|
138 | 150 |
The keys returned are all user-defined, except: |
139 | 151 |
'name': The container name |
152 |
|
|
140 | 153 |
'count': The number of objects |
154 |
|
|
141 | 155 |
'bytes': The total data size |
156 |
|
|
142 | 157 |
'modified': Last modification timestamp (overall) |
158 |
|
|
143 | 159 |
'until_timestamp': Last modification until the timestamp provided |
144 | 160 |
|
145 | 161 |
Raises: |
146 | 162 |
NotAllowedError: Operation not permitted |
163 |
|
|
147 | 164 |
NameError: Container does not exist |
148 | 165 |
""" |
149 | 166 |
return {} |
... | ... | |
153 | 170 |
|
154 | 171 |
Parameters: |
155 | 172 |
'meta': Dictionary with metadata to update |
173 |
|
|
156 | 174 |
'replace': Replace instead of update |
157 | 175 |
|
158 | 176 |
Raises: |
159 | 177 |
NotAllowedError: Operation not permitted |
178 |
|
|
160 | 179 |
NameError: Container does not exist |
161 | 180 |
""" |
162 | 181 |
return |
... | ... | |
166 | 185 |
|
167 | 186 |
The keys returned are: |
168 | 187 |
'quota': The maximum bytes allowed (default is 0 - unlimited) |
188 |
|
|
169 | 189 |
'versioning': Can be 'auto', 'manual' or 'none' (default is 'manual') |
170 | 190 |
|
171 | 191 |
Raises: |
172 | 192 |
NotAllowedError: Operation not permitted |
193 |
|
|
173 | 194 |
NameError: Container does not exist |
174 | 195 |
""" |
175 | 196 |
return {} |
... | ... | |
179 | 200 |
|
180 | 201 |
Raises: |
181 | 202 |
NotAllowedError: Operation not permitted |
203 |
|
|
182 | 204 |
NameError: Container does not exist |
205 |
|
|
183 | 206 |
ValueError: Invalid policy defined |
184 | 207 |
""" |
185 | 208 |
return |
... | ... | |
189 | 212 |
|
190 | 213 |
Raises: |
191 | 214 |
NotAllowedError: Operation not permitted |
215 |
|
|
192 | 216 |
NameError: Container already exists |
217 |
|
|
193 | 218 |
ValueError: Invalid policy defined |
194 | 219 |
""" |
195 | 220 |
return |
... | ... | |
199 | 224 |
|
200 | 225 |
Raises: |
201 | 226 |
NotAllowedError: Operation not permitted |
227 |
|
|
202 | 228 |
NameError: Container does not exist |
229 |
|
|
203 | 230 |
IndexError: Container is not empty |
204 | 231 |
""" |
205 | 232 |
return |
... | ... | |
209 | 236 |
|
210 | 237 |
Parameters: |
211 | 238 |
'prefix': List objects starting with 'prefix' |
239 |
|
|
212 | 240 |
'delimiter': Return unique names before 'delimiter' and after 'prefix' |
241 |
|
|
213 | 242 |
'marker': Start list from the next item after 'marker' |
243 |
|
|
214 | 244 |
'limit': Number of objects to return |
215 |
'virtual': If not set, the result will only include names starting\ |
|
216 |
with 'prefix' and ending without a 'delimiter' or with\ |
|
217 |
the first occurance of the 'delimiter' after 'prefix'.\ |
|
218 |
If set, the result will include all names after 'prefix',\ |
|
245 |
|
|
246 |
'virtual': If not set, the result will only include names starting |
|
247 |
with 'prefix' and ending without a 'delimiter' or with |
|
248 |
the first occurance of the 'delimiter' after 'prefix'. |
|
249 |
If set, the result will include all names after 'prefix', |
|
219 | 250 |
up to and including the 'delimiter' if it is found |
251 |
|
|
220 | 252 |
'keys': Include objects that have meta with the keys in the list |
253 |
|
|
221 | 254 |
'shared': Only list objects with permissions set |
222 | 255 |
|
223 | 256 |
Raises: |
224 | 257 |
NotAllowedError: Operation not permitted |
258 |
|
|
225 | 259 |
NameError: Container does not exist |
226 | 260 |
""" |
227 | 261 |
return [] |
... | ... | |
231 | 265 |
|
232 | 266 |
Raises: |
233 | 267 |
NotAllowedError: Operation not permitted |
268 |
|
|
234 | 269 |
NameError: Container does not exist |
235 | 270 |
""" |
236 | 271 |
return [] |
... | ... | |
240 | 275 |
|
241 | 276 |
The keys returned are all user-defined, except: |
242 | 277 |
'name': The object name |
278 |
|
|
243 | 279 |
'bytes': The total data size |
280 |
|
|
244 | 281 |
'modified': Last modification timestamp (overall) |
282 |
|
|
245 | 283 |
'modified_by': The user that committed the object (version requested) |
284 |
|
|
246 | 285 |
'version': The version identifier |
286 |
|
|
247 | 287 |
'version_timestamp': The version's modification timestamp |
248 | 288 |
|
249 | 289 |
Raises: |
250 | 290 |
NotAllowedError: Operation not permitted |
291 |
|
|
251 | 292 |
NameError: Container/object does not exist |
293 |
|
|
252 | 294 |
IndexError: Version does not exist |
253 | 295 |
""" |
254 | 296 |
return {} |
... | ... | |
258 | 300 |
|
259 | 301 |
Parameters: |
260 | 302 |
'meta': Dictionary with metadata to update |
303 |
|
|
261 | 304 |
'replace': Replace instead of update |
262 | 305 |
|
263 | 306 |
Raises: |
264 | 307 |
NotAllowedError: Operation not permitted |
308 |
|
|
265 | 309 |
NameError: Container/object does not exist |
266 | 310 |
""" |
267 | 311 |
return |
268 | 312 |
|
269 | 313 |
def get_object_permissions(self, user, account, container, name): |
270 |
"""Return the path from which this object gets its permissions from,\
|
|
314 |
"""Return the path from which this object gets its permissions from, |
|
271 | 315 |
along with a dictionary containing the permissions. |
272 | 316 |
|
273 | 317 |
The keys are: |
274 | 318 |
'read': The object is readable by the users/groups in the list |
319 |
|
|
275 | 320 |
'write': The object is writable by the users/groups in the list |
276 | 321 |
|
277 | 322 |
Raises: |
278 | 323 |
NotAllowedError: Operation not permitted |
324 |
|
|
279 | 325 |
NameError: Container/object does not exist |
280 | 326 |
""" |
281 | 327 |
return {} |
... | ... | |
288 | 334 |
|
289 | 335 |
Raises: |
290 | 336 |
NotAllowedError: Operation not permitted |
337 |
|
|
291 | 338 |
NameError: Container/object does not exist |
339 |
|
|
292 | 340 |
ValueError: Invalid users/groups in permissions |
293 |
AttributeError: Can not set permissions, as this object\ |
|
294 |
is already shared/private by another object higher\ |
|
295 |
in the hierarchy, or setting permissions here will\ |
|
341 |
|
|
342 |
AttributeError: Can not set permissions, as this object |
|
343 |
is already shared/private by another object higher |
|
344 |
in the hierarchy, or setting permissions here will |
|
296 | 345 |
invalidate other permissions deeper in the hierarchy |
297 | 346 |
""" |
298 | 347 |
return |
... | ... | |
302 | 351 |
|
303 | 352 |
Raises: |
304 | 353 |
NotAllowedError: Operation not permitted |
354 |
|
|
305 | 355 |
NameError: Container/object does not exist |
306 | 356 |
""" |
307 | 357 |
return None |
... | ... | |
314 | 364 |
|
315 | 365 |
Raises: |
316 | 366 |
NotAllowedError: Operation not permitted |
367 |
|
|
317 | 368 |
NameError: Container/object does not exist |
318 | 369 |
""" |
319 | 370 |
return |
... | ... | |
323 | 374 |
|
324 | 375 |
Raises: |
325 | 376 |
NotAllowedError: Operation not permitted |
377 |
|
|
326 | 378 |
NameError: Container/object does not exist |
379 |
|
|
327 | 380 |
IndexError: Version does not exist |
328 | 381 |
""" |
329 | 382 |
return 0, [] |
... | ... | |
333 | 386 |
|
334 | 387 |
Parameters: |
335 | 388 |
'dest_meta': Dictionary with metadata to change |
389 |
|
|
336 | 390 |
'replace_meta': Replace metadata instead of update |
391 |
|
|
337 | 392 |
'permissions': Updated object permissions |
338 | 393 |
|
339 | 394 |
Raises: |
340 | 395 |
NotAllowedError: Operation not permitted |
396 |
|
|
341 | 397 |
NameError: Container does not exist |
398 |
|
|
342 | 399 |
ValueError: Invalid users/groups in permissions |
400 |
|
|
343 | 401 |
AttributeError: Can not set permissions |
344 | 402 |
""" |
345 | 403 |
return |
... | ... | |
349 | 407 |
|
350 | 408 |
Parameters: |
351 | 409 |
'dest_meta': Dictionary with metadata to change from source to destination |
410 |
|
|
352 | 411 |
'replace_meta': Replace metadata instead of update |
412 |
|
|
353 | 413 |
'permissions': New object permissions |
414 |
|
|
354 | 415 |
'src_version': Copy from the version provided |
355 | 416 |
|
356 | 417 |
Raises: |
357 | 418 |
NotAllowedError: Operation not permitted |
419 |
|
|
358 | 420 |
NameError: Container/object does not exist |
421 |
|
|
359 | 422 |
IndexError: Version does not exist |
423 |
|
|
360 | 424 |
ValueError: Invalid users/groups in permissions |
425 |
|
|
361 | 426 |
AttributeError: Can not set permissions |
362 | 427 |
""" |
363 | 428 |
return |
... | ... | |
367 | 432 |
|
368 | 433 |
Parameters: |
369 | 434 |
'dest_meta': Dictionary with metadata to change from source to destination |
435 |
|
|
370 | 436 |
'replace_meta': Replace metadata instead of update |
437 |
|
|
371 | 438 |
'permissions': New object permissions |
372 | 439 |
|
373 | 440 |
Raises: |
374 | 441 |
NotAllowedError: Operation not permitted |
442 |
|
|
375 | 443 |
NameError: Container/object does not exist |
444 |
|
|
376 | 445 |
ValueError: Invalid users/groups in permissions |
446 |
|
|
377 | 447 |
AttributeError: Can not set permissions |
378 | 448 |
""" |
379 | 449 |
return |
... | ... | |
383 | 453 |
|
384 | 454 |
Raises: |
385 | 455 |
NotAllowedError: Operation not permitted |
456 |
|
|
386 | 457 |
NameError: Container/object does not exist |
387 | 458 |
""" |
388 | 459 |
return |
b/pithos/backends/lib/hashfiler/__init__.py | ||
---|---|---|
34 | 34 |
from blocker import Blocker |
35 | 35 |
from mapper import Mapper |
36 | 36 |
|
37 |
__all__ = ["Blocker", "Mapper"] |
|
38 |
|
b/pithos/backends/lib/hashfiler/blocker.py | ||
---|---|---|
74 | 74 |
self.hashlen = len(emptyhash) |
75 | 75 |
self.emptyhash = emptyhash |
76 | 76 |
|
77 |
def get_rear_block(self, blkhash, create=0): |
|
77 |
def _get_rear_block(self, blkhash, create=0):
|
|
78 | 78 |
filename = hexlify(blkhash) |
79 | 79 |
dir = join(self.blockpath, filename[0:2], filename[2:4], filename[4:6]) |
80 | 80 |
if not exists(dir): |
... | ... | |
82 | 82 |
name = join(dir, filename) |
83 | 83 |
return ContextFile(name, create) |
84 | 84 |
|
85 |
def check_rear_block(self, blkhash): |
|
85 |
def _check_rear_block(self, blkhash):
|
|
86 | 86 |
filename = hexlify(blkhash) |
87 | 87 |
dir = join(self.blockpath, filename[0:2], filename[2:4], filename[4:6]) |
88 | 88 |
name = join(dir, filename) |
... | ... | |
101 | 101 |
missing = [] |
102 | 102 |
append = missing.append |
103 | 103 |
for i, h in enumerate(hashes): |
104 |
if not self.check_rear_block(h): |
|
104 |
if not self._check_rear_block(h):
|
|
105 | 105 |
append(i) |
106 | 106 |
return missing |
107 | 107 |
|
... | ... | |
113 | 113 |
block = None |
114 | 114 |
|
115 | 115 |
for h in hashes: |
116 |
with self.get_rear_block(h, 0) as rbl: |
|
116 |
with self._get_rear_block(h, 0) as rbl:
|
|
117 | 117 |
if not rbl: |
118 | 118 |
break |
119 | 119 |
for block in rbl.sync_read_chunks(blocksize, 1, 0): |
... | ... | |
135 | 135 |
mf = None |
136 | 136 |
missing = self.block_ping(hashlist) |
137 | 137 |
for i in missing: |
138 |
with self.get_rear_block(hashlist[i], 1) as rbl: |
|
138 |
with self._get_rear_block(hashlist[i], 1) as rbl:
|
|
139 | 139 |
rbl.sync_write(blocklist[i]) #XXX: verify? |
140 | 140 |
|
141 | 141 |
return hashlist, missing |
b/pithos/backends/lib/hashfiler/mapper.py | ||
---|---|---|
57 | 57 |
raise ValueError("Variable mappath '%s' is not a directory" % (mappath,)) |
58 | 58 |
self.mappath = mappath |
59 | 59 |
|
60 |
def get_rear_map(self, name, create=0): |
|
60 |
def _get_rear_map(self, name, create=0):
|
|
61 | 61 |
name = join(self.mappath, hex(int(name))) |
62 | 62 |
return ContextFile(name, create) |
63 | 63 |
|
64 |
def delete_rear_map(self, name): |
|
64 |
def _delete_rear_map(self, name):
|
|
65 | 65 |
name = join(self.mappath, hex(int(name))) |
66 | 66 |
try: |
67 | 67 |
unlink(name) |
... | ... | |
79 | 79 |
namelen = self.namelen |
80 | 80 |
hashes = () |
81 | 81 |
|
82 |
with self.get_rear_map(name, 0) as rmap: |
|
82 |
with self._get_rear_map(name, 0) as rmap:
|
|
83 | 83 |
if rmap: |
84 | 84 |
hashes = list(rmap.sync_read_chunks(namelen, nr, blkoff)) |
85 | 85 |
return hashes |
... | ... | |
87 | 87 |
def map_stor(self, name, hashes=(), blkoff=0, create=1): |
88 | 88 |
"""Store hashes in the given hashes map, replacing the old ones.""" |
89 | 89 |
namelen = self.namelen |
90 |
with self.get_rear_map(name, 1) as rmap: |
|
90 |
with self._get_rear_map(name, 1) as rmap:
|
|
91 | 91 |
rmap.sync_write_chunks(namelen, blkoff, hashes, None) |
92 | 92 |
|
93 | 93 |
# def map_copy(self, src, dst): |
94 | 94 |
# """Copy a hashes map to another one, replacing it.""" |
95 |
# with self.get_rear_map(src, 0) as rmap: |
|
95 |
# with self._get_rear_map(src, 0) as rmap:
|
|
96 | 96 |
# if rmap: |
97 | 97 |
# rmap.copy_to(dst) |
98 | 98 |
|
99 | 99 |
def map_remv(self, name): |
100 | 100 |
"""Remove a hashes map. Returns true if the map was found and removed.""" |
101 |
return self.delete_rear_map(name) |
|
101 |
return self._delete_rear_map(name)
|
|
102 | 102 |
|
b/pithos/backends/simple.py | ||
---|---|---|
609 | 609 |
|
610 | 610 |
@backend_method(autocommit=0) |
611 | 611 |
def put_block(self, data): |
612 |
"""Create a block and return the hash."""
|
|
612 |
"""Store a block and return the hash."""
|
|
613 | 613 |
|
614 | 614 |
logger.debug("put_block: %s", len(data)) |
615 | 615 |
hashes, absent = self.blocker.block_stor((data,)) |
Also available in: Unified diff