Revision f7ab99df
b/pithos/lib/client.py | ||
---|---|---|
34 | 34 |
self.api = api |
35 | 35 |
self.verbose = verbose or debug |
36 | 36 |
self.debug = debug |
37 |
|
|
37 |
|
|
38 | 38 |
def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None, |
39 | 39 |
blocksize=1024): |
40 | 40 |
http = HTTPConnection(self.host) |
... | ... | |
81 | 81 |
print '%s: %s' % (key.capitalize(), val) |
82 | 82 |
|
83 | 83 |
|
84 |
length = hasattr(headers, 'Content-length') \ |
|
85 |
and headers['Content-length'] or None |
|
86 |
|
|
84 |
length = resp.getheader('Content-length', None) |
|
87 | 85 |
data = resp.read(length) |
88 | 86 |
if self.debug: |
89 | 87 |
print data |
... | ... | |
92 | 90 |
if int(resp.status) in ERROR_CODES.keys(): |
93 | 91 |
raise Fault(data, int(resp.status)) |
94 | 92 |
|
93 |
#print '*', resp.status, headers, data |
|
95 | 94 |
return resp.status, headers, data |
96 |
|
|
95 |
|
|
97 | 96 |
def req(self, method, path, body=None, headers=None, format='text', |
98 | 97 |
params=None): |
99 | 98 |
full_path = '/%s/%s%s?format=%s' % (self.api, self.account, path, |
... | ... | |
131 | 130 |
print '%s: %s' % (key.capitalize(), val) |
132 | 131 |
|
133 | 132 |
|
134 |
length = hasattr(headers, 'Content-length') \ |
|
135 |
and headers['Content-length'] or None |
|
136 |
|
|
133 |
length = resp.getheader('Content-length', None) |
|
137 | 134 |
data = resp.read(length) |
138 | 135 |
if self.debug: |
139 | 136 |
print data |
... | ... | |
144 | 141 |
|
145 | 142 |
#print '*', resp.status, headers, data |
146 | 143 |
return resp.status, headers, data |
147 |
|
|
144 |
|
|
148 | 145 |
def delete(self, path, format='text'): |
149 | 146 |
return self.req('DELETE', path, format=format) |
150 |
|
|
147 |
|
|
151 | 148 |
def get(self, path, format='text', headers=None, params=None): |
152 | 149 |
return self.req('GET', path, headers=headers, format=format, |
153 | 150 |
params=params) |
154 |
|
|
151 |
|
|
155 | 152 |
def head(self, path, format='text', params=None): |
156 | 153 |
return self.req('HEAD', path, format=format, params=params) |
157 |
|
|
154 |
|
|
158 | 155 |
def post(self, path, body=None, format='text', headers=None): |
159 | 156 |
return self.req('POST', path, body, headers=headers, format=format) |
160 |
|
|
157 |
|
|
161 | 158 |
def put(self, path, body=None, format='text', headers=None): |
162 | 159 |
return self.req('PUT', path, body, headers=headers, format=format) |
163 |
|
|
160 |
|
|
164 | 161 |
def _list(self, path, detail=False, params=None, headers=None): |
165 | 162 |
format = 'json' if detail else 'text' |
166 | 163 |
status, headers, data = self.get(path, format=format, headers=headers, |
... | ... | |
170 | 167 |
else: |
171 | 168 |
data = data.strip().split('\n') |
172 | 169 |
return data |
173 |
|
|
170 |
|
|
174 | 171 |
def _get_metadata(self, path, prefix=None, params=None): |
175 | 172 |
status, headers, data = self.head(path, params=params) |
176 |
prefixlen = prefix and len(prefix) or 0
|
|
173 |
prefixlen = len(prefix) if prefix else 0
|
|
177 | 174 |
meta = {} |
178 | 175 |
for key, val in headers.items(): |
179 | 176 |
if prefix and not key.startswith(prefix): |
... | ... | |
182 | 179 |
key = key[prefixlen:] |
183 | 180 |
meta[key] = val |
184 | 181 |
return meta |
185 |
|
|
186 |
def _set_metadata(self, path, entity, **meta): |
|
187 |
headers = {} |
|
182 |
|
|
183 |
def _update_metadata(self, path, entity, **meta): |
|
184 |
""" |
|
185 |
adds new and updates the values of previously set metadata |
|
186 |
""" |
|
188 | 187 |
for key, val in meta.items(): |
189 |
http_key = 'X-%s-Meta-%s' %(entity.capitalize(), key.capitalize()) |
|
188 |
meta.pop(key) |
|
189 |
meta['X-%s-Meta-%s' %(entity.capitalize(), key.capitalize())] = val |
|
190 |
prev_meta = self._get_metadata(path) |
|
191 |
prev_meta.update(meta) |
|
192 |
headers = {} |
|
193 |
for key, val in prev_meta.items(): |
|
194 |
headers[key.capitalize()] = val |
|
195 |
self.post(path, headers=headers) |
|
196 |
|
|
197 |
def _delete_metadata(self, path, entity, meta=[]): |
|
198 |
""" |
|
199 |
delete previously set metadata |
|
200 |
""" |
|
201 |
prev_meta = self._get_metadata(path) |
|
202 |
headers = {} |
|
203 |
for key, val in prev_meta.items(): |
|
204 |
if key.split('-')[-1] in meta: |
|
205 |
continue |
|
206 |
http_key = key.capitalize() |
|
190 | 207 |
headers[http_key] = val |
191 | 208 |
self.post(path, headers=headers) |
192 |
|
|
209 |
|
|
193 | 210 |
# Storage Account Services |
194 |
|
|
211 |
|
|
195 | 212 |
def list_containers(self, detail=False, params=None, headers=None): |
196 | 213 |
return self._list('', detail, params, headers) |
197 |
|
|
214 |
|
|
198 | 215 |
def account_metadata(self, restricted=False, until=None): |
199 |
prefix = restricted and 'x-account-meta-' or None
|
|
200 |
params = until and {'until':until} or None
|
|
216 |
prefix = 'x-account-meta-' if restricted else None
|
|
217 |
params = {'until':until} if until else None
|
|
201 | 218 |
return self._get_metadata('', prefix, params=params) |
202 |
|
|
219 |
|
|
203 | 220 |
def update_account_metadata(self, **meta): |
204 |
self._set_metadata('', 'account', **meta) |
|
205 |
|
|
221 |
self._update_metadata('', 'account', **meta) |
|
222 |
|
|
223 |
def delete_account_metadata(self, meta=[]): |
|
224 |
self._delete_metadata('', 'account', meta) |
|
225 |
|
|
206 | 226 |
# Storage Container Services |
207 |
|
|
227 |
|
|
208 | 228 |
def list_objects(self, container, detail=False, params=None, headers=None): |
209 | 229 |
return self._list('/' + container, detail, params, headers) |
210 |
|
|
230 |
|
|
211 | 231 |
def create_container(self, container, headers=None): |
212 | 232 |
status, header, data = self.put('/' + container, headers=headers) |
213 | 233 |
if status == 202: |
... | ... | |
215 | 235 |
elif status != 201: |
216 | 236 |
raise Fault(data, int(status)) |
217 | 237 |
return True |
218 |
|
|
238 |
|
|
219 | 239 |
def delete_container(self, container): |
220 | 240 |
self.delete('/' + container) |
221 |
|
|
241 |
|
|
222 | 242 |
def retrieve_container_metadata(self, container, restricted=False, |
223 | 243 |
until=None): |
224 |
prefix = restricted and 'x-container-meta-' or None
|
|
225 |
params = until and {'until':until} or None
|
|
244 |
prefix = 'x-container-meta-' if restricted else None
|
|
245 |
params = {'until':until} if until else None
|
|
226 | 246 |
return self._get_metadata('/%s' % container, prefix, params=params) |
227 |
|
|
247 |
|
|
228 | 248 |
def update_container_metadata(self, container, **meta): |
229 |
self._set_metadata('/' + container, 'container', **meta) |
|
230 |
|
|
249 |
self._update_metadata('/' + container, 'container', **meta) |
|
250 |
|
|
251 |
def delete_container_metadata(self, container, meta=[]): |
|
252 |
path = '/%s' % (container) |
|
253 |
self._delete_metadata(path, 'container', meta) |
|
254 |
|
|
231 | 255 |
# Storage Object Services |
232 |
|
|
256 |
|
|
233 | 257 |
def retrieve_object(self, container, object, detail=False, headers=None, |
234 | 258 |
version=None): |
235 | 259 |
path = '/%s/%s' % (container, object) |
236 | 260 |
format = 'json' if detail else 'text' |
237 |
params = version and {'version':version} or None
|
|
261 |
params = {'version':version} if version else None
|
|
238 | 262 |
status, headers, data = self.get(path, format, headers, params) |
239 | 263 |
return data |
240 |
|
|
264 |
|
|
265 |
def create_directory_marker(self, container, object): |
|
266 |
if not object: |
|
267 |
raise Fault('Directory markers have to be nested in a container') |
|
268 |
h = {'Content-Type':'application/directory'} |
|
269 |
self.create_object(container, object, f=None, headers=h) |
|
270 |
|
|
241 | 271 |
def create_object(self, container, object, f=stdin, chunked=False, |
242 | 272 |
blocksize=1024, headers=None): |
243 | 273 |
""" |
... | ... | |
247 | 277 |
""" |
248 | 278 |
path = '/%s/%s' % (container, object) |
249 | 279 |
if not chunked and f != stdin: |
250 |
data = f and f.read() or None
|
|
280 |
data = f.read() if f else None
|
|
251 | 281 |
return self.put(path, data, headers=headers) |
252 | 282 |
else: |
253 | 283 |
return self._chunked_transfer(path, 'PUT', f, headers=headers, |
254 | 284 |
blocksize=1024) |
255 |
|
|
285 |
|
|
256 | 286 |
def update_object(self, container, object, f=stdin, chunked=False, |
257 | 287 |
blocksize=1024, headers=None): |
258 | 288 |
if not f: |
... | ... | |
264 | 294 |
else: |
265 | 295 |
self._chunked_transfer(path, 'POST', f, headers=headers, |
266 | 296 |
blocksize=1024) |
267 |
|
|
297 |
|
|
268 | 298 |
def _change_obj_location(self, src_container, src_object, dst_container, |
269 | 299 |
dst_object, remove=False, headers=None): |
270 | 300 |
path = '/%s/%s' % (dst_container, dst_object) |
... | ... | |
276 | 306 |
headers['X-Copy-From'] = '/%s/%s' % (src_container, src_object) |
277 | 307 |
headers['Content-Length'] = 0 |
278 | 308 |
self.put(path, headers=headers) |
279 |
|
|
309 |
|
|
280 | 310 |
def copy_object(self, src_container, src_object, dst_container, |
281 | 311 |
dst_object, headers=None): |
282 | 312 |
self._change_obj_location(src_container, src_object, |
283 | 313 |
dst_container, dst_object, |
284 | 314 |
headers=headers) |
285 |
|
|
315 |
|
|
286 | 316 |
def move_object(self, src_container, src_object, dst_container, |
287 | 317 |
dst_object, headers=None): |
288 | 318 |
self._change_obj_location(src_container, src_object, |
289 | 319 |
dst_container, dst_object, True, headers) |
290 |
|
|
320 |
|
|
291 | 321 |
def delete_object(self, container, object): |
292 | 322 |
self.delete('/%s/%s' % (container, object)) |
293 |
|
|
323 |
|
|
294 | 324 |
def retrieve_object_metadata(self, container, object, restricted=False, |
295 | 325 |
version=None): |
296 | 326 |
path = '/%s/%s' % (container, object) |
297 |
prefix = restricted and 'x-object-meta-' or None
|
|
298 |
params = version and {'version':version} or None
|
|
327 |
prefix = 'x-object-meta-' if restricted else None
|
|
328 |
params = {'version':version} if version else None
|
|
299 | 329 |
return self._get_metadata(path, prefix, params=params) |
300 |
|
|
330 |
|
|
301 | 331 |
def update_object_metadata(self, container, object, **meta): |
302 | 332 |
path = '/%s/%s' % (container, object) |
303 |
self._set_metadata(path, 'object', **meta) |
|
333 |
self._update_metadata(path, 'object', **meta) |
|
334 |
|
|
335 |
def delete_object_metadata(self, container, object, meta=[]): |
|
336 |
path = '/%s/%s' % (container, object) |
|
337 |
self._delete_metadata(path, 'object', meta) |
|
338 |
|
|
339 |
def trash_object(self, container, object): |
|
340 |
""" |
|
341 |
trashes an object |
|
342 |
actually resets all object metadata with trash = true |
|
343 |
""" |
|
344 |
path = '/%s/%s' % (container, object) |
|
345 |
meta = {'trash':'true'} |
|
346 |
self._update_metadata(path, 'object', **meta) |
|
347 |
|
|
348 |
def restore_object(self, container, object): |
|
349 |
""" |
|
350 |
restores a trashed object |
|
351 |
actualy just resets all object metadata except trash |
|
352 |
""" |
|
353 |
self.delete_object_metadata(container, object, ['trash']) |
|
354 |
|
b/tools/store | ||
---|---|---|
57 | 57 |
|
58 | 58 |
self.parser = parser |
59 | 59 |
self.args = args |
60 |
|
|
60 |
|
|
61 | 61 |
def add_options(self, parser): |
62 | 62 |
pass |
63 |
|
|
63 |
|
|
64 | 64 |
def execute(self, *args): |
65 | 65 |
pass |
66 | 66 |
|
... | ... | |
99 | 99 |
default=False, help='show metadata until that date') |
100 | 100 |
parser.add_option('--format', action='store', dest='format', |
101 | 101 |
default='%d/%m/%Y', help='format to parse until date') |
102 |
|
|
102 |
|
|
103 | 103 |
def execute(self, container=None): |
104 | 104 |
if container: |
105 | 105 |
self.list_objects(container) |
106 | 106 |
else: |
107 | 107 |
self.list_containers() |
108 |
|
|
108 |
|
|
109 | 109 |
def list_containers(self): |
110 | 110 |
params = {'limit':self.limit, 'marker':self.marker} |
111 | 111 |
headers = {'IF_MODIFIED_SINCE':self.if_modified_since, |
... | ... | |
117 | 117 |
|
118 | 118 |
l = self.client.list_containers(self.detail, params, headers) |
119 | 119 |
print_list(l) |
120 |
|
|
120 |
|
|
121 | 121 |
def list_objects(self, container): |
122 | 122 |
params = {'limit':self.limit, 'marker':self.marker, |
123 | 123 |
'prefix':self.prefix, 'delimiter':self.delimiter, |
... | ... | |
150 | 150 |
parser.add_option('--version', action='store', dest='version', |
151 | 151 |
default=None, help='show specific version \ |
152 | 152 |
(applies only for objects)') |
153 |
|
|
153 |
|
|
154 | 154 |
def execute(self, path=''): |
155 | 155 |
container, sep, object = path.partition('/') |
156 | 156 |
if self.until: |
... | ... | |
228 | 228 |
parser.add_option('--versionlist', action='store_true', |
229 | 229 |
dest='versionlist', default=False, |
230 | 230 |
help='get the full object version list') |
231 |
|
|
231 |
|
|
232 | 232 |
def execute(self, path): |
233 | 233 |
headers = {} |
234 | 234 |
if self.range: |
... | ... | |
255 | 255 |
f.write(data) |
256 | 256 |
f.close() |
257 | 257 |
|
258 |
@cli_command('mkdir') |
|
259 |
class PutMarker(Command): |
|
260 |
syntax = '<container>/<directory marker>' |
|
261 |
description = 'create a directory marker' |
|
262 |
|
|
263 |
def execute(self, path): |
|
264 |
container, sep, object = path.partition('/') |
|
265 |
self.client.create_directory_marker(container, object) |
|
266 |
|
|
258 | 267 |
@cli_command('put') |
259 | 268 |
class PutObject(Command): |
260 | 269 |
syntax = '<container>/<object> <path> [key=val] [...]' |
261 | 270 |
description = 'create/override object with path contents or standard input' |
262 |
|
|
271 |
|
|
263 | 272 |
def add_options(self, parser): |
264 | 273 |
parser.add_option('--chunked', action='store_true', dest='chunked', |
265 | 274 |
default=False, help='set chunked transfer mode') |
... | ... | |
274 | 283 |
parser.add_option('--manifest', action='store', type='str', |
275 | 284 |
dest='manifest', default=None, |
276 | 285 |
help='use for large file support') |
286 |
parser.add_option('--type', action='store', |
|
287 |
dest='content-type', default=False, |
|
288 |
help='create object with specific content type') |
|
277 | 289 |
parser.add_option('--touch', action='store_true', |
278 | 290 |
dest='touch', default=False, |
279 | 291 |
help='create object with zero data') |
280 |
|
|
292 |
|
|
281 | 293 |
def execute(self, path, srcpath='-', *args): |
282 | 294 |
headers = {} |
283 | 295 |
if self.manifest: |
284 | 296 |
headers['X_OBJECT_MANIFEST'] = self.manifest |
285 | 297 |
|
286 | 298 |
attrs = ['etag', 'content-encoding', 'content-disposition'] |
299 |
|
|
300 |
attrs = ['etag', 'content-encoding', 'content-disposition', |
|
301 |
'content-type'] |
|
287 | 302 |
attrs = [a for a in attrs if getattr(self, a)] |
288 | 303 |
for a in attrs: |
289 | 304 |
headers[a.replace('-', '_').upper()] = getattr(self, a) |
... | ... | |
314 | 329 |
parser.add_option('--version', action='store', |
315 | 330 |
dest='version', default=False, |
316 | 331 |
help='copy specific version') |
317 |
|
|
332 |
|
|
318 | 333 |
def execute(self, src, dst): |
319 | 334 |
src_container, sep, src_object = src.partition('/') |
320 | 335 |
dst_container, sep, dst_object = dst.partition('/') |
... | ... | |
376 | 391 |
parser.add_option('--manifest', action='store', type='str', |
377 | 392 |
dest='manifest', default=None, |
378 | 393 |
help='use for large file support') |
379 |
|
|
394 |
|
|
380 | 395 |
def execute(self, path, srcpath='-', *args): |
381 | 396 |
headers = {} |
382 | 397 |
if self.manifest: |
... | ... | |
429 | 444 |
self.client.move_object(src_container, src_object, dst_container, |
430 | 445 |
dst_object, headers) |
431 | 446 |
|
447 |
@cli_command('remove', 'rm') |
|
448 |
class TrashObject(Command): |
|
449 |
syntax = '<container>/<object>' |
|
450 |
description = 'trashes an object' |
|
451 |
|
|
452 |
def execute(self, src): |
|
453 |
src_container, sep, src_object = src.partition('/') |
|
454 |
|
|
455 |
self.client.trash_object(src_container, src_object) |
|
456 |
|
|
457 |
@cli_command('restore') |
|
458 |
class TrashObject(Command): |
|
459 |
syntax = '<container>/<object>' |
|
460 |
description = 'trashes an object' |
|
461 |
|
|
462 |
def execute(self, src): |
|
463 |
src_container, sep, src_object = src.partition('/') |
|
464 |
|
|
465 |
self.client.restore_object(src_container, src_object) |
|
466 |
|
|
467 |
@cli_command('unset') |
|
468 |
class TrashObject(Command): |
|
469 |
syntax = '<container>/[<object>] key [key] [...]' |
|
470 |
description = 'deletes metadata info' |
|
471 |
|
|
472 |
def execute(self, path, *args): |
|
473 |
#in case of account fix the args |
|
474 |
if path.find('=') != -1: |
|
475 |
args = list(args) |
|
476 |
args.append(path) |
|
477 |
args = tuple(args) |
|
478 |
path = '' |
|
479 |
meta = [] |
|
480 |
for key in args: |
|
481 |
meta.append(key) |
|
482 |
container, sep, object = path.partition('/') |
|
483 |
if object: |
|
484 |
self.client.delete_object_metadata(container, object, meta) |
|
485 |
elif container: |
|
486 |
self.client.delete_container_metadata(container, meta) |
|
487 |
else: |
|
488 |
self.client.delete_account_metadata(meta) |
|
489 |
|
|
432 | 490 |
def print_usage(): |
433 | 491 |
cmd = Command([]) |
434 | 492 |
parser = cmd.parser |
... | ... | |
497 | 555 |
exit(1) |
498 | 556 |
except Fault, f: |
499 | 557 |
print f.status, f.data |
558 |
status = f.status and '%s ' % f.status or '' |
|
559 |
print '%s%s' % (status, f.data) |
|
500 | 560 |
|
501 | 561 |
if __name__ == '__main__': |
502 | 562 |
main() |
Also available in: Unified diff