Revision d2d5c360
| b/pithos/lib/client.py | ||
|---|---|---|
| 73 | 73 |
|
| 74 | 74 |
return resp.status, headers, data |
| 75 | 75 |
|
| 76 |
def req(self, method, path, body=None, headers=None, format='text', params=None): |
|
| 77 |
full_path = '/%s/%s%s?format=%s' % (self.api, self.account, path, format) |
|
| 76 |
def req(self, method, path, body=None, headers=None, format='text', |
|
| 77 |
params=None): |
|
| 78 |
full_path = '/%s/%s%s?format=%s' % (self.api, self.account, path, |
|
| 79 |
format) |
|
| 78 | 80 |
if params: |
| 79 | 81 |
for k,v in params.items(): |
| 80 | 82 |
if v: |
| 81 | 83 |
full_path = '%s&%s=%s' %(full_path, k, v) |
| 82 | 84 |
conn = HTTPConnection(self.host) |
| 83 | 85 |
|
| 86 |
#encode whitespace |
|
| 87 |
full_path = full_path.replace(' ', '%20')
|
|
| 84 | 88 |
|
| 85 | 89 |
kwargs = {}
|
| 86 | 90 |
kwargs['headers'] = headers or {}
|
| ... | ... | |
| 108 | 112 |
print data |
| 109 | 113 |
|
| 110 | 114 |
|
| 115 |
if data: |
|
| 116 |
assert data[-1] == '\n' |
|
| 117 |
#remove trailing enter |
|
| 118 |
data = data and data[:-1] or data |
|
| 119 |
|
|
| 111 | 120 |
return resp.status, headers, data |
| 112 | 121 |
|
| 113 | 122 |
def delete(self, path, format='text'): |
| ... | ... | |
| 117 | 126 |
return self.req('GET', path, headers=headers, format=format,
|
| 118 | 127 |
params=params) |
| 119 | 128 |
|
| 120 |
def head(self, path, format='text'): |
|
| 121 |
return self.req('HEAD', path, format=format)
|
|
| 129 |
def head(self, path, format='text', params=None):
|
|
| 130 |
return self.req('HEAD', path, format=format, params=params)
|
|
| 122 | 131 |
|
| 123 | 132 |
def post(self, path, body=None, format='text', headers=None): |
| 124 | 133 |
return self.req('POST', path, body, headers=headers, format=format)
|
| ... | ... | |
| 136 | 145 |
data = data.strip().split('\n')
|
| 137 | 146 |
return data |
| 138 | 147 |
|
| 139 |
def _get_metadata(self, path, prefix=None): |
|
| 140 |
status, headers, data = self.head(path) |
|
| 148 |
def _get_metadata(self, path, prefix=None, params=None):
|
|
| 149 |
status, headers, data = self.head(path, params=params)
|
|
| 141 | 150 |
if status == '404': |
| 142 | 151 |
return None |
| 143 | 152 |
prefixlen = prefix and len(prefix) or 0 |
| ... | ... | |
| 162 | 171 |
def list_containers(self, detail=False, params=None, headers=None): |
| 163 | 172 |
return self._list('', detail, params, headers)
|
| 164 | 173 |
|
| 165 |
def account_metadata(self, restricted=False): |
|
| 174 |
def account_metadata(self, restricted=False, until=None):
|
|
| 166 | 175 |
prefix = restricted and 'x-account-meta-' or None |
| 167 |
return self._get_metadata('', prefix)
|
|
| 176 |
params = until and {'until':until} or None
|
|
| 177 |
return self._get_metadata('', prefix, params=params)
|
|
| 168 | 178 |
|
| 169 | 179 |
def update_account_metadata(self, **meta): |
| 170 | 180 |
self._set_metadata('', 'account', **meta)
|
| ... | ... | |
| 185 | 195 |
def delete_container(self, container): |
| 186 | 196 |
self.delete('/' + container)
|
| 187 | 197 |
|
| 188 |
def retrieve_container_metadata(self, container, restricted=False): |
|
| 198 |
def retrieve_container_metadata(self, container, restricted=False, |
|
| 199 |
until=None): |
|
| 189 | 200 |
prefix = restricted and 'x-container-meta-' or None |
| 190 |
return self._get_metadata('/%s' % container, prefix)
|
|
| 201 |
params = until and {'until':until} or None
|
|
| 202 |
return self._get_metadata('/%s' % container, prefix, params=params)
|
|
| 191 | 203 |
|
| 192 | 204 |
def update_container_metadata(self, container, **meta): |
| 193 | 205 |
self._set_metadata('/' + container, 'container', **meta)
|
| ... | ... | |
| 202 | 214 |
|
| 203 | 215 |
def create_object(self, container, object, f=stdin, chunked=False, |
| 204 | 216 |
blocksize=1024, headers=None): |
| 205 |
if not f: |
|
| 206 |
return |
|
| 217 |
""" |
|
| 218 |
creates an object |
|
| 219 |
if f is None then creates a zero length object |
|
| 220 |
if f is stdin or chunked is set then performs chunked transfer |
|
| 221 |
""" |
|
| 207 | 222 |
path = '/%s/%s' % (container, object) |
| 208 | 223 |
if not chunked and f != stdin: |
| 209 |
data = f.read()
|
|
| 224 |
data = f and f.read() or None
|
|
| 210 | 225 |
self.put(path, data, headers=headers) |
| 211 | 226 |
else: |
| 212 | 227 |
self._chunked_transfer(path, 'PUT', f, headers=headers, |
| b/tools/store | ||
|---|---|---|
| 5 | 5 |
from os.path import basename |
| 6 | 6 |
from sys import argv, exit, stdin, stdout |
| 7 | 7 |
from pithos.lib.client import Client, Fault |
| 8 |
from datetime import datetime |
|
| 8 | 9 |
|
| 9 | 10 |
import json |
| 10 | 11 |
import logging |
| 11 | 12 |
import types |
| 13 |
import re |
|
| 14 |
import time as _time |
|
| 12 | 15 |
|
| 13 | 16 |
DEFAULT_HOST = 'pithos.dev.grnet.gr' |
| 14 | 17 |
DEFAULT_API = 'v1' |
| ... | ... | |
| 92 | 95 |
parser.add_option('--if-unmodified-since', action='store', type='str',
|
| 93 | 96 |
dest='if_unmodified_since', default=None, |
| 94 | 97 |
help='show output if not modified since then') |
| 98 |
parser.add_option('--until', action='store', dest='until',
|
|
| 99 |
default=False, help='show metadata until that date') |
|
| 100 |
parser.add_option('--format', action='store', dest='format',
|
|
| 101 |
default='%d/%m/%Y', help='format to parse until date') |
|
| 95 | 102 |
|
| 96 | 103 |
def execute(self, container=None): |
| 97 | 104 |
if container: |
| ... | ... | |
| 103 | 110 |
params = {'limit':self.limit, 'marker':self.marker}
|
| 104 | 111 |
headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
|
| 105 | 112 |
'IF_UNMODIFIED_SINCE':self.if_unmodified_since} |
| 113 |
|
|
| 114 |
if self.until: |
|
| 115 |
t = _time.strptime(self.until, self.format) |
|
| 116 |
params['until'] = int(_time.mktime(t)) |
|
| 117 |
|
|
| 106 | 118 |
l = self.client.list_containers(self.detail, params, headers) |
| 107 | 119 |
print_list(l) |
| 108 | 120 |
|
| ... | ... | |
| 112 | 124 |
'path':self.path, 'meta':self.meta} |
| 113 | 125 |
headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
|
| 114 | 126 |
'IF_UNMODIFIED_SINCE':self.if_unmodified_since} |
| 127 |
container, sep, object = container.partition('/')
|
|
| 128 |
if object: |
|
| 129 |
print '%s/%s is an object' %(container, object) |
|
| 130 |
return |
|
| 131 |
|
|
| 132 |
if self.until: |
|
| 133 |
t = _time.strptime(self.until, self.format) |
|
| 134 |
params['until'] = int(_time.mktime(t)) |
|
| 135 |
|
|
| 115 | 136 |
l = self.client.list_objects(container, self.detail, params, headers) |
| 116 | 137 |
print_list(l) |
| 117 | 138 |
|
| ... | ... | |
| 123 | 144 |
def add_options(self, parser): |
| 124 | 145 |
parser.add_option('-r', action='store_true', dest='restricted',
|
| 125 | 146 |
default=False, help='show only user defined metadata') |
| 147 |
parser.add_option('--until', action='store', dest='until',
|
|
| 148 |
default=False, help='show metadata until that date') |
|
| 149 |
parser.add_option('--format', action='store', dest='format',
|
|
| 150 |
default='%d/%m/%Y', help='format to parse until date') |
|
| 126 | 151 |
|
| 127 | 152 |
def execute(self, path=''): |
| 128 | 153 |
container, sep, object = path.partition('/')
|
| 154 |
if self.until: |
|
| 155 |
t = _time.strptime(self.until, self.format) |
|
| 156 |
self.until = int(_time.mktime(t)) |
|
| 129 | 157 |
if object: |
| 130 | 158 |
meta = self.client.retrieve_object_metadata(container, object, |
| 131 | 159 |
self.restricted) |
| 132 | 160 |
elif container: |
| 133 | 161 |
meta = self.client.retrieve_container_metadata(container, |
| 134 |
self.restricted) |
|
| 162 |
self.restricted, |
|
| 163 |
self.until) |
|
| 135 | 164 |
else: |
| 136 |
meta = self.client.account_metadata(self.restricted) |
|
| 165 |
meta = self.client.account_metadata(self.restricted, self.until)
|
|
| 137 | 166 |
if meta == None: |
| 138 | 167 |
print 'Entity does not exist' |
| 139 | 168 |
else: |
| ... | ... | |
| 233 | 262 |
parser.add_option('--manifest', action='store', type='str',
|
| 234 | 263 |
dest='manifest', default=None, |
| 235 | 264 |
help='use for large file support') |
| 265 |
parser.add_option('--touch', action='store_true',
|
|
| 266 |
dest='touch', default=True, |
|
| 267 |
help='create object with zero data') |
|
| 236 | 268 |
|
| 237 |
def execute(self, path, srcpath, *args): |
|
| 269 |
def execute(self, path, srcpath='-', *args):
|
|
| 238 | 270 |
headers = {}
|
| 239 | 271 |
if self.manifest: |
| 240 | 272 |
headers['X_OBJECT_MANIFEST'] = self.manifest |
| ... | ... | |
| 251 | 283 |
|
| 252 | 284 |
container, sep, object = path.partition('/')
|
| 253 | 285 |
|
| 254 |
f = srcpath != '-' and open(srcpath) or stdin |
|
| 255 |
chunked = (self.chunked or f == stdin) and True or False |
|
| 286 |
f = None |
|
| 287 |
chunked = False |
|
| 288 |
if not self.touch: |
|
| 289 |
f = srcpath != '-' and open(srcpath) or stdin |
|
| 290 |
chunked = (self.chunked or f == stdin) and True or False |
|
| 256 | 291 |
self.client.create_object(container, object, f, chunked=chunked, |
| 257 | 292 |
headers=headers) |
| 258 |
f.close() |
|
| 293 |
if f: |
|
| 294 |
f.close() |
|
| 259 | 295 |
|
| 260 | 296 |
@cli_command('copy', 'cp')
|
| 261 | 297 |
class CopyObject(Command): |
| ... | ... | |
| 379 | 415 |
header = header in d and header or 'subdir' |
| 380 | 416 |
if header and header in d: |
| 381 | 417 |
f.write('%s\n' %d.pop(header))
|
| 418 |
patterns = ['^x_(account|container|object)_meta_(\w+)$'] |
|
| 419 |
patterns.append(patterns[0].replace('_', '-'))
|
|
| 382 | 420 |
for key, val in sorted(d.items()): |
| 383 |
f.write('%s: %s\n' % (key.rjust(15), val))
|
|
| 421 |
for p in patterns: |
|
| 422 |
p = re.compile(p) |
|
| 423 |
m = p.match(key) |
|
| 424 |
if m: |
|
| 425 |
key = m.group(2) |
|
| 426 |
f.write('%s: %s\n' % (key.rjust(30), val))
|
|
| 384 | 427 |
|
| 385 | 428 |
def print_list(l, verbose=False, f=stdout): |
| 386 | 429 |
for elem in l: |
Also available in: Unified diff