Revision 58a6c894 pithos/backends/simple.py
b/pithos/backends/simple.py | ||
---|---|---|
52 | 52 |
Uses SQLite for storage. |
53 | 53 |
""" |
54 | 54 |
|
55 |
# TODO: Automatic/manual clean-up after a time interval. |
|
56 |
|
|
55 | 57 |
def __init__(self, db): |
56 | 58 |
self.hash_algorithm = 'sha1' |
57 | 59 |
self.block_size = 128 * 1024 # 128KB |
... | ... | |
61 | 63 |
os.makedirs(basepath) |
62 | 64 |
|
63 | 65 |
self.con = sqlite3.connect(db) |
64 |
sql = '''create table if not exists objects ( |
|
65 |
name text, tstamp text, primary key (name))''' |
|
66 |
sql = '''create table if not exists versions ( |
|
67 |
version_id integer primary key, |
|
68 |
name text, |
|
69 |
tstamp datetime default current_timestamp, |
|
70 |
size integer default 0, |
|
71 |
hide integer default 0)''' |
|
66 | 72 |
self.con.execute(sql) |
67 | 73 |
sql = '''create table if not exists metadata ( |
68 |
name text, key text, value text, primary key (name, key))''' |
|
69 |
self.con.execute(sql) |
|
70 |
sql = '''create table if not exists versions ( |
|
71 |
object_id int, version int, size int, primary key (object_id, version))''' |
|
74 |
version_id integer, key text, value text, primary key (version_id, key))''' |
|
72 | 75 |
self.con.execute(sql) |
73 | 76 |
sql = '''create table if not exists blocks ( |
74 | 77 |
block_id text, data blob, primary key (block_id))''' |
75 | 78 |
self.con.execute(sql) |
76 | 79 |
sql = '''create table if not exists hashmaps ( |
77 |
version_id int, pos int, block_id text, primary key (version_id, pos))'''
|
|
80 |
version_id integer, pos integer, block_id text, primary key (version_id, pos))'''
|
|
78 | 81 |
self.con.execute(sql) |
79 | 82 |
self.con.commit() |
80 | 83 |
|
81 |
def get_account_meta(self, account): |
|
84 |
def delete_account(self, account): |
|
85 |
"""Delete the account with the given name.""" |
|
86 |
|
|
87 |
logger.debug("delete_account: %s", account) |
|
88 |
count, bytes, tstamp = self._get_pathstats(account) |
|
89 |
if count > 0: |
|
90 |
raise IndexError('Account is not empty') |
|
91 |
self._del_path(account) # Point of no return. |
|
92 |
|
|
93 |
def get_account_meta(self, account, until=None): |
|
82 | 94 |
"""Return a dictionary with the account metadata.""" |
83 | 95 |
|
84 |
logger.debug("get_account_meta: %s", account) |
|
85 |
count, bytes = self._get_pathstats(account) |
|
96 |
logger.debug("get_account_meta: %s %s", account, until) |
|
97 |
try: |
|
98 |
version_id, mtime = self._get_accountinfo(account, until) |
|
99 |
except NameError: |
|
100 |
version_id = None |
|
101 |
count, bytes, tstamp = self._get_pathstats(account, until) |
|
102 |
if until is None: |
|
103 |
modified = tstamp |
|
104 |
else: |
|
105 |
modified = self._get_pathstats(account)[2] # Overall last modification |
|
86 | 106 |
|
87 | 107 |
# Proper count. |
88 |
sql = 'select count(name) from objects where name glob ? and not name glob ?' |
|
108 |
sql = 'select count(name) from (%s) where name glob ? and not name glob ?' |
|
109 |
sql = sql % self._sql_until(until) |
|
89 | 110 |
c = self.con.execute(sql, (account + '/*', account + '/*/*')) |
90 | 111 |
row = c.fetchone() |
91 | 112 |
count = row[0] |
92 | 113 |
|
93 |
meta = self._get_metadata(account) |
|
114 |
meta = self._get_metadata(account, version_id)
|
|
94 | 115 |
meta.update({'name': account, 'count': count, 'bytes': bytes}) |
116 |
if modified: |
|
117 |
meta.update({'modified': modified}) |
|
118 |
if until is not None: |
|
119 |
meta.update({'until_timestamp': tstamp}) |
|
95 | 120 |
return meta |
96 | 121 |
|
97 | 122 |
def update_account_meta(self, account, meta, replace=False): |
98 | 123 |
"""Update the metadata associated with the account.""" |
99 | 124 |
|
100 | 125 |
logger.debug("update_account_meta: %s %s %s", account, meta, replace) |
101 |
self._update_metadata(account, None, None, meta, replace) |
|
126 |
self._put_metadata(account, meta, replace) |
|
127 |
|
|
128 |
def list_containers(self, account, marker=None, limit=10000, until=None): |
|
129 |
"""Return a list of containers existing under an account.""" |
|
130 |
|
|
131 |
logger.debug("list_containers: %s %s %s %s", account, marker, limit, until) |
|
132 |
return self._list_objects(account, '', '/', marker, limit, False, [], until) |
|
102 | 133 |
|
103 |
def put_container(self, account, name):
|
|
134 |
def put_container(self, account, container):
|
|
104 | 135 |
"""Create a new container with the given name.""" |
105 | 136 |
|
106 |
logger.debug("put_container: %s %s", account, name)
|
|
137 |
logger.debug("put_container: %s %s", account, container)
|
|
107 | 138 |
try: |
108 |
path, link, tstamp = self._get_containerinfo(account, name)
|
|
139 |
path, version_id, mtime = self._get_containerinfo(account, container)
|
|
109 | 140 |
except NameError: |
110 |
path = os.path.join(account, name)
|
|
111 |
link = self._put_linkinfo(path)
|
|
141 |
path = os.path.join(account, container)
|
|
142 |
version_id = self._put_version(path)
|
|
112 | 143 |
else: |
113 | 144 |
raise NameError('Container already exists') |
114 |
self._update_metadata(account, name, None, None) |
|
115 | 145 |
|
116 |
def delete_container(self, account, name):
|
|
146 |
def delete_container(self, account, container):
|
|
117 | 147 |
"""Delete the container with the given name.""" |
118 | 148 |
|
119 |
logger.debug("delete_container: %s %s", account, name)
|
|
120 |
path, link, tstamp = self._get_containerinfo(account, name)
|
|
121 |
count, bytes = self._get_pathstats(path) |
|
149 |
logger.debug("delete_container: %s %s", account, container)
|
|
150 |
path, version_id, mtime = self._get_containerinfo(account, container)
|
|
151 |
count, bytes, tstamp = self._get_pathstats(path)
|
|
122 | 152 |
if count > 0: |
123 | 153 |
raise IndexError('Container is not empty') |
124 |
self._del_path(path) |
|
125 |
self._update_metadata(account, None, None, None)
|
|
154 |
self._del_path(path) # Point of no return.
|
|
155 |
self._copy_version(account, account, True, True) # New account version.
|
|
126 | 156 |
|
127 |
def get_container_meta(self, account, name):
|
|
157 |
def get_container_meta(self, account, container, until=None):
|
|
128 | 158 |
"""Return a dictionary with the container metadata.""" |
129 | 159 |
|
130 |
logger.debug("get_container_meta: %s %s", account, name) |
|
131 |
path, link, tstamp = self._get_containerinfo(account, name) |
|
132 |
count, bytes = self._get_pathstats(path) |
|
133 |
meta = self._get_metadata(path) |
|
134 |
meta.update({'name': name, 'count': count, 'bytes': bytes, 'created': tstamp}) |
|
160 |
logger.debug("get_container_meta: %s %s %s", account, container, until) |
|
161 |
|
|
162 |
path, version_id, mtime = self._get_containerinfo(account, container, until) |
|
163 |
count, bytes, tstamp = self._get_pathstats(path, until) |
|
164 |
if until is None: |
|
165 |
modified = tstamp |
|
166 |
else: |
|
167 |
modified = self._get_pathstats(account)[2] # Overall last modification |
|
168 |
|
|
169 |
meta = self._get_metadata(path, version_id) |
|
170 |
meta.update({'name': container, 'count': count, 'bytes': bytes, 'modified': modified}) |
|
171 |
if until is not None: |
|
172 |
meta.update({'until_timestamp': tstamp}) |
|
135 | 173 |
return meta |
136 | 174 |
|
137 |
def update_container_meta(self, account, name, meta, replace=False):
|
|
175 |
def update_container_meta(self, account, container, meta, replace=False):
|
|
138 | 176 |
"""Update the metadata associated with the container.""" |
139 | 177 |
|
140 |
logger.debug("update_container_meta: %s %s %s %s", account, name, meta, replace) |
|
141 |
path, link, tstamp = self._get_containerinfo(account, name) |
|
142 |
self._update_metadata(account, name, None, meta, replace) |
|
143 |
|
|
144 |
def list_containers(self, account, marker=None, limit=10000): |
|
145 |
"""Return a list of containers existing under an account.""" |
|
146 |
|
|
147 |
logger.debug("list_containers: %s %s %s", account, marker, limit) |
|
148 |
return self._list_objects(account, '', '/', marker, limit, False, []) |
|
178 |
logger.debug("update_container_meta: %s %s %s %s", account, container, meta, replace) |
|
179 |
path, version_id, mtime = self._get_containerinfo(account, container) |
|
180 |
self._put_metadata(path, meta, replace) |
|
149 | 181 |
|
150 |
def list_objects(self, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[]): |
|
182 |
def list_objects(self, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[], until=None):
|
|
151 | 183 |
"""Return a list of objects existing under a container.""" |
152 | 184 |
|
153 |
logger.debug("list_objects: %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit)
|
|
154 |
path, link, tstamp = self._get_containerinfo(account, container)
|
|
155 |
return self._list_objects(path, prefix, delimiter, marker, limit, virtual, keys) |
|
185 |
logger.debug("list_objects: %s %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit, until)
|
|
186 |
path, version_id, mtime = self._get_containerinfo(account, container, until)
|
|
187 |
return self._list_objects(path, prefix, delimiter, marker, limit, virtual, keys, until)
|
|
156 | 188 |
|
157 |
def list_object_meta(self, account, name):
|
|
189 |
def list_object_meta(self, account, container, until=None):
|
|
158 | 190 |
"""Return a list with all the container's object meta keys.""" |
159 | 191 |
|
160 |
logger.debug("list_object_meta: %s %s", account, name) |
|
161 |
path, link, tstamp = self._get_containerinfo(account, name) |
|
162 |
sql = 'select distinct key from metadata where name like ?' |
|
192 |
logger.debug("list_object_meta: %s %s %s", account, container, until) |
|
193 |
path, version_id, mtime = self._get_containerinfo(account, container, until) |
|
194 |
sql = '''select distinct m.key from (%s) o, metadata m |
|
195 |
where m.version_id = o.version_id and o.name like ?''' |
|
196 |
sql = sql % self._sql_until(until) |
|
163 | 197 |
c = self.con.execute(sql, (path + '/%',)) |
164 | 198 |
return [x[0] for x in c.fetchall()] |
165 | 199 |
|
166 |
def get_object_meta(self, account, container, name): |
|
200 |
def get_object_meta(self, account, container, name, version=None):
|
|
167 | 201 |
"""Return a dictionary with the object metadata.""" |
168 | 202 |
|
169 |
logger.debug("get_object_meta: %s %s %s", account, container, name) |
|
170 |
path, link, tstamp = self._get_containerinfo(account, container) |
|
171 |
path, link, tstamp, version, size = self._get_objectinfo(account, container, name) |
|
172 |
meta = self._get_metadata(path) |
|
173 |
meta.update({'name': name, 'bytes': size, 'version': version, 'created': tstamp}) |
|
203 |
logger.debug("get_object_meta: %s %s %s %s", account, container, name, version) |
|
204 |
path, version_id, mtime, size = self._get_objectinfo(account, container, name, version) |
|
205 |
if version is None: |
|
206 |
modified = mtime |
|
207 |
else: |
|
208 |
modified = self._get_version(path)[1] # Overall last modification |
|
209 |
|
|
210 |
meta = self._get_metadata(path, version_id) |
|
211 |
meta.update({'name': name, 'bytes': size, 'version': version_id, 'version_timestamp': mtime, 'modified': modified}) |
|
174 | 212 |
return meta |
175 | 213 |
|
176 | 214 |
def update_object_meta(self, account, container, name, meta, replace=False): |
177 | 215 |
"""Update the metadata associated with the object.""" |
178 | 216 |
|
179 | 217 |
logger.debug("update_object_meta: %s %s %s %s %s", account, container, name, meta, replace) |
180 |
path, link, tstamp = self._get_containerinfo(account, container) |
|
181 |
path, link, tstamp, version, size = self._get_objectinfo(account, container, name) |
|
182 |
if 'versioned' in meta: |
|
183 |
if meta['versioned']: |
|
184 |
if version == 0: |
|
185 |
sql = 'update versions set version = 1 where object_id = ?' |
|
186 |
self.con.execute(sql, (link,)) |
|
187 |
self.con.commit() |
|
188 |
else: |
|
189 |
if version > 0: |
|
190 |
self._del_uptoversion(link, version) |
|
191 |
sql = 'update versions set version = 0 where object_id = ?' |
|
192 |
self.con.execute(sql, (link,)) |
|
193 |
self.con.commit() |
|
194 |
del(meta['versioned']) |
|
195 |
self._update_metadata(account, container, name, meta, replace) |
|
218 |
path, version_id, mtime, size = self._get_objectinfo(account, container, name) |
|
219 |
self._put_metadata(path, meta, replace) |
|
196 | 220 |
|
197 | 221 |
def get_object_hashmap(self, account, container, name, version=None): |
198 | 222 |
"""Return the object's size and a list with partial hashes.""" |
199 | 223 |
|
200 | 224 |
logger.debug("get_object_hashmap: %s %s %s %s", account, container, name, version) |
201 |
path, link, tstamp = self._get_containerinfo(account, container) |
|
202 |
path, link, tstamp, version, size = self._get_objectinfo(account, container, name, version) |
|
203 |
|
|
204 |
sql = '''select block_id from hashmaps where version_id = |
|
205 |
(select rowid from versions where object_id = ? and version = ?) |
|
206 |
order by pos''' |
|
207 |
c = self.con.execute(sql, (link, version)) |
|
225 |
path, version_id, mtime, size = self._get_objectinfo(account, container, name, version) |
|
226 |
sql = 'select block_id from hashmaps where version_id = ? order by pos asc' |
|
227 |
c = self.con.execute(sql, (version_id,)) |
|
208 | 228 |
hashmap = [x[0] for x in c.fetchall()] |
209 | 229 |
return size, hashmap |
210 | 230 |
|
... | ... | |
212 | 232 |
"""Create/update an object with the specified size and partial hashes.""" |
213 | 233 |
|
214 | 234 |
logger.debug("update_object_hashmap: %s %s %s %s %s", account, container, name, size, hashmap) |
215 |
path, link, tstamp = self._get_containerinfo(account, container) |
|
216 |
try: |
|
217 |
path, link, tstamp, version, s = self._get_objectinfo(account, container, name) |
|
218 |
except NameError: |
|
219 |
version = 0 |
|
220 |
|
|
221 |
if version == 0: |
|
222 |
path = os.path.join(account, container, name) |
|
223 |
|
|
224 |
self._del_path(path, delmeta=False) |
|
225 |
link = self._put_linkinfo(path) |
|
226 |
else: |
|
227 |
version += 1 |
|
228 |
|
|
229 |
sql = 'insert or replace into versions (object_id, version, size) values (?, ?, ?)' |
|
230 |
version_id = self.con.execute(sql, (link, version, size)).lastrowid |
|
235 |
path = self._get_containerinfo(account, container)[0] |
|
236 |
path = os.path.join(path, name) |
|
237 |
src_version_id, dest_version_id = self._copy_version(path, path, True, False) |
|
238 |
sql = 'update versions set size = ? where version_id = ?' |
|
239 |
self.con.execute(sql, (size, dest_version_id)) |
|
240 |
# TODO: Check for block_id existence. |
|
231 | 241 |
for i in range(len(hashmap)): |
232 | 242 |
sql = 'insert or replace into hashmaps (version_id, pos, block_id) values (?, ?, ?)' |
233 |
self.con.execute(sql, (version_id, i, hashmap[i])) |
|
243 |
self.con.execute(sql, (dest_version_id, i, hashmap[i]))
|
|
234 | 244 |
self.con.commit() |
235 | 245 |
|
236 |
def copy_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False): |
|
246 |
def copy_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False, src_version=None):
|
|
237 | 247 |
"""Copy an object's data and metadata.""" |
238 | 248 |
|
239 |
logger.debug("copy_object: %s %s %s %s %s %s %s", account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta) |
|
240 |
size, hashmap = self.get_object_hashmap(account, src_container, src_name) |
|
241 |
self.update_object_hashmap(account, dest_container, dest_name, size, hashmap) |
|
242 |
if not replace_meta: |
|
243 |
meta = self._get_metadata(os.path.join(account, src_container, src_name)) |
|
244 |
meta.update(dest_meta) |
|
249 |
logger.debug("copy_object: %s %s %s %s %s %s %s %s", account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta, src_version) |
|
250 |
if src_version is None: |
|
251 |
src_path = self._get_objectinfo(account, src_container, src_name)[0] |
|
245 | 252 |
else: |
246 |
meta = dest_meta |
|
247 |
self._update_metadata(account, dest_container, dest_name, meta, replace_meta) |
|
253 |
src_path = os.path.join(account, src_container, src_name) |
|
254 |
dest_path = self._get_containerinfo(account, dest_container)[0] |
|
255 |
dest_path = os.path.join(dest_path, dest_name) |
|
256 |
src_version_id, dest_version_id = self._copy_version(src_path, dest_path, not replace_meta, True, src_version) |
|
257 |
for k, v in dest_meta.iteritems(): |
|
258 |
sql = 'insert or replace into metadata (version_id, key, value) values (?, ?, ?)' |
|
259 |
self.con.execute(sql, (dest_version_id, k, v)) |
|
260 |
self.con.commit() |
|
248 | 261 |
|
249 |
def move_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False): |
|
262 |
def move_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}, replace_meta=False, src_version=None):
|
|
250 | 263 |
"""Move an object's data and metadata.""" |
251 | 264 |
|
252 |
logger.debug("move_object: %s %s %s %s %s %s %s", account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta)
|
|
253 |
self.copy_object(account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta) |
|
265 |
logger.debug("move_object: %s %s %s %s, %s %s %s %s", account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta, src_version)
|
|
266 |
self.copy_object(account, src_container, src_name, dest_container, dest_name, dest_meta, replace_meta, src_version)
|
|
254 | 267 |
self.delete_object(account, src_container, src_name) |
255 | 268 |
|
256 | 269 |
def delete_object(self, account, container, name): |
257 | 270 |
"""Delete an object.""" |
258 | 271 |
|
259 | 272 |
logger.debug("delete_object: %s %s %s", account, container, name) |
260 |
path, link, tstamp = self._get_containerinfo(account, container) |
|
273 |
path, version_id, mtime, size = self._get_objectinfo(account, container, name) |
|
274 |
self._put_version(path, 0, 1) |
|
275 |
|
|
276 |
def list_versions(self, account, container, name): |
|
277 |
"""Return a list of version (version_id, version_modified) tuples for an object.""" |
|
278 |
|
|
279 |
# This will even show deleted versions. |
|
261 | 280 |
path = os.path.join(account, container, name) |
262 |
link, tstamp = self._get_linkinfo(path)
|
|
263 |
self._del_path(path)
|
|
264 |
self._update_metadata(account, container, None, None)
|
|
281 |
sql = '''select distinct version_id, strftime('%s', tstamp) from versions where name = ?'''
|
|
282 |
c = self.con.execute(sql, (path,))
|
|
283 |
return [(str(x[0]), int(x[1])) for x in c.fetchall()]
|
|
265 | 284 |
|
266 | 285 |
def get_block(self, hash): |
267 | 286 |
"""Return a block's data.""" |
... | ... | |
299 | 318 |
dest_data = src_data[:offset] + data + src_data[offset + len(data):] |
300 | 319 |
return self.put_block(dest_data) |
301 | 320 |
|
302 |
def _get_linkinfo(self, path): |
|
303 |
c = self.con.execute('select rowid, tstamp from objects where name = ?', (path,)) |
|
321 |
def _sql_until(self, until=None): |
|
322 |
"""Return the sql to get the latest versions until the timestamp given.""" |
|
323 |
if until is None: |
|
324 |
until = int(time.time()) |
|
325 |
sql = '''select version_id, name, strftime('%s', tstamp) as tstamp, size from versions v |
|
326 |
where version_id = (select max(version_id) from versions |
|
327 |
where v.name = name and tstamp <= datetime(%s, 'unixepoch')) |
|
328 |
and hide = 0''' |
|
329 |
return sql % ('%s', until) |
|
330 |
|
|
331 |
def _get_pathstats(self, path, until=None): |
|
332 |
"""Return count and sum of size of everything under path and latest timestamp.""" |
|
333 |
|
|
334 |
sql = 'select count(version_id), total(size), max(tstamp) from (%s) where name like ?' |
|
335 |
sql = sql % self._sql_until(until) |
|
336 |
c = self.con.execute(sql, (path + '/%',)) |
|
304 | 337 |
row = c.fetchone() |
305 |
if row: |
|
306 |
return str(row[0]), str(row[1]) |
|
338 |
tstamp = row[2] if row[2] is not None else 0 |
|
339 |
return int(row[0]), int(row[1]), int(tstamp) |
|
340 |
|
|
341 |
def _get_version(self, path, version=None): |
|
342 |
if version is None: |
|
343 |
sql = '''select version_id, strftime('%s', tstamp), size, hide from versions where name = ? |
|
344 |
order by version_id desc limit 1''' |
|
345 |
c = self.con.execute(sql, (path,)) |
|
346 |
row = c.fetchone() |
|
347 |
if not row or int(row[3]): |
|
348 |
raise NameError('Object does not exist') |
|
307 | 349 |
else: |
308 |
raise NameError('Object does not exist') |
|
309 |
|
|
310 |
def _put_linkinfo(self, path): |
|
311 |
sql = 'insert into objects (name, tstamp) values (?, ?)' |
|
312 |
id = self.con.execute(sql, (path, int(time.time()))).lastrowid |
|
350 |
sql = '''select version_id, strftime('%s', tstamp), size from versions where name = ? |
|
351 |
and version_id = ?''' |
|
352 |
c = self.con.execute(sql, (path, version)) |
|
353 |
row = c.fetchone() |
|
354 |
if not row: |
|
355 |
raise IndexError('Version does not exist') |
|
356 |
return str(row[0]), int(row[1]), int(row[2]) |
|
357 |
|
|
358 |
def _put_version(self, path, size=0, hide=0): |
|
359 |
sql = 'insert into versions (name, size, hide) values (?, ?, ?)' |
|
360 |
id = self.con.execute(sql, (path, size, hide)).lastrowid |
|
313 | 361 |
self.con.commit() |
314 | 362 |
return str(id) |
315 | 363 |
|
316 |
def _get_containerinfo(self, account, container): |
|
317 |
path = os.path.join(account, container) |
|
364 |
def _copy_version(self, src_path, dest_path, copy_meta=True, copy_data=True, src_version=None): |
|
365 |
if src_version is not None: |
|
366 |
src_version_id, mtime, size = self._get_version(src_path, src_version) |
|
367 |
else: |
|
368 |
# Latest or create from scratch. |
|
369 |
try: |
|
370 |
src_version_id, mtime, size = self._get_version(src_path) |
|
371 |
except NameError: |
|
372 |
src_version_id = None |
|
373 |
size = 0 |
|
374 |
if not copy_data: |
|
375 |
size = 0 |
|
376 |
dest_version_id = self._put_version(dest_path, size) |
|
377 |
if copy_meta and src_version_id is not None: |
|
378 |
sql = 'insert into metadata select %s, key, value from metadata where version_id = ?' |
|
379 |
sql = sql % dest_version_id |
|
380 |
self.con.execute(sql, (src_version_id,)) |
|
381 |
if copy_data and src_version_id is not None: |
|
382 |
sql = 'insert into hashmaps select %s, pos, block_id from hashmaps where version_id = ?' |
|
383 |
sql = sql % dest_version_id |
|
384 |
self.con.execute(sql, (src_version_id,)) |
|
385 |
self.con.commit() |
|
386 |
return src_version_id, dest_version_id |
|
387 |
|
|
388 |
def _get_versioninfo(self, account, container, name, until=None): |
|
389 |
"""Return path, latest version, associated timestamp and size until the timestamp given.""" |
|
390 |
|
|
391 |
p = (account, container, name) |
|
318 | 392 |
try: |
319 |
link, tstamp = self._get_linkinfo(path) |
|
320 |
except NameError: |
|
393 |
p = p[:p.index(None)] |
|
394 |
except ValueError: |
|
395 |
pass |
|
396 |
path = os.path.join(*p) |
|
397 |
sql = '''select version_id, tstamp, size from (%s) where name = ?''' |
|
398 |
sql = sql % self._sql_until(until) |
|
399 |
c = self.con.execute(sql, (path,)) |
|
400 |
row = c.fetchone() |
|
401 |
if row is None: |
|
402 |
raise NameError('Path does not exist') |
|
403 |
return path, str(row[0]), int(row[1]), int(row[2]) |
|
404 |
|
|
405 |
def _get_accountinfo(self, account, until=None): |
|
406 |
try: |
|
407 |
path, version_id, mtime, size = self._get_versioninfo(account, None, None, until) |
|
408 |
return version_id, mtime |
|
409 |
except: |
|
410 |
raise NameError('Account does not exist') |
|
411 |
|
|
412 |
def _get_containerinfo(self, account, container, until=None): |
|
413 |
try: |
|
414 |
path, version_id, mtime, size = self._get_versioninfo(account, container, None, until) |
|
415 |
return path, version_id, mtime |
|
416 |
except: |
|
321 | 417 |
raise NameError('Container does not exist') |
322 |
return path, link, tstamp |
|
323 | 418 |
|
324 | 419 |
def _get_objectinfo(self, account, container, name, version=None): |
325 | 420 |
path = os.path.join(account, container, name) |
326 |
link, tstamp = self._get_linkinfo(path) |
|
327 |
if not version: # If zero or None. |
|
328 |
sql = '''select version, size from versions v, |
|
329 |
(select object_id, max(version) as m from versions |
|
330 |
where object_id = ? group by object_id) as g |
|
331 |
where v.object_id = g.object_id and v.version = g.m''' |
|
332 |
c = self.con.execute(sql, (link,)) |
|
333 |
else: |
|
334 |
sql = 'select version, size from versions where object_id = ? and version = ?' |
|
335 |
c = self.con.execute(sql, (link, version)) |
|
336 |
row = c.fetchone() |
|
337 |
if not row: |
|
338 |
raise IndexError('Version does not exist') |
|
339 |
|
|
340 |
return path, link, tstamp, int(row[0]), int(row[1]) |
|
341 |
|
|
342 |
def _get_pathstats(self, path): |
|
343 |
"""Return count and sum of size of all objects under path.""" |
|
344 |
|
|
345 |
sql = '''select count(o), total(size) from ( |
|
346 |
select v.object_id as o, v.size from versions v, |
|
347 |
(select object_id, max(version) as m from versions where object_id in |
|
348 |
(select rowid from objects where name like ?) group by object_id) as g |
|
349 |
where v.object_id = g.object_id and v.version = g.m |
|
350 |
union |
|
351 |
select rowid as o, 0 as size from objects where name like ? |
|
352 |
and rowid not in (select object_id from versions))''' |
|
353 |
c = self.con.execute(sql, (path + '/%', path + '/%')) |
|
354 |
row = c.fetchone() |
|
355 |
return int(row[0]), int(row[1]) |
|
421 |
version_id, mtime, size = self._get_version(path, version) |
|
422 |
return path, version_id, mtime, size |
|
356 | 423 |
|
357 |
def _list_objects(self, path, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[]): |
|
424 |
def _get_metadata(self, path, version): |
|
425 |
sql = 'select key, value from metadata where version_id = ?' |
|
426 |
c = self.con.execute(sql, (version,)) |
|
427 |
return dict(c.fetchall()) |
|
428 |
|
|
429 |
def _put_metadata(self, path, meta, replace=False): |
|
430 |
"""Create a new version and store metadata.""" |
|
431 |
|
|
432 |
src_version_id, dest_version_id = self._copy_version(path, path, not replace, True) |
|
433 |
for k, v in meta.iteritems(): |
|
434 |
sql = 'insert or replace into metadata (version_id, key, value) values (?, ?, ?)' |
|
435 |
self.con.execute(sql, (dest_version_id, k, v)) |
|
436 |
self.con.commit() |
|
437 |
|
|
438 |
def _list_objects(self, path, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[], until=None): |
|
358 | 439 |
cont_prefix = path + '/' |
359 | 440 |
if keys and len(keys) > 0: |
360 |
sql = '''select distinct o.name from objects o, metadata m where o.name like ? and
|
|
361 |
m.name = o.name and m.key in (%s) order by o.name'''
|
|
362 |
sql = sql % ', '.join('?' * len(keys))
|
|
441 |
sql = '''select distinct o.name, o.version_id from (%s) o, metadata m where o.name like ? and
|
|
442 |
m.version_id = o.version_id and m.key in (%s) order by o.name'''
|
|
443 |
sql = sql % (self._sql_until(until), ', '.join('?' * len(keys)))
|
|
363 | 444 |
param = (cont_prefix + prefix + '%',) + tuple(keys) |
364 | 445 |
else: |
365 |
sql = 'select name from objects where name like ? order by name' |
|
446 |
sql = 'select name, version_id from (%s) where name like ? order by name' |
|
447 |
sql = sql % self._sql_until(until) |
|
366 | 448 |
param = (cont_prefix + prefix + '%',) |
367 | 449 |
c = self.con.execute(sql, param) |
368 |
objects = [x[0][len(cont_prefix):] for x in c.fetchall()]
|
|
450 |
objects = [(x[0][len(cont_prefix):], x[1]) for x in c.fetchall()]
|
|
369 | 451 |
if delimiter: |
370 | 452 |
pseudo_objects = [] |
371 | 453 |
for x in objects: |
372 |
pseudo_name = x |
|
454 |
pseudo_name = x[0]
|
|
373 | 455 |
i = pseudo_name.find(delimiter, len(prefix)) |
374 | 456 |
if not virtual: |
375 | 457 |
# If the delimiter is not found, or the name ends |
376 | 458 |
# with the delimiter's first occurence. |
377 | 459 |
if i == -1 or len(pseudo_name) == i + len(delimiter): |
378 |
pseudo_objects.append(pseudo_name)
|
|
460 |
pseudo_objects.append(x)
|
|
379 | 461 |
else: |
380 | 462 |
# If the delimiter is found, keep up to (and including) the delimiter. |
381 | 463 |
if i != -1: |
382 | 464 |
pseudo_name = pseudo_name[:i + len(delimiter)] |
383 |
if pseudo_name not in pseudo_objects:
|
|
384 |
pseudo_objects.append(pseudo_name)
|
|
465 |
if pseudo_name not in [y[0] for y in pseudo_objects]:
|
|
466 |
pseudo_objects.append((pseudo_name, x[1]))
|
|
385 | 467 |
objects = pseudo_objects |
386 | 468 |
|
387 | 469 |
start = 0 |
388 | 470 |
if marker: |
389 | 471 |
try: |
390 |
start = objects.index(marker) + 1
|
|
472 |
start = [x[0] for x in objects].index(marker) + 1
|
|
391 | 473 |
except ValueError: |
392 | 474 |
pass |
393 | 475 |
if not limit or limit > 10000: |
394 | 476 |
limit = 10000 |
395 | 477 |
return objects[start:start + limit] |
396 | 478 |
|
397 |
def _get_metadata(self, path): |
|
398 |
sql = 'select key, value from metadata where name = ?' |
|
399 |
c = self.con.execute(sql, (path,)) |
|
400 |
return dict(c.fetchall()) |
|
401 |
|
|
402 |
def _put_metadata(self, path, meta, replace=False): |
|
403 |
if replace: |
|
404 |
sql = 'delete from metadata where name = ?' |
|
405 |
self.con.execute(sql, (path,)) |
|
406 |
for k, v in meta.iteritems(): |
|
407 |
sql = 'insert or replace into metadata (name, key, value) values (?, ?, ?)' |
|
408 |
self.con.execute(sql, (path, k, v)) |
|
409 |
self.con.commit() |
|
410 |
|
|
411 |
def _update_metadata(self, account, container, name, meta, replace=False): |
|
412 |
"""Recursively update metadata and set modification time.""" |
|
413 |
|
|
414 |
modified = {'modified': int(time.time())} |
|
415 |
if not meta: |
|
416 |
meta = {} |
|
417 |
meta.update(modified) |
|
418 |
path = (account, container, name) |
|
419 |
for x in reversed(range(3)): |
|
420 |
if not path[x]: |
|
421 |
continue |
|
422 |
self._put_metadata(os.path.join(*path[:x+1]), meta, replace) |
|
423 |
break |
|
424 |
for y in reversed(range(x)): |
|
425 |
self._put_metadata(os.path.join(*path[:y+1]), modified) |
|
426 |
|
|
427 |
def _del_uptoversion(self, link, version): |
|
428 |
sql = '''delete from hashmaps where version_id |
|
429 |
(select rowid from versions where object_id = ? and version < ?)''' |
|
430 |
self.con.execute(sql, (link, version)) |
|
431 |
self.con.execute('delete from versions where object_id = ?', (link,)) |
|
432 |
self.con.commit() |
|
433 |
|
|
434 |
def _del_path(self, path, delmeta=True): |
|
479 |
def _del_path(self, path): |
|
435 | 480 |
sql = '''delete from hashmaps where version_id in |
436 |
(select rowid from versions where object_id in |
|
437 |
(select rowid from objects where name = ?))''' |
|
481 |
(select version_id from versions where name = ?)''' |
|
482 |
self.con.execute(sql, (path,)) |
|
483 |
sql = '''delete from metadata where version_id in |
|
484 |
(select version_id from versions where name = ?)''' |
|
438 | 485 |
self.con.execute(sql, (path,)) |
439 |
sql = '''delete from versions where object_id in |
|
440 |
(select rowid from objects where name = ?)''' |
|
486 |
sql = '''delete from versions where name = ?''' |
|
441 | 487 |
self.con.execute(sql, (path,)) |
442 |
self.con.execute('delete from objects where name = ?', (path,)) |
|
443 |
if delmeta: |
|
444 |
self.con.execute('delete from metadata where name = ?', (path,)) |
|
445 | 488 |
self.con.commit() |
Also available in: Unified diff