root / pithos / backends / simple.py @ 16fd4b63
History | View | Annotate | Download (13 kB)
1 |
import os |
---|---|
2 |
import time |
3 |
import sqlite3 |
4 |
import logging |
5 |
import types |
6 |
import hashlib |
7 |
import shutil |
8 |
|
9 |
from base import BaseBackend |
10 |
|
11 |
|
12 |
logger = logging.getLogger(__name__) |
13 |
|
14 |
|
15 |
class SimpleBackend(BaseBackend): |
16 |
def __init__(self, basepath): |
17 |
self.basepath = basepath
|
18 |
|
19 |
if not os.path.exists(basepath): |
20 |
os.makedirs(basepath) |
21 |
db = os.path.join(basepath, 'db')
|
22 |
self.con = sqlite3.connect(db)
|
23 |
# Create tables
|
24 |
sql = 'create table if not exists objects (name TEXT)'
|
25 |
self.con.execute(sql)
|
26 |
sql = '''create table if not exists metadata (
|
27 |
object_id int, key text, value text, primary key (object_id, key))'''
|
28 |
self.con.execute(sql)
|
29 |
self.con.commit()
|
30 |
|
31 |
def get_account_meta(self, account): |
32 |
"""Return a dictionary with the account metadata."""
|
33 |
|
34 |
logger.debug("get_account_meta: %s", account)
|
35 |
try:
|
36 |
fullname = self._get_accountinfo(account)
|
37 |
except NameError: |
38 |
return {'name': account, 'count': 0, 'bytes': 0} |
39 |
contents = os.listdir(fullname) |
40 |
count = len(contents)
|
41 |
size = 0
|
42 |
for y in (os.path.join(fullname, z) for z in contents): |
43 |
size += sum((os.path.getsize(os.path.join(y, x)) for x in os.listdir(y))) |
44 |
meta = self._get_metadata(account)
|
45 |
meta.update({'name': account, 'count': count, 'bytes': size}) |
46 |
return meta
|
47 |
|
48 |
def update_account_meta(self, account, meta): |
49 |
"""Update the metadata associated with the account."""
|
50 |
|
51 |
logger.debug("update_account_meta: %s %s", account, meta)
|
52 |
fullname = os.path.join(self.basepath, account)
|
53 |
if not os.path.exists(fullname): |
54 |
os.makedirs(fullname) |
55 |
self._update_metadata(account, None, None, meta) |
56 |
|
57 |
def create_container(self, account, name): |
58 |
"""Create a new container with the given name."""
|
59 |
|
60 |
logger.debug("create_container: %s %s", account, name)
|
61 |
fullname = os.path.join(self.basepath, account, name)
|
62 |
if not os.path.exists(fullname): |
63 |
os.makedirs(fullname) |
64 |
else:
|
65 |
raise NameError('Container already exists') |
66 |
self._update_metadata(account, name, None, None) |
67 |
|
68 |
def delete_container(self, account, name): |
69 |
"""Delete the container with the given name."""
|
70 |
|
71 |
logger.debug("delete_container: %s %s", account, name)
|
72 |
fullname = self._get_containerinfo(account, name)
|
73 |
if os.listdir(fullname):
|
74 |
raise IndexError('Container is not empty') |
75 |
os.rmdir(fullname) |
76 |
self._del_dbpath(os.path.join(account, name))
|
77 |
self._update_metadata(account, None, None, None) |
78 |
|
79 |
def get_container_meta(self, account, name): |
80 |
"""Return a dictionary with the container metadata."""
|
81 |
|
82 |
logger.debug("get_container_meta: %s %s", account, name)
|
83 |
fullname = self._get_containerinfo(account, name)
|
84 |
contents = os.listdir(fullname) |
85 |
count = len(contents)
|
86 |
size = sum((os.path.getsize(os.path.join(fullname, x)) for x in contents)) |
87 |
meta = self._get_metadata(os.path.join(account, name))
|
88 |
meta.update({'name': name, 'count': count, 'bytes': size}) |
89 |
return meta
|
90 |
|
91 |
def update_container_meta(self, account, name, meta): |
92 |
"""Update the metadata associated with the container."""
|
93 |
|
94 |
logger.debug("update_container_meta: %s %s %s", account, name, meta)
|
95 |
fullname = self._get_containerinfo(account, name)
|
96 |
self._update_metadata(account, name, None, meta) |
97 |
|
98 |
def list_containers(self, account, marker=None, limit=10000): |
99 |
"""Return a list of containers existing under an account."""
|
100 |
|
101 |
logger.debug("list_containers: %s %s %s", account, marker, limit)
|
102 |
try:
|
103 |
fullname = self._get_accountinfo(account)
|
104 |
except NameError: |
105 |
containers = [] |
106 |
containers = os.listdir(fullname) |
107 |
containers.sort() |
108 |
|
109 |
start = 0
|
110 |
if marker:
|
111 |
try:
|
112 |
start = containers.index(marker) + 1
|
113 |
except ValueError: |
114 |
pass
|
115 |
if not limit or limit > 10000: |
116 |
limit = 10000
|
117 |
return containers[start:start + limit]
|
118 |
|
119 |
def list_objects(self, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True): |
120 |
"""Return a list of objects existing under a container."""
|
121 |
|
122 |
logger.debug("list_objects: %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit)
|
123 |
fullname = self._get_containerinfo(account, container)
|
124 |
|
125 |
cont_prefix = os.path.join(account, container) + '/'
|
126 |
sql = 'select * from objects where name like ? order by name'
|
127 |
c = self.con.execute(sql, (cont_prefix + prefix + '%',)) |
128 |
objects = [x[0][len(cont_prefix):] for x in c.fetchall()] |
129 |
if delimiter:
|
130 |
pseudo_objects = [] |
131 |
for x in objects: |
132 |
pseudo_name = x |
133 |
i = pseudo_name.find(delimiter, len(prefix))
|
134 |
if not virtual: |
135 |
# If the delimiter is not found, or the name ends
|
136 |
# with the delimiter's first occurence.
|
137 |
if i == -1 or len(pseudo_name) == i + len(delimiter): |
138 |
pseudo_objects.append(pseudo_name) |
139 |
else:
|
140 |
# If the delimiter is found, keep up to (and including) the delimiter.
|
141 |
if i != -1: |
142 |
pseudo_name = pseudo_name[:i + len(delimiter)]
|
143 |
if pseudo_name not in pseudo_objects: |
144 |
pseudo_objects.append(pseudo_name) |
145 |
objects = pseudo_objects |
146 |
|
147 |
start = 0
|
148 |
if marker:
|
149 |
try:
|
150 |
start = objects.index(marker) + 1
|
151 |
except ValueError: |
152 |
pass
|
153 |
if not limit or limit > 10000: |
154 |
limit = 10000
|
155 |
return objects[start:start + limit]
|
156 |
|
157 |
def get_object_meta(self, account, container, name): |
158 |
"""Return a dictionary with the object metadata."""
|
159 |
|
160 |
logger.debug("get_object_meta: %s %s %s", account, container, name)
|
161 |
fullname = self._get_containerinfo(account, container)
|
162 |
link = self._get_linkinfo(os.path.join(account, container, name))
|
163 |
location = os.path.join(self.basepath, account, container, link)
|
164 |
size = os.path.getsize(location) |
165 |
meta = self._get_metadata(os.path.join(account, container, name))
|
166 |
meta.update({'name': name, 'bytes': size}) |
167 |
return meta
|
168 |
|
169 |
def update_object_meta(self, account, container, name, meta): |
170 |
"""Update the metadata associated with the object."""
|
171 |
|
172 |
logger.debug("update_object_meta: %s %s %s %s", account, container, name, meta)
|
173 |
fullname = self._get_containerinfo(account, container)
|
174 |
link = self._get_linkinfo(os.path.join(account, container, name))
|
175 |
self._update_metadata(account, container, name, meta)
|
176 |
|
177 |
def get_object(self, account, container, name, offset=0, length=-1): |
178 |
"""Return the object data."""
|
179 |
|
180 |
logger.debug("get_object: %s %s %s %s %s", account, container, name, offset, length)
|
181 |
fullname = self._get_containerinfo(account, container)
|
182 |
link = self._get_linkinfo(os.path.join(account, container, name))
|
183 |
location = os.path.join(self.basepath, account, container, link)
|
184 |
f = open(location, 'r') |
185 |
if offset:
|
186 |
f.seek(offset) |
187 |
data = f.read(length) |
188 |
f.close() |
189 |
return data
|
190 |
|
191 |
def update_object(self, account, container, name, data, offset=0): |
192 |
"""Create/update an object with the specified data."""
|
193 |
|
194 |
logger.debug("put_object: %s %s %s %s %s", account, container, name, data, offset)
|
195 |
fullname = self._get_containerinfo(account, container)
|
196 |
|
197 |
try:
|
198 |
link = self._get_linkinfo(os.path.join(account, container, name))
|
199 |
except NameError: |
200 |
# new object
|
201 |
link = self._put_linkinfo(os.path.join(account, container, name))
|
202 |
location = os.path.join(self.basepath, account, container, link)
|
203 |
f = open(location, 'w') |
204 |
if offset:
|
205 |
f.seek(offset) |
206 |
f.write(data) |
207 |
f.close() |
208 |
self._update_metadata(account, container, name, None) |
209 |
|
210 |
def copy_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}): |
211 |
"""Copy an object's data and metadata."""
|
212 |
|
213 |
logger.debug("copy_object: %s %s %s %s %s %s",
|
214 |
account, src_container, src_name, dest_container, dest_name, dest_meta) |
215 |
link = self._get_linkinfo(os.path.join(account, src_container, src_name))
|
216 |
src_location = os.path.join(self.basepath, account, src_container, link)
|
217 |
|
218 |
dest_fullname = self._get_containerinfo(account, dest_container)
|
219 |
try:
|
220 |
link = self._get_linkinfo(os.path.join(account, dest_container, dest_name))
|
221 |
except NameError: |
222 |
# new object
|
223 |
link = self._put_linkinfo(os.path.join(account, dest_container, dest_name))
|
224 |
dest_location = os.path.join(self.basepath, account, dest_container, link)
|
225 |
|
226 |
shutil.copyfile(src_location, dest_location) |
227 |
|
228 |
meta = self._get_metadata(os.path.join(account, src_container, src_name))
|
229 |
meta.update(dest_meta) |
230 |
self._update_metadata(account, dest_container, dest_name, meta)
|
231 |
return
|
232 |
|
233 |
def move_object(self, account, src_container, src_name, dest_container, dest_name, dest_meta={}): |
234 |
"""Move an object's data and metadata."""
|
235 |
|
236 |
logger.debug("move_object: %s %s %s %s %s %s",
|
237 |
account, src_container, src_name, dest_container, dest_name, dest_meta) |
238 |
self.copy_object(account, src_container, src_name, dest_container, dest_name, dest_meta)
|
239 |
self.delete_object(account, src_container, src_name)
|
240 |
|
241 |
def delete_object(self, account, container, name): |
242 |
"""Delete an object."""
|
243 |
|
244 |
logger.debug("delete_object: %s %s %s", account, container, name)
|
245 |
fullname = self._get_containerinfo(account, container)
|
246 |
|
247 |
# delete object data
|
248 |
link = self._get_linkinfo(os.path.join(account, container, name))
|
249 |
location = os.path.join(self.basepath, account, container, link)
|
250 |
try:
|
251 |
os.remove(location) |
252 |
except:
|
253 |
pass
|
254 |
# delete object metadata
|
255 |
self._del_dbpath(os.path.join(account, container, name))
|
256 |
self._update_metadata(account, container, None, None) |
257 |
|
258 |
def _get_accountinfo(self, account): |
259 |
path = os.path.join(self.basepath, account)
|
260 |
if not os.path.exists(path): |
261 |
raise NameError('Account does not exist') |
262 |
return path
|
263 |
|
264 |
def _get_containerinfo(self, account, container): |
265 |
path = os.path.join(self.basepath, account, container)
|
266 |
if not os.path.exists(path): |
267 |
raise NameError('Container does not exist') |
268 |
return path
|
269 |
|
270 |
def _get_linkinfo(self, path): |
271 |
c = self.con.execute('select rowid from objects where name=?', (path,)) |
272 |
row = c.fetchone() |
273 |
if row:
|
274 |
return str(row[0]) |
275 |
else:
|
276 |
raise NameError('Object does not exist') |
277 |
|
278 |
def _put_linkinfo(self, path): |
279 |
id = self.con.execute('insert into objects (name) values (?)', (path,)).lastrowid |
280 |
self.con.commit()
|
281 |
return str(id) |
282 |
|
283 |
def _get_metadata(self, path): |
284 |
sql = 'select m.key, m.value from metadata m, objects o where o.rowid = m.object_id and o.name = ?'
|
285 |
c = self.con.execute(sql, (path,))
|
286 |
return dict(c.fetchall()) |
287 |
|
288 |
def _put_metadata(self, path, meta): |
289 |
c = self.con.execute('select rowid from objects where name=?', (path,)) |
290 |
row = c.fetchone() |
291 |
if row:
|
292 |
link = str(row[0]) |
293 |
else:
|
294 |
link = self.con.execute('insert into objects (name) values (?)', (path,)).lastrowid |
295 |
for k, v in meta.iteritems(): |
296 |
sql = 'insert or replace into metadata (object_id, key, value) values (?, ?, ?)'
|
297 |
self.con.execute(sql, (link, k, v))
|
298 |
self.con.commit()
|
299 |
|
300 |
def _update_metadata(self, account, container, name, meta): |
301 |
"""Recursively update metadata and set modification time."""
|
302 |
|
303 |
modified = {'modified': int(time.time())} |
304 |
if not meta: |
305 |
meta = {} |
306 |
meta.update(modified) |
307 |
path = (account, container, name) |
308 |
for x in reversed(range(3)): |
309 |
if not path[x]: |
310 |
continue
|
311 |
self._put_metadata(os.path.join(*path[:x+1]), meta) |
312 |
break
|
313 |
for y in reversed(range(x)): |
314 |
self._put_metadata(os.path.join(*path[:y+1]), modified) |
315 |
|
316 |
def _del_dbpath(self, path): |
317 |
sql = 'delete from metadata where object_id in (select rowid from objects where name = ?)'
|
318 |
self.con.execute(sql, (path,))
|
319 |
self.con.execute('delete from objects where name = ?', (path,)) |
320 |
self.con.commit()
|
321 |
return
|