Revision c4399e62
b/snf-pithos-backend/pithos/backends/lib/hashfiler/context_file.py | ||
---|---|---|
31 | 31 |
# interpreted as representing official policies, either expressed |
32 | 32 |
# or implied, of GRNET S.A. |
33 | 33 |
|
34 |
from os import SEEK_CUR, SEEK_SET, fsync, makedirs |
|
34 |
from os import SEEK_CUR, SEEK_SET, fsync, makedirs, rename
|
|
35 | 35 |
from os.path import dirname |
36 |
from errno import ENOENT, EROFS
|
|
36 |
from errno import ENOENT, ESPIPE
|
|
37 | 37 |
|
38 | 38 |
|
39 | 39 |
_zeros = '' |
40 |
fsync # get used |
|
40 | 41 |
|
41 | 42 |
|
42 | 43 |
def zeros(nr): |
... | ... | |
65 | 66 |
|
66 | 67 |
try: |
67 | 68 |
seek(offset * chunksize) |
68 |
except IOError, e: |
|
69 |
except IOError as e: |
|
70 |
if e.errno != ESPIPE: |
|
71 |
raise |
|
72 |
|
|
73 |
# substitute seeking in a non-seekable file (e.g. pipe) |
|
69 | 74 |
seek = None |
70 | 75 |
for x in xrange(offset): |
71 | 76 |
fwrite(zeros(chunksize)) |
... | ... | |
106 | 111 |
seek = openfile.seek |
107 | 112 |
try: |
108 | 113 |
seek(remains) |
109 |
except IOError, e: |
|
114 |
except IOError as e: |
|
115 |
if e.errno != ESPIPE: |
|
116 |
raise |
|
117 |
|
|
118 |
# substitute seeking in a non-seekable file (e.g. pipe) |
|
110 | 119 |
seek = None |
111 | 120 |
while 1: |
112 | 121 |
s = fread(remains) |
... | ... | |
132 | 141 |
|
133 | 142 |
|
134 | 143 |
class ContextFile(object): |
135 |
__slots__ = ("name", "fdesc", "create") |
|
136 | 144 |
|
137 |
def __init__(self, name, create=0):
|
|
145 |
def __init__(self, name, create=True, write=False, temp=''):
|
|
138 | 146 |
self.name = name |
139 | 147 |
self.fdesc = None |
140 |
self.create = create |
|
148 |
self._create = create |
|
149 |
self._write = write |
|
150 |
self._mode = 'rb+' if write else 'rb' |
|
151 |
self._temp = temp |
|
152 |
self._rename = False |
|
141 | 153 |
#self.dirty = 0 |
142 | 154 |
|
143 | 155 |
def __enter__(self): |
144 | 156 |
name = self.name |
145 | 157 |
try: |
146 |
fdesc = open(name, 'rb+')
|
|
158 |
fdesc = open(name, self._mode)
|
|
147 | 159 |
except IOError, e: |
148 |
if self.create and e.errno == ENOENT: |
|
149 |
try: |
|
150 |
fdesc = open(name, 'w+') |
|
151 |
except IOError as e: |
|
152 |
if e.errno != ENOENT: |
|
153 |
raise |
|
154 |
|
|
155 |
# ENOENT means parent dirs do not exist. |
|
156 |
# Create them. |
|
157 |
makedirs(dirname(name)) |
|
158 |
fdesc = open(name, 'w+') |
|
159 |
elif not self.create and e.errno == EROFS: |
|
160 |
fdesc = open(name, 'rb') |
|
161 |
else: |
|
160 |
if not (self._create and e.errno == ENOENT): |
|
162 | 161 |
raise |
163 | 162 |
|
163 |
# File does not exist. Create it, |
|
164 |
# optionally under a temporary filename. |
|
165 |
temp = self._temp |
|
166 |
self._rename = False |
|
167 |
if temp: |
|
168 |
name += '_' + temp |
|
169 |
self._rename = name |
|
170 |
|
|
171 |
try: |
|
172 |
fdesc = open(name, 'w+') |
|
173 |
except IOError as e: |
|
174 |
if e.errno != ENOENT: |
|
175 |
raise |
|
176 |
|
|
177 |
# ENOENT means parent dirs do not exist. |
|
178 |
# Create them. |
|
179 |
makedirs(dirname(name)) |
|
180 |
fdesc = open(name, 'w+') |
|
181 |
|
|
164 | 182 |
self.fdesc = fdesc |
165 | 183 |
return self |
166 | 184 |
|
... | ... | |
170 | 188 |
#if self.dirty: |
171 | 189 |
# fsync(fdesc.fileno()) |
172 | 190 |
fdesc.close() |
191 |
|
|
192 |
tempname = self._rename |
|
193 |
if tempname: |
|
194 |
# Temporary file needs to be moved into proper place |
|
195 |
# This is usually an atomic filesystem operation. |
|
196 |
rename(tempname, self.name) |
|
197 |
|
|
173 | 198 |
return False # propagate exceptions |
174 | 199 |
|
175 | 200 |
def seek(self, offset, whence=SEEK_SET): |
b/snf-pithos-backend/pithos/backends/lib/hashfiler/fileblocker.py | ||
---|---|---|
77 | 77 |
def _pad(self, block): |
78 | 78 |
return block + ('\x00' * (self.blocksize - len(block))) |
79 | 79 |
|
80 |
def _get_rear_block(self, blkhash, create=0):
|
|
80 |
def _get_rear_block(self, blkhash, create=False, write=False):
|
|
81 | 81 |
filename = hexlify(blkhash) |
82 |
dir = join(self.blockpath, filename[0:2], filename[2:4], filename[4:6]) |
|
83 |
if not exists(dir): |
|
84 |
makedirs(dir) |
|
85 |
name = join(dir, filename) |
|
86 |
return ContextFile(name, create) |
|
82 |
path = join(self.blockpath, |
|
83 |
filename[0:2], filename[2:4], filename[4:6], |
|
84 |
filename) |
|
85 |
return ContextFile(path, create=create, write=False) |
|
87 | 86 |
|
88 | 87 |
def _check_rear_block(self, blkhash): |
89 | 88 |
filename = hexlify(blkhash) |
90 |
dir = join(self.blockpath, filename[0:2], filename[2:4], filename[4:6]) |
|
91 |
name = join(dir, filename) |
|
92 |
return exists(name) |
|
89 |
path = join(self.blockpath, |
|
90 |
filename[0:2], filename[2:4], filename[4:6], |
|
91 |
filename) |
|
92 |
return exists(path) |
|
93 | 93 |
|
94 | 94 |
def block_hash(self, data): |
95 | 95 |
"""Hash a block of data""" |
... | ... | |
121 | 121 |
if h == self.emptyhash: |
122 | 122 |
append(self._pad('')) |
123 | 123 |
continue |
124 |
with self._get_rear_block(h, 0) as rbl:
|
|
124 |
with self._get_rear_block(h, create=False, write=False) as rbl:
|
|
125 | 125 |
if not rbl: |
126 | 126 |
break |
127 | 127 |
for block in rbl.sync_read_chunks(blocksize, 1, 0): |
... | ... | |
140 | 140 |
""" |
141 | 141 |
block_hash = self.block_hash |
142 | 142 |
hashlist = [block_hash(b) for b in blocklist] |
143 |
mf = None
|
|
144 |
missing = [i for i, h in enumerate(hashlist) if not self._check_rear_block(h)]
|
|
143 |
missing = [i for i, h in enumerate(hashlist)
|
|
144 |
if not self._check_rear_block(h)]
|
|
145 | 145 |
for i in missing: |
146 |
with self._get_rear_block(hashlist[i], 1) as rbl: |
|
147 |
rbl.sync_write(blocklist[i]) #XXX: verify? |
|
146 |
with self._get_rear_block(hashlist[i], |
|
147 |
create=True, write=False) as rbl: |
|
148 |
# ^^^^^^^^^^^ |
|
149 |
# do not overwrite if exists |
|
150 |
rbl.sync_write(blocklist[i]) # XXX: verify? |
|
148 | 151 |
|
149 | 152 |
return hashlist, missing |
150 | 153 |
|
b/snf-pithos-backend/pithos/backends/lib/hashfiler/filemapper.py | ||
---|---|---|
31 | 31 |
# interpreted as representing official policies, either expressed |
32 | 32 |
# or implied, of GRNET S.A. |
33 | 33 |
|
34 |
from os import makedirs, unlink
|
|
34 |
from os import makedirs |
|
35 | 35 |
from os.path import isdir, realpath, exists, join |
36 | 36 |
from binascii import hexlify |
37 | 37 |
|
... | ... | |
57 | 57 |
raise ValueError("Variable mappath '%s' is not a directory" % (mappath,)) |
58 | 58 |
self.mappath = mappath |
59 | 59 |
|
60 |
def _get_rear_map(self, maphash, create=0):
|
|
60 |
def _get_rear_map(self, maphash, create=True, write=False):
|
|
61 | 61 |
filename = hexlify(maphash) |
62 | 62 |
path = join(self.mappath, |
63 | 63 |
filename[0:2], filename[2:4], filename[4:6], |
64 | 64 |
filename) |
65 |
return ContextFile(path, create) |
|
65 |
return ContextFile(path, create=create, write=write)
|
|
66 | 66 |
|
67 | 67 |
def _check_rear_map(self, maphash): |
68 | 68 |
filename = hexlify(maphash) |
... | ... | |
79 | 79 |
namelen = self.namelen |
80 | 80 |
hashes = () |
81 | 81 |
|
82 |
with self._get_rear_map(maphash, 0) as rmap:
|
|
82 |
with self._get_rear_map(maphash, create=False, write=False) as rmap:
|
|
83 | 83 |
if rmap: |
84 | 84 |
hashes = list(rmap.sync_read_chunks(namelen, nr, blkoff)) |
85 | 85 |
return hashes |
86 | 86 |
|
87 |
def map_stor(self, maphash, hashes=(), blkoff=0, create=1):
|
|
87 |
def map_stor(self, maphash, hashes=(), blkoff=0, create=True):
|
|
88 | 88 |
"""Store hashes in the given hashes map.""" |
89 | 89 |
namelen = self.namelen |
90 | 90 |
if self._check_rear_map(maphash): |
91 | 91 |
return |
92 |
with self._get_rear_map(maphash, 1) as rmap:
|
|
92 |
with self._get_rear_map(maphash, create=create, write=True) as rmap:
|
|
93 | 93 |
rmap.sync_write_chunks(namelen, blkoff, hashes, None) |
94 |
|
|
94 |
# |
Also available in: Unified diff