Revision c4399e62

b/snf-pithos-backend/pithos/backends/lib/hashfiler/context_file.py
31 31
# interpreted as representing official policies, either expressed
32 32
# or implied, of GRNET S.A.
33 33

  
34
from os import SEEK_CUR, SEEK_SET, fsync, makedirs
34
from os import SEEK_CUR, SEEK_SET, fsync, makedirs, rename
35 35
from os.path import dirname
36
from errno import ENOENT, EROFS
36
from errno import ENOENT, ESPIPE
37 37

  
38 38

  
39 39
_zeros = ''
40
fsync  # get used
40 41

  
41 42

  
42 43
def zeros(nr):
......
65 66

  
66 67
    try:
67 68
        seek(offset * chunksize)
68
    except IOError, e:
69
    except IOError as e:
70
        if e.errno != ESPIPE:
71
            raise
72

  
73
        # substitute seeking in a non-seekable file (e.g. pipe)
69 74
        seek = None
70 75
        for x in xrange(offset):
71 76
            fwrite(zeros(chunksize))
......
106 111
    seek = openfile.seek
107 112
    try:
108 113
        seek(remains)
109
    except IOError, e:
114
    except IOError as e:
115
        if e.errno != ESPIPE:
116
            raise
117

  
118
        # substitute seeking in a non-seekable file (e.g. pipe)
110 119
        seek = None
111 120
        while 1:
112 121
            s = fread(remains)
......
132 141

  
133 142

  
134 143
class ContextFile(object):
135
    __slots__ = ("name", "fdesc", "create")
136 144

  
137
    def __init__(self, name, create=0):
145
    def __init__(self, name, create=True, write=False, temp=''):
138 146
        self.name = name
139 147
        self.fdesc = None
140
        self.create = create
148
        self._create = create
149
        self._write = write
150
        self._mode = 'rb+' if write else 'rb'
151
        self._temp = temp
152
        self._rename = False
141 153
        #self.dirty = 0
142 154

  
143 155
    def __enter__(self):
144 156
        name = self.name
145 157
        try:
146
            fdesc = open(name, 'rb+')
158
            fdesc = open(name, self._mode)
147 159
        except IOError, e:
148
            if self.create and e.errno == ENOENT:
149
                try:
150
                    fdesc = open(name, 'w+')
151
                except IOError as e:
152
                    if e.errno != ENOENT:
153
                        raise
154

  
155
                    # ENOENT means parent dirs do not exist.
156
                    # Create them.
157
                    makedirs(dirname(name))
158
                    fdesc = open(name, 'w+')
159
            elif not self.create and e.errno == EROFS:
160
                fdesc = open(name, 'rb')
161
            else:
160
            if not (self._create and e.errno == ENOENT):
162 161
                raise
163 162

  
163
            # File does not exist. Create it,
164
            # optionally under a temporary filename.
165
            temp = self._temp
166
            self._rename = False
167
            if temp:
168
                name += '_' + temp
169
                self._rename = name
170

  
171
            try:
172
                fdesc = open(name, 'w+')
173
            except IOError as e:
174
                if e.errno != ENOENT:
175
                    raise
176

  
177
                # ENOENT means parent dirs do not exist.
178
                # Create them.
179
                makedirs(dirname(name))
180
                fdesc = open(name, 'w+')
181

  
164 182
        self.fdesc = fdesc
165 183
        return self
166 184

  
......
170 188
            #if self.dirty:
171 189
            #    fsync(fdesc.fileno())
172 190
            fdesc.close()
191

  
192
        tempname = self._rename
193
        if tempname:
194
            # Temporary file needs to be moved into proper place
195
            # This is usually an atomic filesystem operation.
196
            rename(tempname, self.name)
197

  
173 198
        return False  # propagate exceptions
174 199

  
175 200
    def seek(self, offset, whence=SEEK_SET):
b/snf-pithos-backend/pithos/backends/lib/hashfiler/fileblocker.py
77 77
    def _pad(self, block):
78 78
        return block + ('\x00' * (self.blocksize - len(block)))
79 79

  
80
    def _get_rear_block(self, blkhash, create=0):
80
    def _get_rear_block(self, blkhash, create=False, write=False):
81 81
        filename = hexlify(blkhash)
82
        dir = join(self.blockpath, filename[0:2], filename[2:4], filename[4:6])
83
        if not exists(dir):
84
            makedirs(dir)
85
        name = join(dir, filename)
86
        return ContextFile(name, create)
82
        path = join(self.blockpath,
83
                    filename[0:2], filename[2:4], filename[4:6],
84
                    filename)
85
        return ContextFile(path, create=create, write=False)
87 86

  
88 87
    def _check_rear_block(self, blkhash):
89 88
        filename = hexlify(blkhash)
90
        dir = join(self.blockpath, filename[0:2], filename[2:4], filename[4:6])
91
        name = join(dir, filename)
92
        return exists(name)
89
        path = join(self.blockpath,
90
                    filename[0:2], filename[2:4], filename[4:6],
91
                    filename)
92
        return exists(path)
93 93

  
94 94
    def block_hash(self, data):
95 95
        """Hash a block of data"""
......
121 121
            if h == self.emptyhash:
122 122
                append(self._pad(''))
123 123
                continue
124
            with self._get_rear_block(h, 0) as rbl:
124
            with self._get_rear_block(h, create=False, write=False) as rbl:
125 125
                if not rbl:
126 126
                    break
127 127
                for block in rbl.sync_read_chunks(blocksize, 1, 0):
......
140 140
        """
141 141
        block_hash = self.block_hash
142 142
        hashlist = [block_hash(b) for b in blocklist]
143
        mf = None
144
        missing = [i for i, h in enumerate(hashlist) if not self._check_rear_block(h)]
143
        missing = [i for i, h in enumerate(hashlist)
144
                   if not self._check_rear_block(h)]
145 145
        for i in missing:
146
            with self._get_rear_block(hashlist[i], 1) as rbl:
147
                 rbl.sync_write(blocklist[i]) #XXX: verify?
146
            with self._get_rear_block(hashlist[i],
147
                                      create=True, write=False) as rbl:
148
                                      #            ^^^^^^^^^^^
149
                                      # do not overwrite if exists
150
                rbl.sync_write(blocklist[i])  # XXX: verify?
148 151

  
149 152
        return hashlist, missing
150 153

  
b/snf-pithos-backend/pithos/backends/lib/hashfiler/filemapper.py
31 31
# interpreted as representing official policies, either expressed
32 32
# or implied, of GRNET S.A.
33 33

  
34
from os import makedirs, unlink
34
from os import makedirs
35 35
from os.path import isdir, realpath, exists, join
36 36
from binascii import hexlify
37 37

  
......
57 57
                raise ValueError("Variable mappath '%s' is not a directory" % (mappath,))
58 58
        self.mappath = mappath
59 59

  
60
    def _get_rear_map(self, maphash, create=0):
60
    def _get_rear_map(self, maphash, create=True, write=False):
61 61
        filename = hexlify(maphash)
62 62
        path = join(self.mappath,
63 63
                    filename[0:2], filename[2:4], filename[4:6],
64 64
                    filename)
65
        return ContextFile(path, create)
65
        return ContextFile(path, create=create, write=write)
66 66

  
67 67
    def _check_rear_map(self, maphash):
68 68
        filename = hexlify(maphash)
......
79 79
        namelen = self.namelen
80 80
        hashes = ()
81 81

  
82
        with self._get_rear_map(maphash, 0) as rmap:
82
        with self._get_rear_map(maphash, create=False, write=False) as rmap:
83 83
            if rmap:
84 84
                hashes = list(rmap.sync_read_chunks(namelen, nr, blkoff))
85 85
        return hashes
86 86

  
87
    def map_stor(self, maphash, hashes=(), blkoff=0, create=1):
87
    def map_stor(self, maphash, hashes=(), blkoff=0, create=True):
88 88
        """Store hashes in the given hashes map."""
89 89
        namelen = self.namelen
90 90
        if self._check_rear_map(maphash):
91 91
            return
92
        with self._get_rear_map(maphash, 1) as rmap:
92
        with self._get_rear_map(maphash, create=create, write=True) as rmap:
93 93
            rmap.sync_write_chunks(namelen, blkoff, hashes, None)
94

  
94
#

Also available in: Unified diff