root / pithos / lib / hashfiler / context_file.py @ 41ddb860
History | View | Annotate | Download (5.4 kB)
1 | 41ddb860 | Antony Chazapis | # Copyright 2011 GRNET S.A. All rights reserved.
|
---|---|---|---|
2 | 41ddb860 | Antony Chazapis | #
|
3 | 41ddb860 | Antony Chazapis | # Redistribution and use in source and binary forms, with or
|
4 | 41ddb860 | Antony Chazapis | # without modification, are permitted provided that the following
|
5 | 41ddb860 | Antony Chazapis | # conditions are met:
|
6 | 41ddb860 | Antony Chazapis | #
|
7 | 41ddb860 | Antony Chazapis | # 1. Redistributions of source code must retain the above
|
8 | 41ddb860 | Antony Chazapis | # copyright notice, this list of conditions and the following
|
9 | 41ddb860 | Antony Chazapis | # disclaimer.
|
10 | 41ddb860 | Antony Chazapis | #
|
11 | 41ddb860 | Antony Chazapis | # 2. Redistributions in binary form must reproduce the above
|
12 | 41ddb860 | Antony Chazapis | # copyright notice, this list of conditions and the following
|
13 | 41ddb860 | Antony Chazapis | # disclaimer in the documentation and/or other materials
|
14 | 41ddb860 | Antony Chazapis | # provided with the distribution.
|
15 | 41ddb860 | Antony Chazapis | #
|
16 | 41ddb860 | Antony Chazapis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 | 41ddb860 | Antony Chazapis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 | 41ddb860 | Antony Chazapis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 | 41ddb860 | Antony Chazapis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 | 41ddb860 | Antony Chazapis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 | 41ddb860 | Antony Chazapis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 | 41ddb860 | Antony Chazapis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 | 41ddb860 | Antony Chazapis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 | 41ddb860 | Antony Chazapis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 | 41ddb860 | Antony Chazapis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 | 41ddb860 | Antony Chazapis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 | 41ddb860 | Antony Chazapis | # POSSIBILITY OF SUCH DAMAGE.
|
28 | 41ddb860 | Antony Chazapis | #
|
29 | 41ddb860 | Antony Chazapis | # The views and conclusions contained in the software and
|
30 | 41ddb860 | Antony Chazapis | # documentation are those of the authors and should not be
|
31 | 41ddb860 | Antony Chazapis | # interpreted as representing official policies, either expressed
|
32 | 41ddb860 | Antony Chazapis | # or implied, of GRNET S.A.
|
33 | 41ddb860 | Antony Chazapis | |
34 | 5bd53e3b | Georgios D. Tsoukalas | from os import SEEK_CUR, SEEK_SET, fsync |
35 | 5bd53e3b | Georgios D. Tsoukalas | from errno import ENOENT |
36 | 5bd53e3b | Georgios D. Tsoukalas | |
37 | 41ddb860 | Antony Chazapis | |
38 | 5bd53e3b | Georgios D. Tsoukalas | _zeros = ''
|
39 | 5bd53e3b | Georgios D. Tsoukalas | |
40 | 41ddb860 | Antony Chazapis | |
41 | 5bd53e3b | Georgios D. Tsoukalas | def zeros(nr): |
42 | 5bd53e3b | Georgios D. Tsoukalas | global _zeros
|
43 | 5bd53e3b | Georgios D. Tsoukalas | size = len(_zeros)
|
44 | 5bd53e3b | Georgios D. Tsoukalas | if nr == size:
|
45 | 5bd53e3b | Georgios D. Tsoukalas | return _zeros
|
46 | 5bd53e3b | Georgios D. Tsoukalas | |
47 | 5bd53e3b | Georgios D. Tsoukalas | if nr > size:
|
48 | 5bd53e3b | Georgios D. Tsoukalas | _zeros += '\0' * (nr - size)
|
49 | 5bd53e3b | Georgios D. Tsoukalas | return _zeros
|
50 | 5bd53e3b | Georgios D. Tsoukalas | |
51 | 5bd53e3b | Georgios D. Tsoukalas | if nr < size:
|
52 | 5bd53e3b | Georgios D. Tsoukalas | _zeros = _zeros[:nr] |
53 | 5bd53e3b | Georgios D. Tsoukalas | return _zeros
|
54 | 5bd53e3b | Georgios D. Tsoukalas | |
55 | 41ddb860 | Antony Chazapis | |
56 | 5bd53e3b | Georgios D. Tsoukalas | def file_sync_write_chunks(openfile, chunksize, offset, chunks, size=None): |
57 | 5bd53e3b | Georgios D. Tsoukalas | """Write given chunks to the given buffered file object.
|
58 | 5bd53e3b | Georgios D. Tsoukalas | Writes never span across chunk boundaries.
|
59 | 5bd53e3b | Georgios D. Tsoukalas | If size is given stop after or pad until size bytes have been written.
|
60 | 5bd53e3b | Georgios D. Tsoukalas | """
|
61 | 5bd53e3b | Georgios D. Tsoukalas | fwrite = openfile.write |
62 | 5bd53e3b | Georgios D. Tsoukalas | seek = openfile.seek |
63 | 5bd53e3b | Georgios D. Tsoukalas | padding = 0
|
64 | 5bd53e3b | Georgios D. Tsoukalas | |
65 | 5bd53e3b | Georgios D. Tsoukalas | try:
|
66 | 5bd53e3b | Georgios D. Tsoukalas | seek(offset * chunksize) |
67 | 5bd53e3b | Georgios D. Tsoukalas | except IOError, e: |
68 | 5bd53e3b | Georgios D. Tsoukalas | seek = None
|
69 | 5bd53e3b | Georgios D. Tsoukalas | for x in xrange(offset): |
70 | 5bd53e3b | Georgios D. Tsoukalas | fwrite(zeros(chunksize)) |
71 | 5bd53e3b | Georgios D. Tsoukalas | |
72 | 5bd53e3b | Georgios D. Tsoukalas | cursize = offset * chunksize |
73 | 5bd53e3b | Georgios D. Tsoukalas | |
74 | 5bd53e3b | Georgios D. Tsoukalas | for chunk in chunks: |
75 | 5bd53e3b | Georgios D. Tsoukalas | if padding:
|
76 | 5bd53e3b | Georgios D. Tsoukalas | if seek:
|
77 | 5bd53e3b | Georgios D. Tsoukalas | seek(padding -1, SEEK_CUR)
|
78 | 5bd53e3b | Georgios D. Tsoukalas | fwrite("\x00")
|
79 | 5bd53e3b | Georgios D. Tsoukalas | else:
|
80 | 5bd53e3b | Georgios D. Tsoukalas | fwrite(buffer(zeros(chunksize), 0, padding)) |
81 | 5bd53e3b | Georgios D. Tsoukalas | if size is not None and cursize + chunksize >= size: |
82 | 5bd53e3b | Georgios D. Tsoukalas | chunk = chunk[:chunksize - (cursize - size)] |
83 | 5bd53e3b | Georgios D. Tsoukalas | fwrite(chunk) |
84 | 5bd53e3b | Georgios D. Tsoukalas | cursize += len(chunk)
|
85 | 5bd53e3b | Georgios D. Tsoukalas | break
|
86 | 5bd53e3b | Georgios D. Tsoukalas | fwrite(chunk) |
87 | 5bd53e3b | Georgios D. Tsoukalas | padding = chunksize - len(chunk)
|
88 | 5bd53e3b | Georgios D. Tsoukalas | |
89 | 5bd53e3b | Georgios D. Tsoukalas | padding = size - cursize if size is not None else 0 |
90 | 5bd53e3b | Georgios D. Tsoukalas | if padding <= 0: |
91 | 5bd53e3b | Georgios D. Tsoukalas | return
|
92 | 5bd53e3b | Georgios D. Tsoukalas | |
93 | 5bd53e3b | Georgios D. Tsoukalas | q, r = divmod(padding, chunksize)
|
94 | 5bd53e3b | Georgios D. Tsoukalas | for x in xrange(q): |
95 | 5bd53e3b | Georgios D. Tsoukalas | fwrite(zeros(chunksize)) |
96 | 5bd53e3b | Georgios D. Tsoukalas | fwrite(buffer(zeros(chunksize), 0, r)) |
97 | 5bd53e3b | Georgios D. Tsoukalas | |
98 | 5bd53e3b | Georgios D. Tsoukalas | |
99 | 5bd53e3b | Georgios D. Tsoukalas | def file_sync_read_chunks(openfile, chunksize, nr, offset=0): |
100 | 5bd53e3b | Georgios D. Tsoukalas | """Read and yield groups of chunks from a buffered file object at offset.
|
101 | 5bd53e3b | Georgios D. Tsoukalas | Reads never span accros chunksize boundaries.
|
102 | 5bd53e3b | Georgios D. Tsoukalas | """
|
103 | 5bd53e3b | Georgios D. Tsoukalas | fread = openfile.read |
104 | 5bd53e3b | Georgios D. Tsoukalas | remains = offset * chunksize |
105 | 5bd53e3b | Georgios D. Tsoukalas | seek = openfile.seek |
106 | 5bd53e3b | Georgios D. Tsoukalas | try:
|
107 | 5bd53e3b | Georgios D. Tsoukalas | seek(remains) |
108 | 5bd53e3b | Georgios D. Tsoukalas | except IOError, e: |
109 | 5bd53e3b | Georgios D. Tsoukalas | seek = None
|
110 | 5bd53e3b | Georgios D. Tsoukalas | while 1: |
111 | 5bd53e3b | Georgios D. Tsoukalas | s = fread(remains) |
112 | 5bd53e3b | Georgios D. Tsoukalas | remains -= len(s)
|
113 | 5bd53e3b | Georgios D. Tsoukalas | if remains <= 0: |
114 | 5bd53e3b | Georgios D. Tsoukalas | break
|
115 | 5bd53e3b | Georgios D. Tsoukalas | |
116 | 5bd53e3b | Georgios D. Tsoukalas | while nr:
|
117 | 5bd53e3b | Georgios D. Tsoukalas | remains = chunksize |
118 | 5bd53e3b | Georgios D. Tsoukalas | chunk = ''
|
119 | 5bd53e3b | Georgios D. Tsoukalas | while 1: |
120 | 5bd53e3b | Georgios D. Tsoukalas | s = fread(remains) |
121 | 5bd53e3b | Georgios D. Tsoukalas | if not s: |
122 | 5bd53e3b | Georgios D. Tsoukalas | if chunk:
|
123 | 5bd53e3b | Georgios D. Tsoukalas | yield chunk
|
124 | 5bd53e3b | Georgios D. Tsoukalas | return
|
125 | 5bd53e3b | Georgios D. Tsoukalas | chunk += s |
126 | 5bd53e3b | Georgios D. Tsoukalas | remains -= len(s)
|
127 | 5bd53e3b | Georgios D. Tsoukalas | if remains <= 0: |
128 | 5bd53e3b | Georgios D. Tsoukalas | break
|
129 | 5bd53e3b | Georgios D. Tsoukalas | yield chunk
|
130 | 5bd53e3b | Georgios D. Tsoukalas | nr -= 1
|
131 | 5bd53e3b | Georgios D. Tsoukalas | |
132 | 5bd53e3b | Georgios D. Tsoukalas | |
133 | 5bd53e3b | Georgios D. Tsoukalas | class ContextFile(object): |
134 | 5bd53e3b | Georgios D. Tsoukalas | __slots__ = ("name", "fdesc", "create") |
135 | 5bd53e3b | Georgios D. Tsoukalas | |
136 | 5bd53e3b | Georgios D. Tsoukalas | def __init__(self, name, create=0): |
137 | 5bd53e3b | Georgios D. Tsoukalas | self.name = name
|
138 | 5bd53e3b | Georgios D. Tsoukalas | self.fdesc = None |
139 | 5bd53e3b | Georgios D. Tsoukalas | self.create = create
|
140 | 5bd53e3b | Georgios D. Tsoukalas | #self.dirty = 0
|
141 | 5bd53e3b | Georgios D. Tsoukalas | |
142 | 5bd53e3b | Georgios D. Tsoukalas | def __enter__(self): |
143 | 5bd53e3b | Georgios D. Tsoukalas | name = self.name
|
144 | 5bd53e3b | Georgios D. Tsoukalas | try:
|
145 | 5bd53e3b | Georgios D. Tsoukalas | fdesc = open(name, 'rb+') |
146 | 5bd53e3b | Georgios D. Tsoukalas | except IOError, e: |
147 | 5bd53e3b | Georgios D. Tsoukalas | if not self.create or e.errno != ENOENT: |
148 | 5bd53e3b | Georgios D. Tsoukalas | raise
|
149 | 5bd53e3b | Georgios D. Tsoukalas | fdesc = open(name, 'w+') |
150 | 5bd53e3b | Georgios D. Tsoukalas | |
151 | 5bd53e3b | Georgios D. Tsoukalas | self.fdesc = fdesc
|
152 | 5bd53e3b | Georgios D. Tsoukalas | return self |
153 | 5bd53e3b | Georgios D. Tsoukalas | |
154 | 5bd53e3b | Georgios D. Tsoukalas | def __exit__(self, exc, arg, trace): |
155 | 5bd53e3b | Georgios D. Tsoukalas | fdesc = self.fdesc
|
156 | 5bd53e3b | Georgios D. Tsoukalas | if fdesc is not None: |
157 | 5bd53e3b | Georgios D. Tsoukalas | #if self.dirty:
|
158 | 5bd53e3b | Georgios D. Tsoukalas | # fsync(fdesc.fileno())
|
159 | 5bd53e3b | Georgios D. Tsoukalas | fdesc.close() |
160 | 5bd53e3b | Georgios D. Tsoukalas | return False # propagate exceptions |
161 | 5bd53e3b | Georgios D. Tsoukalas | |
162 | 5bd53e3b | Georgios D. Tsoukalas | def seek(self, offset, whence=SEEK_SET): |
163 | 5bd53e3b | Georgios D. Tsoukalas | return self.fdesc.seek(offset, whence) |
164 | 5bd53e3b | Georgios D. Tsoukalas | |
165 | 5bd53e3b | Georgios D. Tsoukalas | def tell(self): |
166 | 5bd53e3b | Georgios D. Tsoukalas | return self.fdesc.tell() |
167 | 5bd53e3b | Georgios D. Tsoukalas | |
168 | 5bd53e3b | Georgios D. Tsoukalas | def truncate(self, size): |
169 | 5bd53e3b | Georgios D. Tsoukalas | self.fdesc.truncate(size)
|
170 | 5bd53e3b | Georgios D. Tsoukalas | |
171 | 5bd53e3b | Georgios D. Tsoukalas | def sync_write(self, data): |
172 | 5bd53e3b | Georgios D. Tsoukalas | #self.dirty = 1
|
173 | 5bd53e3b | Georgios D. Tsoukalas | self.fdesc.write(data)
|
174 | 5bd53e3b | Georgios D. Tsoukalas | |
175 | 5bd53e3b | Georgios D. Tsoukalas | def sync_write_chunks(self, chunksize, offset, chunks, size=None): |
176 | 5bd53e3b | Georgios D. Tsoukalas | #self.dirty = 1
|
177 | 5bd53e3b | Georgios D. Tsoukalas | return file_sync_write_chunks(self.fdesc, chunksize, offset, chunks, size) |
178 | 5bd53e3b | Georgios D. Tsoukalas | |
179 | 5bd53e3b | Georgios D. Tsoukalas | def sync_read(self, size): |
180 | 5bd53e3b | Georgios D. Tsoukalas | read = self.fdesc.read
|
181 | 5bd53e3b | Georgios D. Tsoukalas | data = ''
|
182 | 5bd53e3b | Georgios D. Tsoukalas | while 1: |
183 | 5bd53e3b | Georgios D. Tsoukalas | s = read(size) |
184 | 5bd53e3b | Georgios D. Tsoukalas | if not s: |
185 | 5bd53e3b | Georgios D. Tsoukalas | break
|
186 | 5bd53e3b | Georgios D. Tsoukalas | data += s |
187 | 5bd53e3b | Georgios D. Tsoukalas | return data
|
188 | 5bd53e3b | Georgios D. Tsoukalas | |
189 | 5bd53e3b | Georgios D. Tsoukalas | def sync_read_chunks(self, chunksize, nr, offset=0): |
190 | 5bd53e3b | Georgios D. Tsoukalas | return file_sync_read_chunks(self.fdesc, chunksize, nr, offset) |