root / snf-pithos-backend / pithos / backends / lib / hashfiler / context_object.py @ 185340e0
History | View | Annotate | Download (4.9 kB)
1 | c30635bf | Filippos Giannakos | # Copyright 2011-2012 GRNET S.A. All rights reserved.
|
---|---|---|---|
2 | c30635bf | Filippos Giannakos | #
|
3 | c30635bf | Filippos Giannakos | # Redistribution and use in source and binary forms, with or
|
4 | c30635bf | Filippos Giannakos | # without modification, are permitted provided that the following
|
5 | c30635bf | Filippos Giannakos | # conditions are met:
|
6 | c30635bf | Filippos Giannakos | #
|
7 | c30635bf | Filippos Giannakos | # 1. Redistributions of source code must retain the above
|
8 | c30635bf | Filippos Giannakos | # copyright notice, this list of conditions and the following
|
9 | c30635bf | Filippos Giannakos | # disclaimer.
|
10 | c30635bf | Filippos Giannakos | #
|
11 | c30635bf | Filippos Giannakos | # 2. Redistributions in binary form must reproduce the above
|
12 | c30635bf | Filippos Giannakos | # copyright notice, this list of conditions and the following
|
13 | c30635bf | Filippos Giannakos | # disclaimer in the documentation and/or other materials
|
14 | c30635bf | Filippos Giannakos | # provided with the distribution.
|
15 | c30635bf | Filippos Giannakos | #
|
16 | c30635bf | Filippos Giannakos | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 | c30635bf | Filippos Giannakos | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 | c30635bf | Filippos Giannakos | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 | c30635bf | Filippos Giannakos | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 | c30635bf | Filippos Giannakos | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 | c30635bf | Filippos Giannakos | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 | c30635bf | Filippos Giannakos | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 | c30635bf | Filippos Giannakos | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 | c30635bf | Filippos Giannakos | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 | c30635bf | Filippos Giannakos | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 | c30635bf | Filippos Giannakos | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 | c30635bf | Filippos Giannakos | # POSSIBILITY OF SUCH DAMAGE.
|
28 | c30635bf | Filippos Giannakos | #
|
29 | c30635bf | Filippos Giannakos | # The views and conclusions contained in the software and
|
30 | c30635bf | Filippos Giannakos | # documentation are those of the authors and should not be
|
31 | c30635bf | Filippos Giannakos | # interpreted as representing official policies, either expressed
|
32 | c30635bf | Filippos Giannakos | # or implied, of GRNET S.A.
|
33 | c30635bf | Filippos Giannakos | |
34 | c30635bf | Filippos Giannakos | from os import SEEK_CUR, SEEK_SET |
35 | 29148653 | Sofia Papagiannaki | from rados import ObjectNotFound |
36 | c30635bf | Filippos Giannakos | |
37 | c30635bf | Filippos Giannakos | _zeros = ''
|
38 | c30635bf | Filippos Giannakos | |
39 | c30635bf | Filippos Giannakos | |
40 | c30635bf | Filippos Giannakos | def zeros(nr): |
41 | c30635bf | Filippos Giannakos | global _zeros
|
42 | c30635bf | Filippos Giannakos | size = len(_zeros)
|
43 | c30635bf | Filippos Giannakos | if nr == size:
|
44 | c30635bf | Filippos Giannakos | return _zeros
|
45 | c30635bf | Filippos Giannakos | |
46 | c30635bf | Filippos Giannakos | if nr > size:
|
47 | c30635bf | Filippos Giannakos | _zeros += '\0' * (nr - size)
|
48 | c30635bf | Filippos Giannakos | return _zeros
|
49 | c30635bf | Filippos Giannakos | |
50 | c30635bf | Filippos Giannakos | if nr < size:
|
51 | c30635bf | Filippos Giannakos | _zeros = _zeros[:nr] |
52 | c30635bf | Filippos Giannakos | return _zeros
|
53 | c30635bf | Filippos Giannakos | |
54 | c30635bf | Filippos Giannakos | |
55 | c30635bf | Filippos Giannakos | def file_sync_write_chunks(radosobject, chunksize, offset, chunks, size=None): |
56 | c30635bf | Filippos Giannakos | """Write given chunks to the given buffered file object.
|
57 | c30635bf | Filippos Giannakos | Writes never span across chunk boundaries.
|
58 | c30635bf | Filippos Giannakos | If size is given stop after or pad until size bytes have been written.
|
59 | c30635bf | Filippos Giannakos | """
|
60 | c30635bf | Filippos Giannakos | padding = 0
|
61 | c30635bf | Filippos Giannakos | cursize = chunksize * offset |
62 | c30635bf | Filippos Giannakos | radosobject.seek(cursize) |
63 | c30635bf | Filippos Giannakos | for chunk in chunks: |
64 | c30635bf | Filippos Giannakos | if padding:
|
65 | c30635bf | Filippos Giannakos | radosobject.sync_write(buffer(zeros(chunksize), 0, padding)) |
66 | c30635bf | Filippos Giannakos | if size is not None and cursize + chunksize >= size: |
67 | c30635bf | Filippos Giannakos | chunk = chunk[:chunksize - (cursize - size)] |
68 | c30635bf | Filippos Giannakos | radosobject.sync_write(chunk) |
69 | c30635bf | Filippos Giannakos | cursize += len(chunk)
|
70 | c30635bf | Filippos Giannakos | break
|
71 | c30635bf | Filippos Giannakos | radosobject.sync_write(chunk) |
72 | c30635bf | Filippos Giannakos | padding = chunksize - len(chunk)
|
73 | c30635bf | Filippos Giannakos | |
74 | c30635bf | Filippos Giannakos | padding = size - cursize if size is not None else 0 |
75 | c30635bf | Filippos Giannakos | if padding <= 0: |
76 | c30635bf | Filippos Giannakos | return
|
77 | c30635bf | Filippos Giannakos | |
78 | c30635bf | Filippos Giannakos | q, r = divmod(padding, chunksize)
|
79 | c30635bf | Filippos Giannakos | for x in xrange(q): |
80 | c30635bf | Filippos Giannakos | radosobject.sunc_write(zeros(chunksize)) |
81 | c30635bf | Filippos Giannakos | radosobject.sync_write(buffer(zeros(chunksize), 0, r)) |
82 | c30635bf | Filippos Giannakos | |
83 | 29148653 | Sofia Papagiannaki | |
84 | c30635bf | Filippos Giannakos | def file_sync_read_chunks(radosobject, chunksize, nr, offset=0): |
85 | c30635bf | Filippos Giannakos | """Read and yield groups of chunks from a buffered file object at offset.
|
86 | c30635bf | Filippos Giannakos | Reads never span accros chunksize boundaries.
|
87 | c30635bf | Filippos Giannakos | """
|
88 | c30635bf | Filippos Giannakos | radosobject.seek(offset * chunksize) |
89 | c30635bf | Filippos Giannakos | while nr:
|
90 | c30635bf | Filippos Giannakos | remains = chunksize |
91 | c30635bf | Filippos Giannakos | chunk = ''
|
92 | c30635bf | Filippos Giannakos | while 1: |
93 | c30635bf | Filippos Giannakos | s = radosobject.sync_read(remains) |
94 | c30635bf | Filippos Giannakos | if not s: |
95 | c30635bf | Filippos Giannakos | if chunk:
|
96 | c30635bf | Filippos Giannakos | yield chunk
|
97 | c30635bf | Filippos Giannakos | return
|
98 | c30635bf | Filippos Giannakos | chunk += s |
99 | c30635bf | Filippos Giannakos | remains -= len(s)
|
100 | c30635bf | Filippos Giannakos | if remains <= 0: |
101 | c30635bf | Filippos Giannakos | break
|
102 | c30635bf | Filippos Giannakos | yield chunk
|
103 | c30635bf | Filippos Giannakos | nr -= 1
|
104 | c30635bf | Filippos Giannakos | |
105 | 29148653 | Sofia Papagiannaki | |
106 | c30635bf | Filippos Giannakos | class RadosObject(object): |
107 | 185340e0 | Chrysostomos Nanakos | __slots__ = ("name", "ioctx", "offset") |
108 | c30635bf | Filippos Giannakos | |
109 | 185340e0 | Chrysostomos Nanakos | def __init__(self, name, ioctx): |
110 | c30635bf | Filippos Giannakos | self.name = name
|
111 | c30635bf | Filippos Giannakos | self.ioctx = ioctx
|
112 | c30635bf | Filippos Giannakos | self.offset = 0 |
113 | c30635bf | Filippos Giannakos | #self.dirty = 0
|
114 | c30635bf | Filippos Giannakos | |
115 | c30635bf | Filippos Giannakos | def __enter__(self): |
116 | c30635bf | Filippos Giannakos | return self |
117 | c30635bf | Filippos Giannakos | |
118 | c30635bf | Filippos Giannakos | def __exit__(self, exc, arg, trace): |
119 | c30635bf | Filippos Giannakos | return False |
120 | c30635bf | Filippos Giannakos | |
121 | c30635bf | Filippos Giannakos | def seek(self, offset, whence=SEEK_SET): |
122 | c30635bf | Filippos Giannakos | if whence == SEEK_CUR:
|
123 | c30635bf | Filippos Giannakos | offset += self.offset
|
124 | c30635bf | Filippos Giannakos | self.offset = offset
|
125 | c30635bf | Filippos Giannakos | return offset
|
126 | c30635bf | Filippos Giannakos | |
127 | c30635bf | Filippos Giannakos | def tell(self): |
128 | c30635bf | Filippos Giannakos | return self.offset |
129 | c30635bf | Filippos Giannakos | |
130 | c30635bf | Filippos Giannakos | def truncate(self, size): |
131 | c30635bf | Filippos Giannakos | self.ioctx.trunc(self.name, size) |
132 | c30635bf | Filippos Giannakos | |
133 | c30635bf | Filippos Giannakos | def sync_write(self, data): |
134 | c30635bf | Filippos Giannakos | #self.dirty = 1
|
135 | c30635bf | Filippos Giannakos | self.ioctx.write(self.name, data, self.offset) |
136 | c30635bf | Filippos Giannakos | self.offset += len(data) |
137 | c30635bf | Filippos Giannakos | |
138 | c30635bf | Filippos Giannakos | def sync_write_chunks(self, chunksize, offset, chunks, size=None): |
139 | c30635bf | Filippos Giannakos | #self.dirty = 1
|
140 | c30635bf | Filippos Giannakos | return file_sync_write_chunks(self, chunksize, offset, chunks, size) |
141 | c30635bf | Filippos Giannakos | |
142 | c30635bf | Filippos Giannakos | def sync_read(self, size): |
143 | c30635bf | Filippos Giannakos | read = self.ioctx.read
|
144 | c30635bf | Filippos Giannakos | data = ''
|
145 | c30635bf | Filippos Giannakos | datalen = 0
|
146 | c30635bf | Filippos Giannakos | while 1: |
147 | c30635bf | Filippos Giannakos | try:
|
148 | 29148653 | Sofia Papagiannaki | s = read(self.name, size - datalen, self.offset) |
149 | c30635bf | Filippos Giannakos | except ObjectNotFound:
|
150 | c30635bf | Filippos Giannakos | s = None
|
151 | c30635bf | Filippos Giannakos | if not s: |
152 | c30635bf | Filippos Giannakos | break
|
153 | c30635bf | Filippos Giannakos | data += s |
154 | c30635bf | Filippos Giannakos | datalen += len(s)
|
155 | c30635bf | Filippos Giannakos | self.offset += len(s) |
156 | c30635bf | Filippos Giannakos | if datalen >= size:
|
157 | c30635bf | Filippos Giannakos | break
|
158 | c30635bf | Filippos Giannakos | return data
|
159 | c30635bf | Filippos Giannakos | |
160 | c30635bf | Filippos Giannakos | def sync_read_chunks(self, chunksize, nr, offset=0): |
161 | c30635bf | Filippos Giannakos | return file_sync_read_chunks(self, chunksize, nr, offset) |