Refs: #4871
Fstat was used to compute the actual file size in order to
distinguish if an empty read was because the file ended.
It now considers an empty read to be an end of file.
This allows reading non-regular files.
hash_gen = hash_cb(nblocks)
hash_gen.next()
hash_gen = hash_cb(nblocks)
hash_gen.next()
- for i in range(nblocks):
+ for i in xrange(nblocks):
block = readall(fileobj, min(blocksize, size - offset))
bytes = len(block)
block = readall(fileobj, min(blocksize, size - offset))
bytes = len(block)
hash = _pithos_hash(block, blockhash)
hashes.append(hash)
hmap[hash] = (offset, bytes)
offset += bytes
if hash_cb:
hash_gen.next()
hash = _pithos_hash(block, blockhash)
hashes.append(hash)
hmap[hash] = (offset, bytes)
offset += bytes
if hash_cb:
hash_gen.next()
- msg = ('Failed to calculate uploaded blocks:'
- ' Offset and object size do not match')
+ msg = ('Failed to calculate uploading blocks: '
+ 'read bytes(%s) != requested size (%s)' % (offset, size))
assert offset == size, msg
def _upload_missing_blocks(self, missing, hmap, fileobj, upload_gen=None):
assert offset == size, msg
def _upload_missing_blocks(self, missing, hmap, fileobj, upload_gen=None):
def readall(openfile, size, retries=7):
"""Read a file until size is reached"""
def readall(openfile, size, retries=7):
"""Read a file until size is reached"""
- from os import fstat
- actual_size = fstat(openfile.fileno()).st_size - openfile.tell()
- size = actual_size if actual_size < size else size
remains = size if size > 0 else 0
buf = ''
for i in range(retries):
remains = size if size > 0 else 0
buf = ''
for i in range(retries):
- buf += openfile.read(remains)
- remains = size - len(buf)
- if remains:
- continue
+ tmp_buf = openfile.read(remains)
+ if tmp_buf:
+ buf += tmp_buf
+ remains -= len(tmp_buf)
+ if remains > 0:
+ continue
return buf
raise IOError('Failed to read %s bytes from file' % size)
return buf
raise IOError('Failed to read %s bytes from file' % size)