Revision db63c1ae
| b/trunk/Pithos.Network/BlockHashAlgorithms.cs | ||
|---|---|---|
| 230 | 230 |
return _bufferMgr; |
| 231 | 231 |
} |
| 232 | 232 |
|
| 233 |
public static async Task<ConcurrentDictionary<long, byte[]>> CalculateBlockHashesInPlacePFor(FileStream stream, int blockSize, string algorithm, int parallelism,IProgress<double> progress )
|
|
| 233 |
public static async Task<ConcurrentDictionary<long, byte[]>> CalculateBlockHashesInPlacePFor(FileStream stream, int blockSize, string algorithm, int parallelism, Action<long, byte[], int> postAction, IProgress<double> progress)
|
|
| 234 | 234 |
{
|
| 235 | 235 |
if (stream == null) |
| 236 | 236 |
throw new ArgumentNullException("stream");
|
| ... | ... | |
| 273 | 273 |
int bufIdx = 0; |
| 274 | 274 |
long index = 0; |
| 275 | 275 |
|
| 276 |
long block = 0; |
|
| 276 | 277 |
|
| 277 | 278 |
while ((read = await stream.ReadAsync(buffer[bufIdx], 0, blockSize).ConfigureAwait(false)) > 0) |
| 278 | 279 |
{
|
| 279 | 280 |
index += read; |
| 280 | 281 |
indices[bufIdx] = index; |
| 281 | 282 |
bufferCount[bufIdx] = read; |
| 283 |
postAction(block++, buffer[bufIdx], read); |
|
| 284 |
|
|
| 282 | 285 |
//If we have filled the last buffer or if we have read from the last block, |
| 283 | 286 |
//we can calculate the clocks in parallel |
| 284 | 287 |
if (bufIdx == parallelism - 1 || read < blockSize) |
| ... | ... | |
| 305 | 308 |
progress.Report((long)hashes.Count*blockSize*1.0/stream.Length); |
| 306 | 309 |
}); |
| 307 | 310 |
} |
| 308 |
bufIdx = (bufIdx + 1)%parallelism;
|
|
| 311 |
bufIdx = (bufIdx +1)%parallelism; |
|
| 309 | 312 |
} |
| 310 | 313 |
} |
| 311 | 314 |
finally |
| b/trunk/Pithos.Network/MD5BlockCalculator.cs | ||
|---|---|---|
| 1 |
using System; |
|
| 2 |
using System.Collections.Generic; |
|
| 3 |
using System.Diagnostics; |
|
| 4 |
using System.Linq; |
|
| 5 |
using System.Security.Cryptography; |
|
| 6 |
using System.Text; |
|
| 7 |
using System.Threading; |
|
| 8 |
using System.Threading.Tasks; |
|
| 9 |
using System.Threading.Tasks.Dataflow; |
|
| 10 |
|
|
| 11 |
namespace Pithos.Network |
|
| 12 |
{
|
|
| 13 |
class MD5BlockCalculator:IDisposable |
|
| 14 |
{
|
|
| 15 |
private HashAlgorithm _hasher = HashAlgorithm.Create("md5");
|
|
| 16 |
|
|
| 17 |
private ActionBlock<Tuple<long, byte[],int>> _actionBlock; |
|
| 18 |
|
|
| 19 |
private long _currentBlock = 0; |
|
| 20 |
|
|
| 21 |
public MD5BlockCalculator() |
|
| 22 |
{
|
|
| 23 |
_actionBlock=new ActionBlock<Tuple<long, byte[],int>>(t=> ProcessBlock(t)); |
|
| 24 |
} |
|
| 25 |
|
|
| 26 |
private void ProcessBlock(Tuple<long,byte[],int> tuple) |
|
| 27 |
{
|
|
| 28 |
if (tuple.Item1 == _currentBlock) |
|
| 29 |
{
|
|
| 30 |
_hasher.TransformBlock(tuple.Item2, 0, tuple.Item3, null, 0); |
|
| 31 |
Interlocked.Increment(ref _currentBlock); |
|
| 32 |
} |
|
| 33 |
else |
|
| 34 |
{
|
|
| 35 |
_actionBlock.Post(tuple); |
|
| 36 |
} |
|
| 37 |
} |
|
| 38 |
|
|
| 39 |
public void PostBlock(long blockIndex,byte[] buffer,int size) |
|
| 40 |
{
|
|
| 41 |
_actionBlock.Post(Tuple.Create(blockIndex, buffer, size)); |
|
| 42 |
} |
|
| 43 |
|
|
| 44 |
public async Task<string> GetHash() |
|
| 45 |
{
|
|
| 46 |
_actionBlock.Complete(); |
|
| 47 |
await _actionBlock.Completion; |
|
| 48 |
Debug.Assert(_actionBlock.InputCount == 0); |
|
| 49 |
_hasher.TransformFinalBlock(new byte[0], 0, 0); |
|
| 50 |
var hash=_hasher.Hash.ToHashString(); |
|
| 51 |
return hash; |
|
| 52 |
} |
|
| 53 |
|
|
| 54 |
public void Dispose() |
|
| 55 |
{
|
|
| 56 |
Dispose(true); |
|
| 57 |
GC.SuppressFinalize(this); |
|
| 58 |
} |
|
| 59 |
|
|
| 60 |
protected virtual void Dispose(bool disposing) |
|
| 61 |
{
|
|
| 62 |
if (disposing && _hasher!=null) |
|
| 63 |
_hasher.Dispose(); |
|
| 64 |
_hasher = null; |
|
| 65 |
} |
|
| 66 |
} |
|
| 67 |
} |
|
| b/trunk/Pithos.Network/Pithos.Network.csproj | ||
|---|---|---|
| 233 | 233 |
<Compile Include="CloudFilesClient.cs" /> |
| 234 | 234 |
<Compile Include="ContainerInfo.cs" /> |
| 235 | 235 |
<Compile Include="ICloudClient.cs" /> |
| 236 |
<Compile Include="MD5BlockCalculator.cs" /> |
|
| 236 | 237 |
<Compile Include="NoModificationInfo.cs" /> |
| 237 | 238 |
<Compile Include="RestClient.cs"> |
| 238 | 239 |
<SubType>Component</SubType> |
| b/trunk/Pithos.Network/Signature.cs | ||
|---|---|---|
| 198 | 198 |
//Calculate the hash of all blocks using a blockhash iterator |
| 199 | 199 |
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, true)) |
| 200 | 200 |
{
|
| 201 |
var md5 = new MD5BlockCalculator(); |
|
| 202 |
Action<long, byte[], int> postAction = md5.PostBlock; |
|
| 201 | 203 |
//Calculate the blocks asyncrhonously |
| 202 |
var hashes = BlockHashAlgorithms.CalculateBlockHashesInPlacePFor(stream, blockSize, algorithm, parallelism,progress).Result; |
|
| 204 |
var hashes = BlockHashAlgorithms.CalculateBlockHashesInPlacePFor(stream, blockSize, algorithm, parallelism,postAction, progress).Result;
|
|
| 203 | 205 |
|
| 204 | 206 |
//And then proceed with creating and returning a TreeHash |
| 205 | 207 |
var length = stream.Length; |
| ... | ... | |
| 213 | 215 |
}; |
| 214 | 216 |
|
| 215 | 217 |
string fileHash; |
| 218 |
|
|
| 219 |
var md5Hash=md5.GetHash().Result; |
|
| 220 |
/* |
|
| 216 | 221 |
var hasher = HashAlgorithm.Create("MD5");
|
| 217 | 222 |
stream.Position = 0; |
| 218 |
treeHash.MD5= hasher.ComputeHash(stream).ToHashString(); |
|
| 223 |
*/ |
|
| 224 |
treeHash.MD5= md5Hash; |
|
| 219 | 225 |
|
| 220 | 226 |
return treeHash; |
| 221 | 227 |
} |
Also available in: Unified diff