+#region
+/* -----------------------------------------------------------------------
+ * <copyright file="Signature.cs" company="GRNet">
+ *
+ * Copyright 2011-2012 GRNET S.A. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and
+ * documentation are those of the authors and should not be
+ * interpreted as representing official policies, either expressed
+ * or implied, of GRNET S.A.
+ * </copyright>
+ * -----------------------------------------------------------------------
+ */
+#endregion
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.IO;
+using System.Reflection;
using System.Runtime.Remoting.Metadata.W3cXsd2001;
using System.Security.Cryptography;
-using System.Text;
using System.Threading.Tasks;
using System.Linq;
{
public static class Signature
{
+ private static readonly log4net.ILog Log = log4net.LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
public static string CalculateMD5(FileInfo info)
{
if (info==null)
string hash;
using (var hasher = MD5.Create())
- using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
+ using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 65536, true))
{
var hashBytes = hasher.ComputeHash(stream);
hash = hashBytes.ToHashString();
return shb.ToString().ToLower();
}
- public static TreeHash CalculateTreeHash(FileInfo fileInfo, int blockSize, string algorithm)
+ public static TreeHash CalculateTreeHash(FileSystemInfo fileInfo, int blockSize, string algorithm)
{
- if (fileInfo==null)
+ if (fileInfo == null)
throw new ArgumentNullException("fileInfo");
if (String.IsNullOrWhiteSpace(fileInfo.FullName))
- throw new ArgumentException("fileInfo.FullName is empty","fileInfo");
+ throw new ArgumentException("fileInfo.FullName is empty", "fileInfo");
if (blockSize <= 0)
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
+ if (fileInfo is DirectoryInfo || !fileInfo.Exists)
+ return TreeHash.Empty;
+
return CalculateTreeHash(fileInfo.FullName, blockSize, algorithm);
}
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
- var hash=CalculateTreeHashAsync(filePath, blockSize, algorithm);
+ var hash=CalculateTreeHashAsync(filePath, blockSize, algorithm, 2);
return hash.Result;
}
- public static async Task<TreeHash> CalculateTreeHashAsync(FileInfo fileInfo, int blockSize, string algorithm)
+ public static async Task<TreeHash> CalculateTreeHashAsync(FileInfo fileInfo, int blockSize, string algorithm, byte parallelism)
{
if (fileInfo == null)
throw new ArgumentNullException("fileInfo");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
-
- return await CalculateTreeHashAsync(fileInfo.FullName, blockSize, algorithm);
+
+ return await CalculateTreeHashAsync(fileInfo.FullName, blockSize, algorithm, parallelism);
}
- public static async Task<TreeHash> CalculateTreeHashAsync(string filePath, int blockSize,string algorithm)
+ public static async Task<TreeHash> CalculateTreeHashAsync(string filePath, int blockSize,string algorithm, int parallelism)
{
if (String.IsNullOrWhiteSpace(filePath))
throw new ArgumentNullException("filePath");
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
+ if (Log.IsDebugEnabled)
+ Log.DebugFormat("Calc Signature [{0}]",filePath);
+
//DON'T calculate hashes for folders
if (Directory.Exists(filePath))
return new TreeHash(algorithm);
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, true))
{
//Calculate the blocks asyncrhonously
- var hashes = await CalculateBlockHashesAsync(stream, blockSize, algorithm);
+ var hashes = await BlockHashAlgorithms.CalculateBlockHashesInPlacePFor(stream, blockSize, algorithm, parallelism);
//And then proceed with creating and returning a TreeHash
var length = stream.Length;
}
- //The Task Iterator style allows the execution of a sequence of patterns using
- //iterator syntax.
- //This particular variation returns the result of the last task, if there is one
- public static Task<TResult> Iterate<TResult>(IEnumerable<Task> asyncIterator)
- {
- if (asyncIterator == null)
- throw new ArgumentNullException("asyncIterator");
- var enumerator = asyncIterator.GetEnumerator();
- if (enumerator == null)
- throw new InvalidOperationException("Invalid enumerable - GetEnumerator returned null");
-
- var tcs = new TaskCompletionSource<TResult>();
- tcs.Task.ContinueWith(_ => enumerator.Dispose(), TaskContinuationOptions.ExecuteSynchronously);
-
- Action<Task> recursiveBody = null;
- recursiveBody = delegate
- {
- try
- {
- if (enumerator.MoveNext())
- enumerator.Current.ContinueWith(recursiveBody,
- TaskContinuationOptions.ExecuteSynchronously);
- else
- {
- var lastTask = enumerator.Current as Task<TResult>;
- var result = (lastTask !=null )
- ? lastTask.Result
- : default(TResult);
-
- tcs.TrySetResult(result);
- }
- }
- catch (Exception exc)
- {
- tcs.TrySetException(exc);
- }
- };
- recursiveBody(null);
- return tcs.Task;
- }
-
-
- /* public static byte[] CalculateTopHash(IEnumerable<byte[]> hashMap, string algorithm)
- {
- if (hashMap == null)
- throw new ArgumentNullException("hashMap");
- if (String.IsNullOrWhiteSpace(algorithm))
- throw new ArgumentNullException("algorithm");
- Contract.EndContractBlock();
-
- var hashCount = hashMap.Count();
- if (hashCount == 0)
- return null;
- using (var hasher = HashAlgorithm.Create(algorithm))
- {
- var i = 0;
- var count = hashCount;
- foreach (var block in hashMap)
- {
- if (i++ != count - 1)
- hasher.TransformBlock(block, 0, block.Length, null, 0);
- else
- hasher.TransformFinalBlock(block, 0, block.Length);
- }
-
- var finalHash = hasher.Hash;
-
- return finalHash;
- }
- }*/
-
public static byte[] CalculateTopHash(IList<byte[]> hashMap, string algorithm)
{
if (hashMap == null)
newHashes[step] = finalHash;
}
});
-/*
- Parallel.For(0, leafs/2,
- () => HashAlgorithm.Create(algorithm),
- (step, state, hasher) =>
- {
-
- var i = step*2;
- var block1 = i <= hashCount - 1 ? hashMap[i]: empty;
- var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty;
-
- hasher.TransformBlock(block1, 0, block1.Length, null, 0);
- hasher.TransformFinalBlock(block2, 0, block2.Length);
-
- var finalHash = hasher.Hash;
- //Store the final value in its proper place
- newHashes[step] = finalHash;
-
- return hasher;
- },
- hasher => hasher.Dispose());
-*/
+
//Extract the hashes to a list ordered by their step
var hashes = newHashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList();
return CalculateTopHash(hashes, algorithm);
- }
-
-
-
- /* public static string CalculateTopHash(string hashString, string algorithm)
- {
- if (String.IsNullOrWhiteSpace(algorithm))
- throw new ArgumentNullException("algorithm");
- Contract.EndContractBlock();
- if (String.IsNullOrWhiteSpace(hashString))
- return String.Empty;
-
- using (var hasher = HashAlgorithm.Create(algorithm))
- {
- var bytes=Encoding.ASCII.GetBytes(hashString.ToLower());
- var hash=hasher.ComputeHash(bytes);
- return hash.ToHashString();
- }
- }*/
-
- private static Task<ConcurrentDictionary<int, byte[]>> CalculateBlockHashesAsync(FileStream stream, int blockSize, string algorithm, ConcurrentDictionary<int, byte[]> hashes=null, int index = 0)
- {
- if (stream==null)
- throw new ArgumentNullException("stream");
- if (String.IsNullOrWhiteSpace(algorithm))
- throw new ArgumentNullException("algorithm");
- if (blockSize <= 0)
- throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
- if (index< 0)
- throw new ArgumentOutOfRangeException("index", "index must be a non-negative value");
- Contract.EndContractBlock();
-
-
- if (hashes==null)
- hashes= new ConcurrentDictionary<int, byte[]>();
-
- var buffer = new byte[blockSize];
- return stream.ReadAsync(buffer, 0, blockSize).ContinueWith(t =>
- {
- var read = t.Result;
-
- var nextTask = read == blockSize
- ? CalculateBlockHashesAsync(stream, blockSize, algorithm, hashes, index + 1)
- : Task.Factory.StartNew(() => hashes);
- if (read>0)
- using (var hasher = HashAlgorithm.Create(algorithm))
- {
- //This code was added for compatibility with the way Pithos calculates the last hash
- //We calculate the hash only up to the last non-null byte
- //TODO: Remove if the server starts using the full block instead of the trimmed block
- var lastByteIndex = Array.FindLastIndex(buffer, read - 1, aByte => aByte != 0);
-
- var hash = hasher.ComputeHash(buffer, 0, lastByteIndex+1);
- hashes[index]=hash;
- }
- return nextTask;
- }).Unwrap();
- }
-
+ }
public static byte[] CalculateHash(byte[] buffer,string algorithm)
projects / pithos-ms-client / blobdiff