using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.IO;
using System.Runtime.Remoting.Metadata.W3cXsd2001;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using System.Linq;
namespace Pithos.Network
{
public static class Signature
{
public static string CalculateMD5(FileInfo info)
{
if (info==null)
throw new ArgumentNullException("info");
Contract.EndContractBlock();
return CalculateMD5(info.FullName);
}
public static string CalculateMD5(string path)
{
if (String.IsNullOrWhiteSpace(path))
throw new ArgumentNullException("path");
Contract.EndContractBlock();
//DON'T calculate hashes for folders
if (Directory.Exists(path))
return "";
string hash;
using (var hasher = MD5.Create())
using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
{
var hashBytes = hasher.ComputeHash(stream);
hash = hashBytes.ToHashString();
}
return hash;
}
/*
public static string BytesToString(byte[] hashBytes)
{
var shb=new SoapHexBinary(hashBytes);
return shb.ToString();
}
public static byte[] StringToBytes(string hash)
{
var shb=SoapHexBinary.Parse(hash);
return shb.Value;
}
*/
public static byte[] ToBytes(this string hash)
{
var shb = SoapHexBinary.Parse(hash);
return shb.Value;
}
public static string ToHashString(this byte[] hashBytes)
{
var shb = new SoapHexBinary(hashBytes);
return shb.ToString().ToLower();
}
public static TreeHash CalculateTreeHash(FileInfo fileInfo, int blockSize, string algorithm)
{
if (fileInfo==null)
throw new ArgumentNullException("fileInfo");
if (blockSize <= 0)
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
return CalculateTreeHash(fileInfo.FullName, blockSize, algorithm);
}
///
/// Calculates a file's tree hash synchronously, using the specified block size
///
/// Path to an existing file
/// Block size used to calculate leaf hashes
///
/// A with the block hashes and top hash
public static TreeHash CalculateTreeHash(string filePath, int blockSize, string algorithm)
{
if (String.IsNullOrWhiteSpace(filePath))
throw new ArgumentNullException("filePath");
if (blockSize<=0)
throw new ArgumentOutOfRangeException("blockSize","blockSize must be a value greater than zero ");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
//DON'T calculate hashes for folders
if (Directory.Exists(filePath))
return null;
var list = new List();
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, false))
using (var hasher = HashAlgorithm.Create(algorithm))
{
var buffer = new byte[blockSize];
int read;
while ((read=stream.Read(buffer, 0, blockSize)) > 0)
{
//This code was added for compatibility with the way Pithos calculates the last hash
//We calculate the hash only up to the last non-null byte
//TODO: Remove if the server starts using the full block instead of the trimmed block
var lastByteIndex=Array.FindLastIndex(buffer,read-1, aByte => aByte != 0);
var hash = hasher.ComputeHash(buffer, 0, lastByteIndex+1);
list.Add(hash);
}
return new TreeHash(algorithm) { Hashes = list,
BlockSize = blockSize,
Bytes = stream.Length};
}
}
public static Task CalculateTreeHashAsync(FileInfo fileInfo, int blockSize, string algorithm)
{
if (fileInfo == null)
throw new ArgumentNullException("fileInfo");
if (blockSize <= 0)
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
return CalculateTreeHashAsync(fileInfo.FullName, blockSize, algorithm);
}
public static Task CalculateTreeHashAsync(string filePath, int blockSize,string algorithm)
{
if (String.IsNullOrWhiteSpace(filePath))
throw new ArgumentNullException("filePath");
if (blockSize <= 0)
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
//DON'T calculate hashes for folders
if (Directory.Exists(filePath))
return Task.Factory.StartNew(()=>new TreeHash(algorithm));
//Calculate the hash of all blocks using a blockhash iterator
var treeHash =Iterate(BlockHashIterator(filePath, blockSize, algorithm));
return treeHash;
}
private static IEnumerable BlockHashIterator(string filePath, int blockSize, string algorithm)
{
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, true))
{
//Calculate the blocks asyncrhonously
var hashes= CalculateBlockHashesAsync(stream, blockSize, algorithm);
yield return hashes;
//And then proceed with creating and returning a TreeHash
var length = stream.Length;
var list = hashes.Result.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList();
var treeHash = new TreeHash(algorithm)
{
Bytes = length,
BlockSize = blockSize,
Hashes = list
};
yield return Task.Factory.FromResult(treeHash);
}
}
//The Task Iterator style allows the execution of a sequence of patterns using
//iterator syntax.
//This particular variation returns the result of the last task, if there is one
public static Task Iterate(IEnumerable asyncIterator)
{
if (asyncIterator == null)
throw new ArgumentNullException("asyncIterator");
var enumerator = asyncIterator.GetEnumerator();
if (enumerator == null)
throw new InvalidOperationException("Invalid enumerable - GetEnumerator returned null");
var tcs = new TaskCompletionSource();
tcs.Task.ContinueWith(_ => enumerator.Dispose(), TaskContinuationOptions.ExecuteSynchronously);
Action recursiveBody = null;
recursiveBody = delegate
{
try
{
if (enumerator.MoveNext())
enumerator.Current.ContinueWith(recursiveBody,
TaskContinuationOptions.ExecuteSynchronously);
else
{
var lastTask = enumerator.Current as Task;
var result = (lastTask !=null )
? lastTask.Result
: default(TResult);
tcs.TrySetResult(result);
}
}
catch (Exception exc)
{
tcs.TrySetException(exc);
}
};
recursiveBody(null);
return tcs.Task;
}
/* public static byte[] CalculateTopHash(IEnumerable hashMap, string algorithm)
{
if (hashMap == null)
throw new ArgumentNullException("hashMap");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
var hashCount = hashMap.Count();
if (hashCount == 0)
return null;
using (var hasher = HashAlgorithm.Create(algorithm))
{
var i = 0;
var count = hashCount;
foreach (var block in hashMap)
{
if (i++ != count - 1)
hasher.TransformBlock(block, 0, block.Length, null, 0);
else
hasher.TransformFinalBlock(block, 0, block.Length);
}
var finalHash = hasher.Hash;
return finalHash;
}
}*/
public static byte[] CalculateTopHash(IList hashMap, string algorithm)
{
if (hashMap == null)
throw new ArgumentNullException("hashMap");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
var hashCount = hashMap.Count;
//The tophash of an empty hashmap is an empty array
if (hashCount == 0)
return new byte[0];
//The tophash of a one-item hashmap is the hash itself
if (hashCount == 1)
return hashMap[0];
//Calculate the required number of leaf nodes
var leafs =(int)Math.Pow(2, Math.Ceiling(Math.Log(hashCount,2)));
//The size of all nodes is the same and equal to the size of the input hashes
var hashSize = hashMap[0].Length;
//If the hashmap containes fewer nodes than the required leaf count, we need to fill
//the rest with empty blocks
byte[] empty=null;
if (hashCount < leafs)
empty = new byte[hashSize];
//New hashes will be stored in a dictionary keyed by their step to preserve order
var newHashes=new ConcurrentDictionary();
Parallel.For(0, leafs/2,
(step, state) =>
{
using (var hasher = HashAlgorithm.Create(algorithm))
{
var i = step*2;
var block1 = i <= hashCount - 1 ? hashMap[i] : empty;
var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty;
hasher.TransformBlock(block1, 0, block1.Length, null, 0);
hasher.TransformFinalBlock(block2, 0, block2.Length);
var finalHash = hasher.Hash;
//Store the final value in its proper place
newHashes[step] = finalHash;
}
});
/*
Parallel.For(0, leafs/2,
() => HashAlgorithm.Create(algorithm),
(step, state, hasher) =>
{
var i = step*2;
var block1 = i <= hashCount - 1 ? hashMap[i]: empty;
var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty;
hasher.TransformBlock(block1, 0, block1.Length, null, 0);
hasher.TransformFinalBlock(block2, 0, block2.Length);
var finalHash = hasher.Hash;
//Store the final value in its proper place
newHashes[step] = finalHash;
return hasher;
},
hasher => hasher.Dispose());
*/
//Extract the hashes to a list ordered by their step
var hashes = newHashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList();
return CalculateTopHash(hashes, algorithm);
}
/* public static string CalculateTopHash(string hashString, string algorithm)
{
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
if (String.IsNullOrWhiteSpace(hashString))
return String.Empty;
using (var hasher = HashAlgorithm.Create(algorithm))
{
var bytes=Encoding.ASCII.GetBytes(hashString.ToLower());
var hash=hasher.ComputeHash(bytes);
return hash.ToHashString();
}
}*/
private static Task> CalculateBlockHashesAsync(FileStream stream, int blockSize, string algorithm, ConcurrentDictionary hashes=null, int index = 0)
{
if (stream==null)
throw new ArgumentNullException("stream");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
if (blockSize <= 0)
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
if (index< 0)
throw new ArgumentOutOfRangeException("index", "index must be a non-negative value");
Contract.EndContractBlock();
if (hashes==null)
hashes= new ConcurrentDictionary();
var buffer = new byte[blockSize];
return stream.ReadAsync(buffer, 0, blockSize).ContinueWith(t =>
{
var read = t.Result;
var nextTask = read == blockSize
? CalculateBlockHashesAsync(stream, blockSize, algorithm, hashes, index + 1)
: Task.Factory.StartNew(() => hashes);
if (read>0)
using (var hasher = HashAlgorithm.Create(algorithm))
{
//This code was added for compatibility with the way Pithos calculates the last hash
//We calculate the hash only up to the last non-null byte
//TODO: Remove if the server starts using the full block instead of the trimmed block
var lastByteIndex = Array.FindLastIndex(buffer, read - 1, aByte => aByte != 0);
var hash = hasher.ComputeHash(buffer, 0, lastByteIndex+1);
hashes[index]=hash;
}
return nextTask;
}).Unwrap();
}
public static byte[] CalculateHash(byte[] buffer,string algorithm)
{
if (buffer == null)
throw new ArgumentNullException("buffer");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
using (var hasher = HashAlgorithm.Create(algorithm))
{
var hash = hasher.ComputeHash(buffer, 0, buffer.Length);
return hash;
}
}
}
}