using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.Contracts; using System.IO; using System.Runtime.Remoting.Metadata.W3cXsd2001; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; using System.Linq; namespace Pithos.Network { public static class Signature { public static string CalculateMD5(FileInfo info) { if (info==null) throw new ArgumentNullException("info"); Contract.EndContractBlock(); return CalculateMD5(info.FullName); } public static string CalculateMD5(string path) { if (String.IsNullOrWhiteSpace(path)) throw new ArgumentNullException("path"); Contract.EndContractBlock(); //DON'T calculate hashes for folders if (Directory.Exists(path)) return ""; string hash; using (var hasher = MD5.Create()) using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { var hashBytes = hasher.ComputeHash(stream); hash = hashBytes.ToHashString(); } return hash; } /* public static string BytesToString(byte[] hashBytes) { var shb=new SoapHexBinary(hashBytes); return shb.ToString(); } public static byte[] StringToBytes(string hash) { var shb=SoapHexBinary.Parse(hash); return shb.Value; } */ public static byte[] ToBytes(this string hash) { var shb = SoapHexBinary.Parse(hash); return shb.Value; } public static string ToHashString(this byte[] hashBytes) { var shb = new SoapHexBinary(hashBytes); return shb.ToString().ToLower(); } public static TreeHash CalculateTreeHash(FileInfo fileInfo, int blockSize, string algorithm) { if (fileInfo==null) throw new ArgumentNullException("fileInfo"); if (blockSize <= 0) throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); return CalculateTreeHash(fileInfo.FullName, blockSize, algorithm); } /// /// Calculates a file's tree hash synchronously, using the specified block size /// /// Path to an existing file /// Block size used to calculate leaf hashes /// /// A with the block hashes and top hash public static TreeHash CalculateTreeHash(string filePath, int blockSize, string algorithm) { if (String.IsNullOrWhiteSpace(filePath)) throw new ArgumentNullException("filePath"); if (blockSize<=0) throw new ArgumentOutOfRangeException("blockSize","blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); //DON'T calculate hashes for folders if (Directory.Exists(filePath)) return null; var list = new List(); using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, false)) using (var hasher = HashAlgorithm.Create(algorithm)) { var buffer = new byte[blockSize]; int read; while ((read=stream.Read(buffer, 0, blockSize)) > 0) { //This code was added for compatibility with the way Pithos calculates the last hash //We calculate the hash only up to the last non-null byte //TODO: Remove if the server starts using the full block instead of the trimmed block var lastByteIndex=Array.FindLastIndex(buffer,read-1, aByte => aByte != 0); var hash = hasher.ComputeHash(buffer, 0, lastByteIndex+1); list.Add(hash); } return new TreeHash(algorithm) { Hashes = list, BlockSize = blockSize, Bytes = stream.Length}; } } public static Task CalculateTreeHashAsync(FileInfo fileInfo, int blockSize, string algorithm) { if (fileInfo == null) throw new ArgumentNullException("fileInfo"); if (blockSize <= 0) throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); return CalculateTreeHashAsync(fileInfo.FullName, blockSize, algorithm); } public static Task CalculateTreeHashAsync(string filePath, int blockSize,string algorithm) { if (String.IsNullOrWhiteSpace(filePath)) throw new ArgumentNullException("filePath"); if (blockSize <= 0) throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); //DON'T calculate hashes for folders if (Directory.Exists(filePath)) return Task.Factory.StartNew(()=>new TreeHash(algorithm)); //Calculate the hash of all blocks using a blockhash iterator var treeHash =Iterate(BlockHashIterator(filePath, blockSize, algorithm)); return treeHash; } private static IEnumerable BlockHashIterator(string filePath, int blockSize, string algorithm) { using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, true)) { //Calculate the blocks asyncrhonously var hashes= CalculateBlockHashesAsync(stream, blockSize, algorithm); yield return hashes; //And then proceed with creating and returning a TreeHash var length = stream.Length; var list = hashes.Result.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList(); var treeHash = new TreeHash(algorithm) { Bytes = length, BlockSize = blockSize, Hashes = list }; yield return Task.Factory.FromResult(treeHash); } } //The Task Iterator style allows the execution of a sequence of patterns using //iterator syntax. //This particular variation returns the result of the last task, if there is one public static Task Iterate(IEnumerable asyncIterator) { if (asyncIterator == null) throw new ArgumentNullException("asyncIterator"); var enumerator = asyncIterator.GetEnumerator(); if (enumerator == null) throw new InvalidOperationException("Invalid enumerable - GetEnumerator returned null"); var tcs = new TaskCompletionSource(); tcs.Task.ContinueWith(_ => enumerator.Dispose(), TaskContinuationOptions.ExecuteSynchronously); Action recursiveBody = null; recursiveBody = delegate { try { if (enumerator.MoveNext()) enumerator.Current.ContinueWith(recursiveBody, TaskContinuationOptions.ExecuteSynchronously); else { var lastTask = enumerator.Current as Task; var result = (lastTask !=null ) ? lastTask.Result : default(TResult); tcs.TrySetResult(result); } } catch (Exception exc) { tcs.TrySetException(exc); } }; recursiveBody(null); return tcs.Task; } /* public static byte[] CalculateTopHash(IEnumerable hashMap, string algorithm) { if (hashMap == null) throw new ArgumentNullException("hashMap"); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); var hashCount = hashMap.Count(); if (hashCount == 0) return null; using (var hasher = HashAlgorithm.Create(algorithm)) { var i = 0; var count = hashCount; foreach (var block in hashMap) { if (i++ != count - 1) hasher.TransformBlock(block, 0, block.Length, null, 0); else hasher.TransformFinalBlock(block, 0, block.Length); } var finalHash = hasher.Hash; return finalHash; } }*/ public static byte[] CalculateTopHash(IList hashMap, string algorithm) { if (hashMap == null) throw new ArgumentNullException("hashMap"); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); var hashCount = hashMap.Count; //The tophash of an empty hashmap is an empty array if (hashCount == 0) return new byte[0]; //The tophash of a one-item hashmap is the hash itself if (hashCount == 1) return hashMap[0]; //Calculate the required number of leaf nodes var leafs =(int)Math.Pow(2, Math.Ceiling(Math.Log(hashCount,2))); //The size of all nodes is the same and equal to the size of the input hashes var hashSize = hashMap[0].Length; //If the hashmap containes fewer nodes than the required leaf count, we need to fill //the rest with empty blocks byte[] empty=null; if (hashCount < leafs) empty = new byte[hashSize]; //New hashes will be stored in a dictionary keyed by their step to preserve order var newHashes=new ConcurrentDictionary(); Parallel.For(0, leafs/2, (step, state) => { using (var hasher = HashAlgorithm.Create(algorithm)) { var i = step*2; var block1 = i <= hashCount - 1 ? hashMap[i] : empty; var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty; hasher.TransformBlock(block1, 0, block1.Length, null, 0); hasher.TransformFinalBlock(block2, 0, block2.Length); var finalHash = hasher.Hash; //Store the final value in its proper place newHashes[step] = finalHash; } }); /* Parallel.For(0, leafs/2, () => HashAlgorithm.Create(algorithm), (step, state, hasher) => { var i = step*2; var block1 = i <= hashCount - 1 ? hashMap[i]: empty; var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty; hasher.TransformBlock(block1, 0, block1.Length, null, 0); hasher.TransformFinalBlock(block2, 0, block2.Length); var finalHash = hasher.Hash; //Store the final value in its proper place newHashes[step] = finalHash; return hasher; }, hasher => hasher.Dispose()); */ //Extract the hashes to a list ordered by their step var hashes = newHashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList(); return CalculateTopHash(hashes, algorithm); } /* public static string CalculateTopHash(string hashString, string algorithm) { if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); if (String.IsNullOrWhiteSpace(hashString)) return String.Empty; using (var hasher = HashAlgorithm.Create(algorithm)) { var bytes=Encoding.ASCII.GetBytes(hashString.ToLower()); var hash=hasher.ComputeHash(bytes); return hash.ToHashString(); } }*/ private static Task> CalculateBlockHashesAsync(FileStream stream, int blockSize, string algorithm, ConcurrentDictionary hashes=null, int index = 0) { if (stream==null) throw new ArgumentNullException("stream"); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); if (blockSize <= 0) throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); if (index< 0) throw new ArgumentOutOfRangeException("index", "index must be a non-negative value"); Contract.EndContractBlock(); if (hashes==null) hashes= new ConcurrentDictionary(); var buffer = new byte[blockSize]; return stream.ReadAsync(buffer, 0, blockSize).ContinueWith(t => { var read = t.Result; var nextTask = read == blockSize ? CalculateBlockHashesAsync(stream, blockSize, algorithm, hashes, index + 1) : Task.Factory.StartNew(() => hashes); if (read>0) using (var hasher = HashAlgorithm.Create(algorithm)) { //This code was added for compatibility with the way Pithos calculates the last hash //We calculate the hash only up to the last non-null byte //TODO: Remove if the server starts using the full block instead of the trimmed block var lastByteIndex = Array.FindLastIndex(buffer, read - 1, aByte => aByte != 0); var hash = hasher.ComputeHash(buffer, 0, lastByteIndex+1); hashes[index]=hash; } return nextTask; }).Unwrap(); } public static byte[] CalculateHash(byte[] buffer,string algorithm) { if (buffer == null) throw new ArgumentNullException("buffer"); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); using (var hasher = HashAlgorithm.Create(algorithm)) { var hash = hasher.ComputeHash(buffer, 0, buffer.Length); return hash; } } } }