#region /* ----------------------------------------------------------------------- * * * Copyright 2011-2012 GRNET S.A. All rights reserved. * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * 1. Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * 2. Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and * documentation are those of the authors and should not be * interpreted as representing official policies, either expressed * or implied, of GRNET S.A. * * ----------------------------------------------------------------------- */ #endregion using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.Contracts; using System.IO; using System.Reflection; using System.Runtime.Remoting.Metadata.W3cXsd2001; using System.Threading; using System.Threading.Tasks; using System.Linq; using Pithos.Interfaces; using log4net; using OpenSSL.Core; using OpenSSL.Crypto; namespace Pithos.Network { public static class Signature { private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); public const int BufferSize = 16384; // public const string MD5_EMPTY = "d41d8cd98f00b204e9800998ecf8427e"; public const string MERKLE_EMPTY = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; public static byte[] ToBytes(this string hash) { var shb = SoapHexBinary.Parse(hash); return shb.Value; } public static string ToHashString(this byte[] hashBytes) { var shb = new SoapHexBinary(hashBytes); return shb.ToString().ToLower(); } public static TreeHash CalculateTreeHash(FileSystemInfo fileInfo, int blockSize, string algorithm,byte parallelism,CancellationToken token,IProgress progress ) { if (fileInfo == null) throw new ArgumentNullException("fileInfo"); if (String.IsNullOrWhiteSpace(fileInfo.FullName)) throw new ArgumentException("fileInfo.FullName is empty", "fileInfo"); if (blockSize <= 0) throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); fileInfo.Refresh(); if (fileInfo is DirectoryInfo || !fileInfo.Exists) return TreeHash.Empty; return TaskEx.Run(async ()=>await CalculateTreeHashAsync(fileInfo, blockSize, algorithm, parallelism, token, progress)).Result; } /// /// Calculates a file's tree hash synchronously, using the specified block size /// /// Path to an existing file /// Block size used to calculate leaf hashes /// /// A with the block hashes and top hash public static TreeHash CalculateTreeHash(string filePath, int blockSize, string algorithm,CancellationToken token,IProgress progress ) { if (String.IsNullOrWhiteSpace(filePath)) throw new ArgumentNullException("filePath"); if (blockSize<=0) throw new ArgumentOutOfRangeException("blockSize","blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); var info = FileInfoExtensions.FromPath(filePath); var hash=TaskEx.Run(async ()=>await CalculateTreeHashAsync(info, blockSize, algorithm, 1,token,progress).ConfigureAwait(false)).Result; return hash; } public static Task CalculateTreeHashAsync(string filePath, int blockSize, string algorithm, byte parallelism,CancellationToken token,IProgress progress ) { if (filePath== null) throw new ArgumentNullException("filePath"); if (blockSize <= 0) throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); var info = FileInfoExtensions.FromPath(filePath); return CalculateTreeHashAsync(info, blockSize, algorithm, parallelism,token,progress); } public static async Task CalculateTreeHashAsync(FileSystemInfo info, int blockSize,string algorithm, byte parallelism,CancellationToken token,IProgress progress ) { if (info==null) throw new ArgumentNullException("info"); if (blockSize <= 0) throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); var filePath = info.FullName; if (Log.IsDebugEnabled) Log.DebugFormat("Calc Signature [{0}]",filePath); if (filePath.Split('/').Contains(".pithos.cache")) throw new ArgumentException(String.Format("Trying to hash file from the cache folder: [{0}]",filePath)); //DON'T calculate hashes for folders if (Directory.Exists(filePath)) return new TreeHash(algorithm); //The hash of a non-existent file is the empty hash if (!File.Exists(filePath)) return new TreeHash(algorithm); if (info is FileInfo && (info as FileInfo).Length==0) return new TreeHash(algorithm); //Calculate the hash of all blocks using a blockhash iterator using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, BufferSize, true)) { //var md5 = new MD5BlockCalculator(); //Action postAction = md5.PostBlock; //Calculate the blocks asyncrhonously var hashes = await BlockHashAlgorithms.CalculateBlockHashesInPlacePFor(stream, blockSize, algorithm, parallelism,token, progress).ConfigureAwait(false); //var hashes = BlockHashAlgorithms.CalculateBlockHashesInPlacePFor(stream, blockSize, algorithm, parallelism, postAction, token, progress).Result; //And then proceed with creating and returning a TreeHash var length = stream.Length; var list = hashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList(); var treeHash = new TreeHash(algorithm) { Bytes = length, BlockSize = blockSize, Hashes = list, }; //string fileHash; //var md5Hash=md5.GetHash().Result; //treeHash.MD5= md5Hash; return treeHash; } } public static byte[] CalculateTopHash(IList hashMap, string algorithm) { if (hashMap == null) throw new ArgumentNullException("hashMap"); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); var hashCount = hashMap.Count; //TODO: Replace this calculation with a constant //The tophash of an empty hashmap is the hash of an empty array if (hashCount == 0) { using (var hasher = new MessageDigestContext(MessageDigest.CreateByName(algorithm))) { hasher.Init(); var emptyHash=hasher.Digest(new byte[0]); return emptyHash; } } //The tophash of a one-item hashmap is the hash itself if (hashCount == 1) return hashMap[0]; //Calculate the required number of leaf nodes var leafs =(int)Math.Pow(2, Math.Ceiling(Math.Log(hashCount,2))); //The size of all nodes is the same and equal to the size of the input hashes var hashSize = hashMap[0].Length; //If the hashmap containes fewer nodes than the required leaf count, we need to fill //the rest with empty blocks byte[] empty=null; if (hashCount < leafs) empty = new byte[hashSize]; //New hashes will be stored in a dictionary keyed by their step to preserve order var newHashes=new ConcurrentDictionary(); Parallel.For(0, leafs/2, (step, state) => { using (var hasher = new MessageDigestContext(MessageDigest.CreateByName(algorithm))) { hasher.Init(); var i = step*2; var block1 = i <= hashCount - 1 ? hashMap[i] : empty; var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty; hasher.Update(block1); hasher.Update(block2); var finalHash = hasher.DigestFinal(); //Store the final value in its proper place newHashes[step] = finalHash; } }); //Extract the hashes to a list ordered by their step var hashes = newHashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList(); return CalculateTopHash(hashes, algorithm); } public static byte[] CalculateHash(byte[] buffer,string algorithm) { if (buffer == null) throw new ArgumentNullException("buffer"); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); using (var hasher = new MessageDigestContext(MessageDigest.CreateByName(algorithm))) { hasher.Init(); var hash = hasher.Digest(buffer); return hash; } } } public class HashProgress { public HashProgress(long current,long total) { Current = current; Total = total; } public float Percentage { get { if (Current > Total) return 1; return Current/(float)Total; }} public long Current { get; set; } public long Total { get; set; } } }