#region /* ----------------------------------------------------------------------- * * * Copyright 2011-2012 GRNET S.A. All rights reserved. * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * 1. Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * 2. Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and * documentation are those of the authors and should not be * interpreted as representing official policies, either expressed * or implied, of GRNET S.A. * * ----------------------------------------------------------------------- */ #endregion using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.Contracts; using System.IO; using System.Reflection; using System.Runtime.Remoting.Metadata.W3cXsd2001; using System.Threading; using System.Threading.Tasks; using System.Linq; using Pithos.Interfaces; using log4net; using OpenSSL.Core; using OpenSSL.Crypto; namespace Pithos.Network { public static class Signature { private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); public const int BufferSize = 16384; public const string MD5_EMPTY = "d41d8cd98f00b204e9800998ecf8427e"; public const string MERKLE_EMPTY = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; public static string CalculateMD5(FileSystemInfo info) { if (info==null) throw new ArgumentNullException("info"); if (String.IsNullOrWhiteSpace(info.FullName)) throw new ArgumentException("info.FullName is empty","info"); Contract.EndContractBlock(); if (info is DirectoryInfo) return MD5_EMPTY; return CalculateMD5(info.FullName); } public static string CalculateMD5(string path) { if (String.IsNullOrWhiteSpace(path)) throw new ArgumentNullException("path"); Contract.EndContractBlock(); //DON'T calculate hashes for folders if (Directory.Exists(path)) return ""; //TODO: Replace with MD5BlockCalculator string hash; using (var hasher = new MessageDigestContext(MessageDigest.CreateByName("md5"))) using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize, true)) { var buffer = new byte[BufferSize]; hasher.Init(); int read; do { read = stream.Read(buffer, 0, buffer.Length); if (read==buffer.Length) { hasher.Update(buffer); } else { var block = new byte[read]; Buffer.BlockCopy(buffer, 0, block, 0, read); hasher.Update(block); } } while (read>0); var hashBytes = hasher.DigestFinal(); hash = hashBytes.ToHashString(); } return hash; } /* public static string BytesToString(byte[] hashBytes) { var shb=new SoapHexBinary(hashBytes); return shb.ToString(); } public static byte[] StringToBytes(string hash) { var shb=SoapHexBinary.Parse(hash); return shb.Value; } */ public static byte[] ToBytes(this string hash) { var shb = SoapHexBinary.Parse(hash); return shb.Value; } public static string ToHashString(this byte[] hashBytes) { var shb = new SoapHexBinary(hashBytes); return shb.ToString().ToLower(); } public static TreeHash CalculateTreeHash(FileSystemInfo fileInfo, int blockSize, string algorithm,CancellationToken token,IProgress progress ) { if (fileInfo == null) throw new ArgumentNullException("fileInfo"); if (String.IsNullOrWhiteSpace(fileInfo.FullName)) throw new ArgumentException("fileInfo.FullName is empty", "fileInfo"); if (blockSize <= 0) throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); fileInfo.Refresh(); if (fileInfo is DirectoryInfo || !fileInfo.Exists) return TreeHash.Empty; return CalculateTreeHashAsync(fileInfo, blockSize, algorithm, 1, token, progress); } /// /// Calculates a file's tree hash synchronously, using the specified block size /// /// Path to an existing file /// Block size used to calculate leaf hashes /// /// A with the block hashes and top hash public static TreeHash CalculateTreeHash(string filePath, int blockSize, string algorithm,CancellationToken token,IProgress progress ) { if (String.IsNullOrWhiteSpace(filePath)) throw new ArgumentNullException("filePath"); if (blockSize<=0) throw new ArgumentOutOfRangeException("blockSize","blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); var info = FileInfoExtensions.FromPath(filePath); var hash=CalculateTreeHashAsync(info, blockSize, algorithm, 1,token,progress); return hash; } public static TreeHash CalculateTreeHashAsync(string filePath, int blockSize, string algorithm, byte parallelism,CancellationToken token,IProgress progress ) { if (filePath== null) throw new ArgumentNullException("filePath"); if (blockSize <= 0) throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); var info = FileInfoExtensions.FromPath(filePath); return CalculateTreeHashAsync(info, blockSize, algorithm, parallelism,token,progress); } public static TreeHash CalculateTreeHashAsync(FileSystemInfo info, int blockSize,string algorithm, int parallelism,CancellationToken token,IProgress progress ) { if (info==null) throw new ArgumentNullException("info"); if (blockSize <= 0) throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); var filePath = info.FullName; if (Log.IsDebugEnabled) Log.DebugFormat("Calc Signature [{0}]",filePath); if (filePath.Split('/').Contains(".pithos.cache")) throw new ArgumentException(String.Format("Trying to hash file from the cache folder: [{0}]",filePath)); //DON'T calculate hashes for folders if (Directory.Exists(filePath)) return new TreeHash(algorithm); //The hash of a non-existent file is the empty hash if (!File.Exists(filePath)) return new TreeHash(algorithm); if (info is FileInfo && (info as FileInfo).Length==0) return new TreeHash(algorithm); //Calculate the hash of all blocks using a blockhash iterator using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, BufferSize, true)) { var md5 = new MD5BlockCalculator(); Action postAction = md5.PostBlock; //Calculate the blocks asyncrhonously var hashes = BlockHashAlgorithms.CalculateBlockHashesInPlacePFor(stream, blockSize, algorithm, parallelism,postAction,token, progress).Result; //And then proceed with creating and returning a TreeHash var length = stream.Length; var list = hashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList(); var treeHash = new TreeHash(algorithm) { Bytes = length, BlockSize = blockSize, Hashes = list, }; string fileHash; var md5Hash=md5.GetHash().Result; treeHash.MD5= md5Hash; return treeHash; } } public static byte[] CalculateTopHash(IList hashMap, string algorithm) { if (hashMap == null) throw new ArgumentNullException("hashMap"); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); var hashCount = hashMap.Count; //TODO: Replace this calculation with a constant //The tophash of an empty hashmap is the hash of an empty array if (hashCount == 0) { using (var hasher = new MessageDigestContext(MessageDigest.CreateByName(algorithm))) { hasher.Init(); var emptyHash=hasher.Digest(new byte[0]); return emptyHash; } } //The tophash of a one-item hashmap is the hash itself if (hashCount == 1) return hashMap[0]; //Calculate the required number of leaf nodes var leafs =(int)Math.Pow(2, Math.Ceiling(Math.Log(hashCount,2))); //The size of all nodes is the same and equal to the size of the input hashes var hashSize = hashMap[0].Length; //If the hashmap containes fewer nodes than the required leaf count, we need to fill //the rest with empty blocks byte[] empty=null; if (hashCount < leafs) empty = new byte[hashSize]; //New hashes will be stored in a dictionary keyed by their step to preserve order var newHashes=new ConcurrentDictionary(); Parallel.For(0, leafs/2, (step, state) => { using (var hasher = new MessageDigestContext(MessageDigest.CreateByName(algorithm))) { hasher.Init(); var i = step*2; var block1 = i <= hashCount - 1 ? hashMap[i] : empty; var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty; hasher.Update(block1); hasher.Update(block2); var finalHash = hasher.DigestFinal(); //Store the final value in its proper place newHashes[step] = finalHash; } }); //Extract the hashes to a list ordered by their step var hashes = newHashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList(); return CalculateTopHash(hashes, algorithm); } public static byte[] CalculateHash(byte[] buffer,string algorithm) { if (buffer == null) throw new ArgumentNullException("buffer"); if (String.IsNullOrWhiteSpace(algorithm)) throw new ArgumentNullException("algorithm"); Contract.EndContractBlock(); using (var hasher = new MessageDigestContext(MessageDigest.CreateByName(algorithm))) { hasher.Init(); var hash = hasher.Digest(buffer); return hash; } } } }