Synnefo » Pithos MS Client

using System;

using System.Collections.Concurrent;

using System.Collections.Generic;

using System.Diagnostics.Contracts;

using System.IO;

using System.Runtime.Remoting.Metadata.W3cXsd2001;

using System.Security.Cryptography;

using System.Text;

using System.Threading.Tasks;

using System.Linq;

namespace Pithos.Network

{

    public static class Signature

    {

        public static string CalculateMD5(FileInfo info)

        {

            if (info==null)

                throw new ArgumentNullException("info");

            Contract.EndContractBlock();

            return CalculateMD5(info.FullName);

        }

        public static string CalculateMD5(string path)

        {

            if (String.IsNullOrWhiteSpace(path))

                throw new ArgumentNullException("path");

            Contract.EndContractBlock();

            //DON'T calculate hashes for folders

            if (Directory.Exists(path))

                return "";

            string hash;

            using (var hasher = MD5.Create())

            using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))

            {

                var hashBytes = hasher.ComputeHash(stream);

                hash = hashBytes.ToHashString();

            }

            return hash;

        }

/*

        public static string BytesToString(byte[] hashBytes)

        {

            var shb=new SoapHexBinary(hashBytes);

            return shb.ToString();

        }

        public static byte[] StringToBytes(string hash)

        {

            var shb=SoapHexBinary.Parse(hash);

            return shb.Value;

        }

*/

        public static byte[] ToBytes(this string hash)

        {

            var shb = SoapHexBinary.Parse(hash);

            return shb.Value;

        }

        public static string ToHashString(this byte[] hashBytes)

        {

            var shb = new SoapHexBinary(hashBytes);

            return shb.ToString().ToLower();

        }

        public static TreeHash CalculateTreeHash(FileInfo fileInfo, int blockSize, string algorithm)

        {

            if (fileInfo==null)

                throw new ArgumentNullException("fileInfo");

            if (blockSize <= 0)

                throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");

            if (String.IsNullOrWhiteSpace(algorithm))

                throw new ArgumentNullException("algorithm");

            Contract.EndContractBlock();

            return CalculateTreeHash(fileInfo.FullName, blockSize, algorithm);

        }

        /// <summary>

        /// Calculates a file's tree hash synchronously, using the specified block size

        /// </summary>

        /// <param name="filePath">Path to an existing file</param>

        /// <param name="blockSize">Block size used to calculate leaf hashes</param>

        /// <param name="algorithm"></param>

        /// <returns>A <see cref="TreeHash"/> with the block hashes and top hash</returns>

        public static TreeHash CalculateTreeHash(string filePath, int blockSize, string algorithm)

        {

            if (String.IsNullOrWhiteSpace(filePath))

                throw new ArgumentNullException("filePath");

            if (blockSize<=0)

                throw new ArgumentOutOfRangeException("blockSize","blockSize must be a value greater than zero ");

            if (String.IsNullOrWhiteSpace(algorithm))

                throw new ArgumentNullException("algorithm");

            Contract.EndContractBlock();

            //DON'T calculate hashes for folders

            if (Directory.Exists(filePath))

                return null;

            var list = new List<byte[]>();

            using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, false))

            using (var hasher = HashAlgorithm.Create(algorithm))

            {

                var buffer = new byte[blockSize];

                int read;

                while ((read=stream.Read(buffer, 0, blockSize)) > 0)

                {

                    //This code was added for compatibility with the way Pithos calculates the last hash

                    //We calculate the hash only up to the last non-null byte

                    //TODO: Remove if the server starts using the full block instead of the trimmed block

                    var lastByteIndex=Array.FindLastIndex(buffer,read-1, aByte => aByte != 0);

                    var hash = hasher.ComputeHash(buffer, 0, lastByteIndex+1);

                    list.Add(hash);                    

                }

                return new TreeHash(algorithm) { Hashes = list,                    

                    BlockSize = blockSize, 

                    Bytes = stream.Length};

            }            

        }

        public static Task<TreeHash> CalculateTreeHashAsync(FileInfo fileInfo, int blockSize, string algorithm)

        {

            if (fileInfo == null)

                throw new ArgumentNullException("fileInfo");

            if (blockSize <= 0)

                throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");

            if (String.IsNullOrWhiteSpace(algorithm))

                throw new ArgumentNullException("algorithm");

            Contract.EndContractBlock();

            return CalculateTreeHashAsync(fileInfo.FullName, blockSize, algorithm);

        }

        public static Task<TreeHash> CalculateTreeHashAsync(string filePath, int blockSize,string algorithm)

        {

            if (String.IsNullOrWhiteSpace(filePath))

                throw new ArgumentNullException("filePath");

            if (blockSize <= 0)

                throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");

            if (String.IsNullOrWhiteSpace(algorithm))

                throw new ArgumentNullException("algorithm");

            Contract.EndContractBlock();

            //DON'T calculate hashes for folders

            if (Directory.Exists(filePath))

                return Task.Factory.StartNew(()=>new TreeHash(algorithm));

            //Calculate the hash of all blocks using a blockhash iterator

            var treeHash =Iterate<TreeHash>(BlockHashIterator(filePath, blockSize, algorithm));

            return treeHash;

        }

        private static IEnumerable<Task> BlockHashIterator(string filePath, int blockSize, string algorithm)

        {

            using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, true))

            {

                //Calculate the blocks asyncrhonously

                var hashes= CalculateBlockHashesAsync(stream, blockSize, algorithm);

                yield return hashes;

                //And then proceed with creating and returning a TreeHash

                var length = stream.Length;                

                var list = hashes.Result.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList();

                var treeHash = new TreeHash(algorithm)

                {

                    Bytes = length, 

                    BlockSize = blockSize, 

                    Hashes = list

                };

                yield return Task.Factory.FromResult(treeHash);

            }

        }

        //The Task Iterator style allows the execution of a sequence of patterns using

        //iterator syntax.

        //This particular variation returns the result of the last task, if there is one

        public static Task<TResult> Iterate<TResult>(IEnumerable<Task> asyncIterator)

        {

            if (asyncIterator == null) 

                throw new ArgumentNullException("asyncIterator");

            var enumerator = asyncIterator.GetEnumerator();

            if (enumerator == null)

                throw new InvalidOperationException("Invalid enumerable - GetEnumerator returned null");

            var tcs = new TaskCompletionSource<TResult>();

            tcs.Task.ContinueWith(_ => enumerator.Dispose(), TaskContinuationOptions.ExecuteSynchronously);

            Action<Task> recursiveBody = null;

            recursiveBody = delegate

            {

                try

                {

                    if (enumerator.MoveNext())

                        enumerator.Current.ContinueWith(recursiveBody,

                                                        TaskContinuationOptions.ExecuteSynchronously);

                    else

                    {

                        var lastTask = enumerator.Current as Task<TResult>;

                        var result = (lastTask !=null ) 

                            ? lastTask.Result

                            : default(TResult);

                        tcs.TrySetResult(result);

                    }

                }

                catch (Exception exc)

                {

                    tcs.TrySetException(exc);

                }

            };

            recursiveBody(null);

            return tcs.Task;

        }

        /*  public static byte[] CalculateTopHash(IEnumerable<byte[]> hashMap, string algorithm)

        {

            if (hashMap == null)

                throw new ArgumentNullException("hashMap");

            if (String.IsNullOrWhiteSpace(algorithm))

                throw new ArgumentNullException("algorithm");

            Contract.EndContractBlock();

            var hashCount = hashMap.Count();

            if (hashCount == 0)

                return null;

            using (var hasher = HashAlgorithm.Create(algorithm))

            {

                var i = 0;

                var count = hashCount;

                foreach (var block in hashMap)

                {

                    if (i++ != count - 1)

                        hasher.TransformBlock(block, 0, block.Length, null, 0);

                    else

                        hasher.TransformFinalBlock(block, 0, block.Length);

                }

                var finalHash = hasher.Hash;

                return finalHash;

            }

        }*/

        public static byte[] CalculateTopHash(IList<byte[]> hashMap, string algorithm)

        {

            if (hashMap == null)

                throw new ArgumentNullException("hashMap");

            if (String.IsNullOrWhiteSpace(algorithm))

                throw new ArgumentNullException("algorithm");

            Contract.EndContractBlock();            

            var hashCount = hashMap.Count;

            //The tophash of an empty hashmap is an empty array

            if (hashCount == 0)

                return new byte[0];

            //The tophash of a one-item hashmap is the hash itself

            if (hashCount == 1)

                return hashMap[0];

            //Calculate the required number of leaf nodes

            var leafs =(int)Math.Pow(2, Math.Ceiling(Math.Log(hashCount,2)));

            //The size of all nodes is the same and equal to the size of the input hashes

            var hashSize = hashMap[0].Length;

            //If the hashmap containes fewer nodes than the required leaf count, we need to fill

            //the rest with empty blocks

            byte[] empty=null;            

            if (hashCount < leafs)

                empty = new byte[hashSize];

            //New hashes will be stored in a dictionary keyed by their step to preserve order

            var newHashes=new ConcurrentDictionary<int, byte[]>();            

            Parallel.For(0, leafs/2,

                (step, state) =>

                {

                    using (var hasher = HashAlgorithm.Create(algorithm))

                    {

                        var i = step*2;

                        var block1 = i <= hashCount - 1 ? hashMap[i] : empty;

                        var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty;

                        hasher.TransformBlock(block1, 0, block1.Length, null, 0);

                        hasher.TransformFinalBlock(block2, 0, block2.Length);

                        var finalHash = hasher.Hash;

                        //Store the final value in its proper place

                        newHashes[step] = finalHash;

                    }

                });

/*

            Parallel.For(0, leafs/2,

                () => HashAlgorithm.Create(algorithm),

                (step, state, hasher) =>

                {

                    var i = step*2;

                    var block1 = i <= hashCount - 1 ? hashMap[i]: empty;

                    var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty;

                    hasher.TransformBlock(block1, 0, block1.Length, null, 0);

                    hasher.TransformFinalBlock(block2, 0, block2.Length);

                    var finalHash = hasher.Hash;

                    //Store the final value in its proper place

                    newHashes[step] = finalHash;

                    return hasher;

                },

                hasher => hasher.Dispose());

*/

            //Extract the hashes to a list ordered by their step 

            var hashes = newHashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList();

            return CalculateTopHash(hashes, algorithm);                   

        }

      /*  public static string CalculateTopHash(string hashString, string algorithm)

        {

            if (String.IsNullOrWhiteSpace(algorithm))

                throw new ArgumentNullException("algorithm");

            Contract.EndContractBlock();

            if (String.IsNullOrWhiteSpace(hashString))

                return String.Empty;

            using (var hasher = HashAlgorithm.Create(algorithm))

            {

                var bytes=Encoding.ASCII.GetBytes(hashString.ToLower());

                var hash=hasher.ComputeHash(bytes);

                return hash.ToHashString();

            }

        }*/

        private static Task<ConcurrentDictionary<int, byte[]>> CalculateBlockHashesAsync(FileStream stream, int blockSize, string algorithm, ConcurrentDictionary<int, byte[]> hashes=null, int index = 0)

        {

            if (stream==null)

                throw new ArgumentNullException("stream");

            if (String.IsNullOrWhiteSpace(algorithm))

                throw new ArgumentNullException("algorithm");

            if (blockSize <= 0)

                throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");

            if (index< 0)

                throw new ArgumentOutOfRangeException("index", "index must be a non-negative value");

            Contract.EndContractBlock();

            if (hashes==null)

                hashes= new ConcurrentDictionary<int, byte[]>();

            var buffer = new byte[blockSize];

            return stream.ReadAsync(buffer, 0, blockSize).ContinueWith(t =>

            {

                var read = t.Result;

                var nextTask = read == blockSize

                                    ? CalculateBlockHashesAsync(stream, blockSize, algorithm, hashes, index + 1) 

                                    : Task.Factory.StartNew(() => hashes);

                if (read>0)

                    using (var hasher = HashAlgorithm.Create(algorithm))

                    {

                        //This code was added for compatibility with the way Pithos calculates the last hash

                        //We calculate the hash only up to the last non-null byte

                        //TODO: Remove if the server starts using the full block instead of the trimmed block

                        var lastByteIndex = Array.FindLastIndex(buffer, read - 1, aByte => aByte != 0);

                        var hash = hasher.ComputeHash(buffer, 0, lastByteIndex+1);

                        hashes[index]=hash;

                    }

                return nextTask;

            }).Unwrap();

        }

        public static byte[] CalculateHash(byte[] buffer,string algorithm)

        {

            if (buffer == null)

                throw new ArgumentNullException("buffer");

            if (String.IsNullOrWhiteSpace(algorithm))

                throw new ArgumentNullException("algorithm");

            Contract.EndContractBlock();

            using (var hasher = HashAlgorithm.Create(algorithm))

            {

                var hash = hasher.ComputeHash(buffer, 0, buffer.Length);

                return hash;

            }        

        }

    }

}