#region
/* -----------------------------------------------------------------------
*
*
* Copyright 2011-2012 GRNET S.A. All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* 1. Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* 2. Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
*
* THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and
* documentation are those of the authors and should not be
* interpreted as representing official policies, either expressed
* or implied, of GRNET S.A.
*
* -----------------------------------------------------------------------
*/
#endregion
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.IO;
using System.Reflection;
using System.Runtime.Remoting.Metadata.W3cXsd2001;
using System.Threading;
using System.Threading.Tasks;
using System.Linq;
using Pithos.Interfaces;
using log4net;
using OpenSSL.Core;
using OpenSSL.Crypto;
namespace Pithos.Network
{
public static class Signature
{
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
public const int BufferSize = 16384;
public const string MD5_EMPTY = "d41d8cd98f00b204e9800998ecf8427e";
public const string MERKLE_EMPTY = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
public static string CalculateMD5(FileSystemInfo info)
{
if (info==null)
throw new ArgumentNullException("info");
if (String.IsNullOrWhiteSpace(info.FullName))
throw new ArgumentException("info.FullName is empty","info");
Contract.EndContractBlock();
if (info is DirectoryInfo)
return MD5_EMPTY;
return CalculateMD5(info.FullName);
}
public static string CalculateMD5(string path)
{
if (String.IsNullOrWhiteSpace(path))
throw new ArgumentNullException("path");
Contract.EndContractBlock();
//DON'T calculate hashes for folders
if (Directory.Exists(path))
return "";
//TODO: Replace with MD5BlockCalculator
string hash;
using (var hasher = new MessageDigestContext(MessageDigest.CreateByName("md5")))
using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize, true))
{
var buffer = new byte[BufferSize];
hasher.Init();
int read;
do
{
read = stream.Read(buffer, 0, buffer.Length);
if (read==buffer.Length)
{
hasher.Update(buffer);
}
else
{
var block = new byte[read];
Buffer.BlockCopy(buffer, 0, block, 0, read);
hasher.Update(block);
}
} while (read>0);
var hashBytes = hasher.DigestFinal();
hash = hashBytes.ToHashString();
}
return hash;
}
/*
public static string BytesToString(byte[] hashBytes)
{
var shb=new SoapHexBinary(hashBytes);
return shb.ToString();
}
public static byte[] StringToBytes(string hash)
{
var shb=SoapHexBinary.Parse(hash);
return shb.Value;
}
*/
public static byte[] ToBytes(this string hash)
{
var shb = SoapHexBinary.Parse(hash);
return shb.Value;
}
public static string ToHashString(this byte[] hashBytes)
{
var shb = new SoapHexBinary(hashBytes);
return shb.ToString().ToLower();
}
public static TreeHash CalculateTreeHash(FileSystemInfo fileInfo, int blockSize, string algorithm,CancellationToken token,IProgress progress )
{
if (fileInfo == null)
throw new ArgumentNullException("fileInfo");
if (String.IsNullOrWhiteSpace(fileInfo.FullName))
throw new ArgumentException("fileInfo.FullName is empty", "fileInfo");
if (blockSize <= 0)
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
fileInfo.Refresh();
if (fileInfo is DirectoryInfo || !fileInfo.Exists)
return TreeHash.Empty;
return CalculateTreeHashAsync(fileInfo, blockSize, algorithm, 1, token, progress);
}
///
/// Calculates a file's tree hash synchronously, using the specified block size
///
/// Path to an existing file
/// Block size used to calculate leaf hashes
///
/// A with the block hashes and top hash
public static TreeHash CalculateTreeHash(string filePath, int blockSize, string algorithm,CancellationToken token,IProgress progress )
{
if (String.IsNullOrWhiteSpace(filePath))
throw new ArgumentNullException("filePath");
if (blockSize<=0)
throw new ArgumentOutOfRangeException("blockSize","blockSize must be a value greater than zero ");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
var info = FileInfoExtensions.FromPath(filePath);
var hash=CalculateTreeHashAsync(info, blockSize, algorithm, 1,token,progress);
return hash;
}
public static TreeHash CalculateTreeHashAsync(string filePath, int blockSize, string algorithm, byte parallelism,CancellationToken token,IProgress progress )
{
if (filePath== null)
throw new ArgumentNullException("filePath");
if (blockSize <= 0)
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
var info = FileInfoExtensions.FromPath(filePath);
return CalculateTreeHashAsync(info, blockSize, algorithm, parallelism,token,progress);
}
public static TreeHash CalculateTreeHashAsync(FileSystemInfo info, int blockSize,string algorithm, int parallelism,CancellationToken token,IProgress progress )
{
if (info==null)
throw new ArgumentNullException("info");
if (blockSize <= 0)
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
var filePath = info.FullName;
if (Log.IsDebugEnabled)
Log.DebugFormat("Calc Signature [{0}]",filePath);
if (filePath.Split('/').Contains(".pithos.cache"))
throw new ArgumentException(String.Format("Trying to hash file from the cache folder: [{0}]",filePath));
//DON'T calculate hashes for folders
if (Directory.Exists(filePath))
return new TreeHash(algorithm);
//The hash of a non-existent file is the empty hash
if (!File.Exists(filePath))
return new TreeHash(algorithm);
if (info is FileInfo && (info as FileInfo).Length==0)
return new TreeHash(algorithm);
//Calculate the hash of all blocks using a blockhash iterator
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, BufferSize, true))
{
var md5 = new MD5BlockCalculator();
Action postAction = md5.PostBlock;
//Calculate the blocks asyncrhonously
var hashes = BlockHashAlgorithms.CalculateBlockHashesInPlacePFor(stream, blockSize, algorithm, parallelism,postAction,token, progress).Result;
//And then proceed with creating and returning a TreeHash
var length = stream.Length;
var list = hashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList();
var treeHash = new TreeHash(algorithm)
{
Bytes = length,
BlockSize = blockSize,
Hashes = list,
};
string fileHash;
var md5Hash=md5.GetHash().Result;
treeHash.MD5= md5Hash;
return treeHash;
}
}
public static byte[] CalculateTopHash(IList hashMap, string algorithm)
{
if (hashMap == null)
throw new ArgumentNullException("hashMap");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
var hashCount = hashMap.Count;
//TODO: Replace this calculation with a constant
//The tophash of an empty hashmap is the hash of an empty array
if (hashCount == 0)
{
using (var hasher = new MessageDigestContext(MessageDigest.CreateByName(algorithm)))
{
hasher.Init();
var emptyHash=hasher.Digest(new byte[0]);
return emptyHash;
}
}
//The tophash of a one-item hashmap is the hash itself
if (hashCount == 1)
return hashMap[0];
//Calculate the required number of leaf nodes
var leafs =(int)Math.Pow(2, Math.Ceiling(Math.Log(hashCount,2)));
//The size of all nodes is the same and equal to the size of the input hashes
var hashSize = hashMap[0].Length;
//If the hashmap containes fewer nodes than the required leaf count, we need to fill
//the rest with empty blocks
byte[] empty=null;
if (hashCount < leafs)
empty = new byte[hashSize];
//New hashes will be stored in a dictionary keyed by their step to preserve order
var newHashes=new ConcurrentDictionary();
Parallel.For(0, leafs/2,
(step, state) =>
{
using (var hasher = new MessageDigestContext(MessageDigest.CreateByName(algorithm)))
{
hasher.Init();
var i = step*2;
var block1 = i <= hashCount - 1 ? hashMap[i] : empty;
var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty;
hasher.Update(block1);
hasher.Update(block2);
var finalHash = hasher.DigestFinal();
//Store the final value in its proper place
newHashes[step] = finalHash;
}
});
//Extract the hashes to a list ordered by their step
var hashes = newHashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList();
return CalculateTopHash(hashes, algorithm);
}
public static byte[] CalculateHash(byte[] buffer,string algorithm)
{
if (buffer == null)
throw new ArgumentNullException("buffer");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
using (var hasher = new MessageDigestContext(MessageDigest.CreateByName(algorithm)))
{
hasher.Init();
var hash = hasher.Digest(buffer);
return hash;
}
}
}
}