code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/Signature.cs

   1 using System;
   2 using System.Collections.Concurrent;
   3 using System.Collections.Generic;
   4 using System.Diagnostics.Contracts;
   5 using System.IO;
   6 using System.Runtime.Remoting.Metadata.W3cXsd2001;
   7 using System.Security.Cryptography;
   8 using System.Text;
   9 using System.Threading.Tasks;
  10 using System.Linq;
  11
  12 namespace Pithos.Network
  13 {
  14     public static class Signature
  15     {
  16         public static string CalculateMD5(FileInfo info)
  17         {
  18             if (info==null)
  19                 throw new ArgumentNullException("info");
  20             if (String.IsNullOrWhiteSpace(info.FullName))
  21                 throw new ArgumentException("info.FullName is empty","info");
  22             Contract.EndContractBlock();
  23
  24             return CalculateMD5(info.FullName);
  25         }
  26
  27         public static string CalculateMD5(string path)
  28         {
  29             if (String.IsNullOrWhiteSpace(path))
  30                 throw new ArgumentNullException("path");
  31             Contract.EndContractBlock();
  32
  33             //DON'T calculate hashes for folders
  34             if (Directory.Exists(path))
  35                 return "";
  36
  37             string hash;
  38             using (var hasher = MD5.Create())
  39             using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
  40             {
  41                 var hashBytes = hasher.ComputeHash(stream);
  42                 hash = hashBytes.ToHashString();
  43             }
  44             return hash;
  45         }
  46
  47 /*
  48         public static string BytesToString(byte[] hashBytes)
  49         {
  50             var shb=new SoapHexBinary(hashBytes);
  51             return shb.ToString();
  52
  53         }
  54
  55
  56         public static byte[] StringToBytes(string hash)
  57         {
  58             var shb=SoapHexBinary.Parse(hash);
  59             return shb.Value;
  60         }
  61 */
  62
  63         public static byte[] ToBytes(this string hash)
  64         {
  65             var shb = SoapHexBinary.Parse(hash);
  66             return shb.Value;
  67         }
  68
  69         public static string ToHashString(this byte[] hashBytes)
  70         {
  71             var shb = new SoapHexBinary(hashBytes);
  72             return shb.ToString().ToLower();
  73         }
  74
  75         public static TreeHash CalculateTreeHash(FileInfo fileInfo, int blockSize, string algorithm)
  76         {
  77             if (fileInfo==null)
  78                 throw new ArgumentNullException("fileInfo");
  79             if (String.IsNullOrWhiteSpace(fileInfo.FullName))
  80                 throw new ArgumentException("fileInfo.FullName is empty","fileInfo");
  81             if (blockSize <= 0)
  82                 throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
  83             if (String.IsNullOrWhiteSpace(algorithm))
  84                 throw new ArgumentNullException("algorithm");
  85             Contract.EndContractBlock();
  86
  87             return CalculateTreeHash(fileInfo.FullName, blockSize, algorithm);
  88         }
  89
  90         /// <summary>
  91         /// Calculates a file's tree hash synchronously, using the specified block size
  92         /// </summary>
  93         /// <param name="filePath">Path to an existing file</param>
  94         /// <param name="blockSize">Block size used to calculate leaf hashes</param>
  95         /// <param name="algorithm"></param>
  96         /// <returns>A <see cref="TreeHash"/> with the block hashes and top hash</returns>
  97         public static TreeHash CalculateTreeHash(string filePath, int blockSize, string algorithm)
  98         {
  99             if (String.IsNullOrWhiteSpace(filePath))
 100                 throw new ArgumentNullException("filePath");
 101             if (blockSize<=0)
 102                 throw new ArgumentOutOfRangeException("blockSize","blockSize must be a value greater than zero ");
 103             if (String.IsNullOrWhiteSpace(algorithm))
 104                 throw new ArgumentNullException("algorithm");
 105             Contract.EndContractBlock();
 106
 107             var hash=CalculateTreeHashAsync(filePath, blockSize, algorithm);
 108             return hash.Result;
 109         }
 110
 111         public static async Task<TreeHash> CalculateTreeHashAsync(FileInfo fileInfo, int blockSize, string algorithm)
 112         {
 113             if (fileInfo == null)
 114                 throw new ArgumentNullException("fileInfo");
 115             if (String.IsNullOrWhiteSpace(fileInfo.FullName))
 116                 throw new ArgumentNullException("fileInfo.FullName is empty","fileInfo");
 117             if (blockSize <= 0)
 118                 throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
 119             if (String.IsNullOrWhiteSpace(algorithm))
 120                 throw new ArgumentNullException("algorithm");
 121             Contract.EndContractBlock();
 122
 123             return await CalculateTreeHashAsync(fileInfo.FullName, blockSize, algorithm);
 124         }
 125
 126
 127         public static async Task<TreeHash> CalculateTreeHashAsync(string filePath, int blockSize,string algorithm)
 128         {
 129             if (String.IsNullOrWhiteSpace(filePath))
 130                 throw new ArgumentNullException("filePath");
 131             if (blockSize <= 0)
 132                 throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
 133             if (String.IsNullOrWhiteSpace(algorithm))
 134                 throw new ArgumentNullException("algorithm");
 135             Contract.EndContractBlock();
 136
 137             //DON'T calculate hashes for folders
 138             if (Directory.Exists(filePath))
 139                 return new TreeHash(algorithm);
 140             //The hash of a non-existent file is the empty hash
 141             if (!File.Exists(filePath))
 142                 return new TreeHash(algorithm);
 143
 144             //Calculate the hash of all blocks using a blockhash iterator
 145             using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, true))
 146             {
 147                 //Calculate the blocks asyncrhonously
 148                 var hashes = await CalculateBlockHashesAsync(stream, blockSize, algorithm);
 149
 150                 //And then proceed with creating and returning a TreeHash
 151                 var length = stream.Length;
 152                 var list = hashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList();
 153
 154                 var treeHash = new TreeHash(algorithm)
 155                 {
 156                     Bytes = length,
 157                     BlockSize = blockSize,
 158                     Hashes = list
 159                 };
 160
 161                 return treeHash;
 162             }
 163         }
 164
 165
 166         //The Task Iterator style allows the execution of a sequence of patterns using
 167         //iterator syntax.
 168         //This particular variation returns the result of the last task, if there is one
 169         public static Task<TResult> Iterate<TResult>(IEnumerable<Task> asyncIterator)
 170         {
 171             if (asyncIterator == null)
 172                 throw new ArgumentNullException("asyncIterator");
 173             var enumerator = asyncIterator.GetEnumerator();
 174             if (enumerator == null)
 175                 throw new InvalidOperationException("Invalid enumerable - GetEnumerator returned null");
 176
 177             var tcs = new TaskCompletionSource<TResult>();
 178             tcs.Task.ContinueWith(_ => enumerator.Dispose(), TaskContinuationOptions.ExecuteSynchronously);
 179
 180             Action<Task> recursiveBody = null;
 181             recursiveBody = delegate
 182             {
 183                 try
 184                 {
 185                     if (enumerator.MoveNext())
 186                         enumerator.Current.ContinueWith(recursiveBody,
 187                                                         TaskContinuationOptions.ExecuteSynchronously);
 188                     else
 189                     {
 190                         var lastTask = enumerator.Current as Task<TResult>;
 191                         var result = (lastTask !=null )
 192                             ? lastTask.Result
 193                             : default(TResult);
 194
 195                         tcs.TrySetResult(result);
 196                     }
 197                 }
 198                 catch (Exception exc)
 199                 {
 200                     tcs.TrySetException(exc);
 201                 }
 202             };
 203             recursiveBody(null);
 204             return tcs.Task;
 205         }
 206
 207
 208         /*  public static byte[] CalculateTopHash(IEnumerable<byte[]> hashMap, string algorithm)
 209         {
 210             if (hashMap == null)
 211                 throw new ArgumentNullException("hashMap");
 212             if (String.IsNullOrWhiteSpace(algorithm))
 213                 throw new ArgumentNullException("algorithm");
 214             Contract.EndContractBlock();
 215
 216             var hashCount = hashMap.Count();
 217             if (hashCount == 0)
 218                 return null;
 219             using (var hasher = HashAlgorithm.Create(algorithm))
 220             {
 221                 var i = 0;
 222                 var count = hashCount;
 223                 foreach (var block in hashMap)
 224                 {
 225                     if (i++ != count - 1)
 226                         hasher.TransformBlock(block, 0, block.Length, null, 0);
 227                     else
 228                         hasher.TransformFinalBlock(block, 0, block.Length);
 229                 }
 230
 231                 var finalHash = hasher.Hash;
 232
 233                 return finalHash;
 234             }
 235         }*/
 236
 237         public static byte[] CalculateTopHash(IList<byte[]> hashMap, string algorithm)
 238         {
 239             if (hashMap == null)
 240                 throw new ArgumentNullException("hashMap");
 241             if (String.IsNullOrWhiteSpace(algorithm))
 242                 throw new ArgumentNullException("algorithm");
 243             Contract.EndContractBlock();
 244
 245             var hashCount = hashMap.Count;
 246             //The tophash of an empty hashmap is an empty array
 247             if (hashCount == 0)
 248                 return new byte[0];
 249             //The tophash of a one-item hashmap is the hash itself
 250             if (hashCount == 1)
 251                 return hashMap[0];
 252
 253             //Calculate the required number of leaf nodes
 254             var leafs =(int)Math.Pow(2, Math.Ceiling(Math.Log(hashCount,2)));
 255             //The size of all nodes is the same and equal to the size of the input hashes
 256             var hashSize = hashMap[0].Length;
 257
 258             //If the hashmap containes fewer nodes than the required leaf count, we need to fill
 259             //the rest with empty blocks
 260             byte[] empty=null;
 261             if (hashCount < leafs)
 262                 empty = new byte[hashSize];
 263
 264             //New hashes will be stored in a dictionary keyed by their step to preserve order
 265             var newHashes=new ConcurrentDictionary<int, byte[]>();
 266
 267             Parallel.For(0, leafs/2,
 268                 (step, state) =>
 269                 {
 270                     using (var hasher = HashAlgorithm.Create(algorithm))
 271                     {
 272                         var i = step*2;
 273                         var block1 = i <= hashCount - 1 ? hashMap[i] : empty;
 274                         var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty;
 275
 276                         hasher.TransformBlock(block1, 0, block1.Length, null, 0);
 277                         hasher.TransformFinalBlock(block2, 0, block2.Length);
 278
 279                         var finalHash = hasher.Hash;
 280                         //Store the final value in its proper place
 281                         newHashes[step] = finalHash;
 282                     }
 283                 });
 284 /*
 285             Parallel.For(0, leafs/2,
 286                 () => HashAlgorithm.Create(algorithm),
 287                 (step, state, hasher) =>
 288                 {
 289
 290                     var i = step*2;
 291                     var block1 = i <= hashCount - 1 ? hashMap[i]: empty;
 292                     var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty;
 293
 294                     hasher.TransformBlock(block1, 0, block1.Length, null, 0);
 295                     hasher.TransformFinalBlock(block2, 0, block2.Length);
 296
 297                     var finalHash = hasher.Hash;
 298                     //Store the final value in its proper place
 299                     newHashes[step] = finalHash;
 300
 301                     return hasher;
 302                 },
 303                 hasher => hasher.Dispose());
 304 */
 305             //Extract the hashes to a list ordered by their step
 306             var hashes = newHashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList();
 307             return CalculateTopHash(hashes, algorithm);
 308         }
 309
 310
 311
 312       /*  public static string CalculateTopHash(string hashString, string algorithm)
 313         {
 314             if (String.IsNullOrWhiteSpace(algorithm))
 315                 throw new ArgumentNullException("algorithm");
 316             Contract.EndContractBlock();
 317             if (String.IsNullOrWhiteSpace(hashString))
 318                 return String.Empty;
 319
 320             using (var hasher = HashAlgorithm.Create(algorithm))
 321             {
 322                 var bytes=Encoding.ASCII.GetBytes(hashString.ToLower());
 323                 var hash=hasher.ComputeHash(bytes);
 324                 return hash.ToHashString();
 325             }
 326         }*/
 327
 328         private static Task<ConcurrentDictionary<int, byte[]>> CalculateBlockHashesAsync(FileStream stream, int blockSize, string algorithm, ConcurrentDictionary<int, byte[]> hashes=null, int index = 0)
 329         {
 330             if (stream==null)
 331                 throw new ArgumentNullException("stream");
 332             if (String.IsNullOrWhiteSpace(algorithm))
 333                 throw new ArgumentNullException("algorithm");
 334             if (blockSize <= 0)
 335                 throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
 336             if (index< 0)
 337                 throw new ArgumentOutOfRangeException("index", "index must be a non-negative value");
 338             Contract.EndContractBlock();
 339
 340
 341             if (hashes==null)
 342                 hashes= new ConcurrentDictionary<int, byte[]>();
 343
 344             var buffer = new byte[blockSize];
 345             return stream.ReadAsync(buffer, 0, blockSize).ContinueWith(t =>
 346             {
 347                 var read = t.Result;
 348
 349                 var nextTask = read == blockSize
 350                                     ? CalculateBlockHashesAsync(stream, blockSize, algorithm, hashes, index + 1)
 351                                     : Task.Factory.StartNew(() => hashes);
 352                 if (read>0)
 353                     using (var hasher = HashAlgorithm.Create(algorithm))
 354                     {
 355                         //This code was added for compatibility with the way Pithos calculates the last hash
 356                         //We calculate the hash only up to the last non-null byte
 357                         //TODO: Remove if the server starts using the full block instead of the trimmed block
 358                         var lastByteIndex = Array.FindLastIndex(buffer, read - 1, aByte => aByte != 0);
 359
 360                         var hash = hasher.ComputeHash(buffer, 0, lastByteIndex+1);
 361                         hashes[index]=hash;
 362                     }
 363                 return nextTask;
 364             }).Unwrap();
 365         }
 366
 367
 368
 369         public static byte[] CalculateHash(byte[] buffer,string algorithm)
 370         {
 371             if (buffer == null)
 372                 throw new ArgumentNullException("buffer");
 373             if (String.IsNullOrWhiteSpace(algorithm))
 374                 throw new ArgumentNullException("algorithm");
 375             Contract.EndContractBlock();
 376
 377             using (var hasher = HashAlgorithm.Create(algorithm))
 378             {
 379                 var hash = hasher.ComputeHash(buffer, 0, buffer.Length);
 380                 return hash;
 381             }
 382         }
 383     }
 384 }
 385
 386
 387