root / trunk / Pithos.Network / Signature.cs @ a64c87c8
History | View | Annotate | Download (16.4 kB)
1 |
using System; |
---|---|
2 |
using System.Collections.Concurrent; |
3 |
using System.Collections.Generic; |
4 |
using System.Diagnostics.Contracts; |
5 |
using System.IO; |
6 |
using System.Runtime.Remoting.Metadata.W3cXsd2001; |
7 |
using System.Security.Cryptography; |
8 |
using System.Text; |
9 |
using System.Threading.Tasks; |
10 |
using System.Linq; |
11 |
|
12 |
namespace Pithos.Network |
13 |
{ |
14 |
public static class Signature |
15 |
{ |
16 |
public static string CalculateMD5(FileInfo info) |
17 |
{ |
18 |
if (info==null) |
19 |
throw new ArgumentNullException("info"); |
20 |
Contract.EndContractBlock(); |
21 |
|
22 |
return CalculateMD5(info.FullName); |
23 |
} |
24 |
|
25 |
public static string CalculateMD5(string path) |
26 |
{ |
27 |
if (String.IsNullOrWhiteSpace(path)) |
28 |
throw new ArgumentNullException("path"); |
29 |
Contract.EndContractBlock(); |
30 |
|
31 |
//DON'T calculate hashes for folders |
32 |
if (Directory.Exists(path)) |
33 |
return ""; |
34 |
|
35 |
string hash; |
36 |
using (var hasher = MD5.Create()) |
37 |
using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) |
38 |
{ |
39 |
var hashBytes = hasher.ComputeHash(stream); |
40 |
hash = hashBytes.ToHashString(); |
41 |
} |
42 |
return hash; |
43 |
} |
44 |
|
45 |
/* |
46 |
public static string BytesToString(byte[] hashBytes) |
47 |
{ |
48 |
var shb=new SoapHexBinary(hashBytes); |
49 |
return shb.ToString(); |
50 |
|
51 |
} |
52 |
|
53 |
|
54 |
public static byte[] StringToBytes(string hash) |
55 |
{ |
56 |
var shb=SoapHexBinary.Parse(hash); |
57 |
return shb.Value; |
58 |
} |
59 |
*/ |
60 |
|
61 |
public static byte[] ToBytes(this string hash) |
62 |
{ |
63 |
var shb = SoapHexBinary.Parse(hash); |
64 |
return shb.Value; |
65 |
} |
66 |
|
67 |
public static string ToHashString(this byte[] hashBytes) |
68 |
{ |
69 |
var shb = new SoapHexBinary(hashBytes); |
70 |
return shb.ToString().ToLower(); |
71 |
} |
72 |
|
73 |
public static TreeHash CalculateTreeHash(FileInfo fileInfo, int blockSize, string algorithm) |
74 |
{ |
75 |
if (fileInfo==null) |
76 |
throw new ArgumentNullException("fileInfo"); |
77 |
if (blockSize <= 0) |
78 |
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); |
79 |
if (String.IsNullOrWhiteSpace(algorithm)) |
80 |
throw new ArgumentNullException("algorithm"); |
81 |
Contract.EndContractBlock(); |
82 |
|
83 |
return CalculateTreeHash(fileInfo.FullName, blockSize, algorithm); |
84 |
} |
85 |
|
86 |
/// <summary> |
87 |
/// Calculates a file's tree hash synchronously, using the specified block size |
88 |
/// </summary> |
89 |
/// <param name="filePath">Path to an existing file</param> |
90 |
/// <param name="blockSize">Block size used to calculate leaf hashes</param> |
91 |
/// <param name="algorithm"></param> |
92 |
/// <returns>A <see cref="TreeHash"/> with the block hashes and top hash</returns> |
93 |
public static TreeHash CalculateTreeHash(string filePath, int blockSize, string algorithm) |
94 |
{ |
95 |
if (String.IsNullOrWhiteSpace(filePath)) |
96 |
throw new ArgumentNullException("filePath"); |
97 |
if (blockSize<=0) |
98 |
throw new ArgumentOutOfRangeException("blockSize","blockSize must be a value greater than zero "); |
99 |
if (String.IsNullOrWhiteSpace(algorithm)) |
100 |
throw new ArgumentNullException("algorithm"); |
101 |
Contract.EndContractBlock(); |
102 |
|
103 |
//DON'T calculate hashes for folders |
104 |
if (Directory.Exists(filePath)) |
105 |
return null; |
106 |
|
107 |
|
108 |
var list = new List<byte[]>(); |
109 |
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, false)) |
110 |
using (var hasher = HashAlgorithm.Create(algorithm)) |
111 |
{ |
112 |
var buffer = new byte[blockSize]; |
113 |
int read; |
114 |
while ((read=stream.Read(buffer, 0, blockSize)) > 0) |
115 |
{ |
116 |
//This code was added for compatibility with the way Pithos calculates the last hash |
117 |
//We calculate the hash only up to the last non-null byte |
118 |
//TODO: Remove if the server starts using the full block instead of the trimmed block |
119 |
var lastByteIndex=Array.FindLastIndex(buffer,read-1, aByte => aByte != 0); |
120 |
|
121 |
var hash = hasher.ComputeHash(buffer, 0, lastByteIndex+1); |
122 |
list.Add(hash); |
123 |
} |
124 |
return new TreeHash(algorithm) { Hashes = list, |
125 |
BlockSize = blockSize, |
126 |
Bytes = stream.Length}; |
127 |
} |
128 |
} |
129 |
|
130 |
public static Task<TreeHash> CalculateTreeHashAsync(FileInfo fileInfo, int blockSize, string algorithm) |
131 |
{ |
132 |
if (fileInfo == null) |
133 |
throw new ArgumentNullException("fileInfo"); |
134 |
if (blockSize <= 0) |
135 |
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); |
136 |
if (String.IsNullOrWhiteSpace(algorithm)) |
137 |
throw new ArgumentNullException("algorithm"); |
138 |
Contract.EndContractBlock(); |
139 |
|
140 |
return CalculateTreeHashAsync(fileInfo.FullName, blockSize, algorithm); |
141 |
} |
142 |
|
143 |
|
144 |
public static Task<TreeHash> CalculateTreeHashAsync(string filePath, int blockSize,string algorithm) |
145 |
{ |
146 |
if (String.IsNullOrWhiteSpace(filePath)) |
147 |
throw new ArgumentNullException("filePath"); |
148 |
if (blockSize <= 0) |
149 |
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); |
150 |
if (String.IsNullOrWhiteSpace(algorithm)) |
151 |
throw new ArgumentNullException("algorithm"); |
152 |
Contract.EndContractBlock(); |
153 |
|
154 |
//DON'T calculate hashes for folders |
155 |
if (Directory.Exists(filePath)) |
156 |
return Task.Factory.StartNew(()=>new TreeHash(algorithm)); |
157 |
//The hash of a non-existent file is the empty hash |
158 |
if (!File.Exists(filePath)) |
159 |
return Task.Factory.StartNew(()=>new TreeHash(algorithm)); |
160 |
|
161 |
//Calculate the hash of all blocks using a blockhash iterator |
162 |
var treeHash =Iterate<TreeHash>(BlockHashIterator(filePath, blockSize, algorithm)); |
163 |
|
164 |
return treeHash; |
165 |
} |
166 |
|
167 |
|
168 |
private static IEnumerable<Task> BlockHashIterator(string filePath, int blockSize, string algorithm) |
169 |
{ |
170 |
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, true)) |
171 |
{ |
172 |
//Calculate the blocks asyncrhonously |
173 |
var hashes= CalculateBlockHashesAsync(stream, blockSize, algorithm); |
174 |
yield return hashes; |
175 |
|
176 |
//And then proceed with creating and returning a TreeHash |
177 |
var length = stream.Length; |
178 |
var list = hashes.Result.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList(); |
179 |
|
180 |
var treeHash = new TreeHash(algorithm) |
181 |
{ |
182 |
Bytes = length, |
183 |
BlockSize = blockSize, |
184 |
Hashes = list |
185 |
}; |
186 |
|
187 |
yield return Task.Factory.FromResult(treeHash); |
188 |
} |
189 |
|
190 |
} |
191 |
|
192 |
//The Task Iterator style allows the execution of a sequence of patterns using |
193 |
//iterator syntax. |
194 |
//This particular variation returns the result of the last task, if there is one |
195 |
public static Task<TResult> Iterate<TResult>(IEnumerable<Task> asyncIterator) |
196 |
{ |
197 |
if (asyncIterator == null) |
198 |
throw new ArgumentNullException("asyncIterator"); |
199 |
var enumerator = asyncIterator.GetEnumerator(); |
200 |
if (enumerator == null) |
201 |
throw new InvalidOperationException("Invalid enumerable - GetEnumerator returned null"); |
202 |
|
203 |
var tcs = new TaskCompletionSource<TResult>(); |
204 |
tcs.Task.ContinueWith(_ => enumerator.Dispose(), TaskContinuationOptions.ExecuteSynchronously); |
205 |
|
206 |
Action<Task> recursiveBody = null; |
207 |
recursiveBody = delegate |
208 |
{ |
209 |
try |
210 |
{ |
211 |
if (enumerator.MoveNext()) |
212 |
enumerator.Current.ContinueWith(recursiveBody, |
213 |
TaskContinuationOptions.ExecuteSynchronously); |
214 |
else |
215 |
{ |
216 |
var lastTask = enumerator.Current as Task<TResult>; |
217 |
var result = (lastTask !=null ) |
218 |
? lastTask.Result |
219 |
: default(TResult); |
220 |
|
221 |
tcs.TrySetResult(result); |
222 |
} |
223 |
} |
224 |
catch (Exception exc) |
225 |
{ |
226 |
tcs.TrySetException(exc); |
227 |
} |
228 |
}; |
229 |
recursiveBody(null); |
230 |
return tcs.Task; |
231 |
} |
232 |
|
233 |
|
234 |
/* public static byte[] CalculateTopHash(IEnumerable<byte[]> hashMap, string algorithm) |
235 |
{ |
236 |
if (hashMap == null) |
237 |
throw new ArgumentNullException("hashMap"); |
238 |
if (String.IsNullOrWhiteSpace(algorithm)) |
239 |
throw new ArgumentNullException("algorithm"); |
240 |
Contract.EndContractBlock(); |
241 |
|
242 |
var hashCount = hashMap.Count(); |
243 |
if (hashCount == 0) |
244 |
return null; |
245 |
using (var hasher = HashAlgorithm.Create(algorithm)) |
246 |
{ |
247 |
var i = 0; |
248 |
var count = hashCount; |
249 |
foreach (var block in hashMap) |
250 |
{ |
251 |
if (i++ != count - 1) |
252 |
hasher.TransformBlock(block, 0, block.Length, null, 0); |
253 |
else |
254 |
hasher.TransformFinalBlock(block, 0, block.Length); |
255 |
} |
256 |
|
257 |
var finalHash = hasher.Hash; |
258 |
|
259 |
return finalHash; |
260 |
} |
261 |
}*/ |
262 |
|
263 |
public static byte[] CalculateTopHash(IList<byte[]> hashMap, string algorithm) |
264 |
{ |
265 |
if (hashMap == null) |
266 |
throw new ArgumentNullException("hashMap"); |
267 |
if (String.IsNullOrWhiteSpace(algorithm)) |
268 |
throw new ArgumentNullException("algorithm"); |
269 |
Contract.EndContractBlock(); |
270 |
|
271 |
var hashCount = hashMap.Count; |
272 |
//The tophash of an empty hashmap is an empty array |
273 |
if (hashCount == 0) |
274 |
return new byte[0]; |
275 |
//The tophash of a one-item hashmap is the hash itself |
276 |
if (hashCount == 1) |
277 |
return hashMap[0]; |
278 |
|
279 |
//Calculate the required number of leaf nodes |
280 |
var leafs =(int)Math.Pow(2, Math.Ceiling(Math.Log(hashCount,2))); |
281 |
//The size of all nodes is the same and equal to the size of the input hashes |
282 |
var hashSize = hashMap[0].Length; |
283 |
|
284 |
//If the hashmap containes fewer nodes than the required leaf count, we need to fill |
285 |
//the rest with empty blocks |
286 |
byte[] empty=null; |
287 |
if (hashCount < leafs) |
288 |
empty = new byte[hashSize]; |
289 |
|
290 |
//New hashes will be stored in a dictionary keyed by their step to preserve order |
291 |
var newHashes=new ConcurrentDictionary<int, byte[]>(); |
292 |
|
293 |
Parallel.For(0, leafs/2, |
294 |
(step, state) => |
295 |
{ |
296 |
using (var hasher = HashAlgorithm.Create(algorithm)) |
297 |
{ |
298 |
var i = step*2; |
299 |
var block1 = i <= hashCount - 1 ? hashMap[i] : empty; |
300 |
var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty; |
301 |
|
302 |
hasher.TransformBlock(block1, 0, block1.Length, null, 0); |
303 |
hasher.TransformFinalBlock(block2, 0, block2.Length); |
304 |
|
305 |
var finalHash = hasher.Hash; |
306 |
//Store the final value in its proper place |
307 |
newHashes[step] = finalHash; |
308 |
} |
309 |
}); |
310 |
/* |
311 |
Parallel.For(0, leafs/2, |
312 |
() => HashAlgorithm.Create(algorithm), |
313 |
(step, state, hasher) => |
314 |
{ |
315 |
|
316 |
var i = step*2; |
317 |
var block1 = i <= hashCount - 1 ? hashMap[i]: empty; |
318 |
var block2 = i <= hashCount - 2 ? hashMap[i + 1] : empty; |
319 |
|
320 |
hasher.TransformBlock(block1, 0, block1.Length, null, 0); |
321 |
hasher.TransformFinalBlock(block2, 0, block2.Length); |
322 |
|
323 |
var finalHash = hasher.Hash; |
324 |
//Store the final value in its proper place |
325 |
newHashes[step] = finalHash; |
326 |
|
327 |
return hasher; |
328 |
}, |
329 |
hasher => hasher.Dispose()); |
330 |
*/ |
331 |
//Extract the hashes to a list ordered by their step |
332 |
var hashes = newHashes.OrderBy(pair => pair.Key).Select(pair => pair.Value).ToList(); |
333 |
return CalculateTopHash(hashes, algorithm); |
334 |
} |
335 |
|
336 |
|
337 |
|
338 |
/* public static string CalculateTopHash(string hashString, string algorithm) |
339 |
{ |
340 |
if (String.IsNullOrWhiteSpace(algorithm)) |
341 |
throw new ArgumentNullException("algorithm"); |
342 |
Contract.EndContractBlock(); |
343 |
if (String.IsNullOrWhiteSpace(hashString)) |
344 |
return String.Empty; |
345 |
|
346 |
using (var hasher = HashAlgorithm.Create(algorithm)) |
347 |
{ |
348 |
var bytes=Encoding.ASCII.GetBytes(hashString.ToLower()); |
349 |
var hash=hasher.ComputeHash(bytes); |
350 |
return hash.ToHashString(); |
351 |
} |
352 |
}*/ |
353 |
|
354 |
private static Task<ConcurrentDictionary<int, byte[]>> CalculateBlockHashesAsync(FileStream stream, int blockSize, string algorithm, ConcurrentDictionary<int, byte[]> hashes=null, int index = 0) |
355 |
{ |
356 |
if (stream==null) |
357 |
throw new ArgumentNullException("stream"); |
358 |
if (String.IsNullOrWhiteSpace(algorithm)) |
359 |
throw new ArgumentNullException("algorithm"); |
360 |
if (blockSize <= 0) |
361 |
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); |
362 |
if (index< 0) |
363 |
throw new ArgumentOutOfRangeException("index", "index must be a non-negative value"); |
364 |
Contract.EndContractBlock(); |
365 |
|
366 |
|
367 |
if (hashes==null) |
368 |
hashes= new ConcurrentDictionary<int, byte[]>(); |
369 |
|
370 |
var buffer = new byte[blockSize]; |
371 |
return stream.ReadAsync(buffer, 0, blockSize).ContinueWith(t => |
372 |
{ |
373 |
var read = t.Result; |
374 |
|
375 |
var nextTask = read == blockSize |
376 |
? CalculateBlockHashesAsync(stream, blockSize, algorithm, hashes, index + 1) |
377 |
: Task.Factory.StartNew(() => hashes); |
378 |
if (read>0) |
379 |
using (var hasher = HashAlgorithm.Create(algorithm)) |
380 |
{ |
381 |
//This code was added for compatibility with the way Pithos calculates the last hash |
382 |
//We calculate the hash only up to the last non-null byte |
383 |
//TODO: Remove if the server starts using the full block instead of the trimmed block |
384 |
var lastByteIndex = Array.FindLastIndex(buffer, read - 1, aByte => aByte != 0); |
385 |
|
386 |
var hash = hasher.ComputeHash(buffer, 0, lastByteIndex+1); |
387 |
hashes[index]=hash; |
388 |
} |
389 |
return nextTask; |
390 |
}).Unwrap(); |
391 |
} |
392 |
|
393 |
|
394 |
|
395 |
public static byte[] CalculateHash(byte[] buffer,string algorithm) |
396 |
{ |
397 |
if (buffer == null) |
398 |
throw new ArgumentNullException("buffer"); |
399 |
if (String.IsNullOrWhiteSpace(algorithm)) |
400 |
throw new ArgumentNullException("algorithm"); |
401 |
Contract.EndContractBlock(); |
402 |
|
403 |
using (var hasher = HashAlgorithm.Create(algorithm)) |
404 |
{ |
405 |
var hash = hasher.ComputeHash(buffer, 0, buffer.Length); |
406 |
return hash; |
407 |
} |
408 |
} |
409 |
} |
410 |
} |
411 |
|
412 |
|
413 |
|