Revision 174bbb6e trunk/Pithos.Core/Agents/NetworkAgent.cs
b/trunk/Pithos.Core/Agents/NetworkAgent.cs | ||
---|---|---|
89 | 89 |
|
90 | 90 |
public void Start() |
91 | 91 |
{ |
92 |
if (_agent != null) |
|
93 |
return; |
|
94 |
|
|
95 |
if (Log.IsDebugEnabled) |
|
96 |
Log.Debug("Starting Network Agent"); |
|
97 |
|
|
92 | 98 |
_agent = Agent<CloudAction>.Start(inbox => |
93 | 99 |
{ |
94 | 100 |
Action loop = null; |
... | ... | |
123 | 129 |
|
124 | 130 |
try |
125 | 131 |
{ |
132 |
StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Processing"); |
|
126 | 133 |
_proceedEvent.Reset(); |
127 | 134 |
//UpdateStatus(PithosStatus.Syncing); |
128 | 135 |
var accountInfo = action.AccountInfo; |
... | ... | |
204 | 211 |
{ |
205 | 212 |
if (_agent.IsEmpty) |
206 | 213 |
_proceedEvent.Set(); |
207 |
UpdateStatus(PithosStatus.InSynch);
|
|
214 |
UpdateStatus(PithosStatus.LocalComplete);
|
|
208 | 215 |
} |
209 | 216 |
} |
210 | 217 |
} |
... | ... | |
212 | 219 |
|
213 | 220 |
private void UpdateStatus(PithosStatus status) |
214 | 221 |
{ |
215 |
StatusKeeper.SetPithosStatus(status);
|
|
216 |
StatusNotification.Notify(new Notification()); |
|
222 |
StatusNotification.SetPithosStatus(status);
|
|
223 |
//StatusNotification.Notify(new Notification());
|
|
217 | 224 |
} |
218 | 225 |
|
219 | 226 |
private void RenameLocalFile(AccountInfo accountInfo, CloudAction action) |
... | ... | |
297 | 304 |
|
298 | 305 |
var cloudHash = cloudFile.Hash.ToLower(); |
299 | 306 |
var previousCloudHash = cloudFile.PreviousHash.ToLower(); |
300 |
var localHash = action.LocalHash.Value.ToLower(); |
|
301 |
var topHash = action.TopHash.Value.ToLower(); |
|
307 |
var localHash = action.TreeHash.Value.TopHash.ToHashString();// LocalHash.Value.ToLower();
|
|
308 |
//var topHash = action.TopHash.Value.ToLower();
|
|
302 | 309 |
|
303 | 310 |
//At this point we know that an object has changed on the server and that a local |
304 | 311 |
//file already exists. We need to decide whether the file has only changed on |
305 | 312 |
//the server or there is a conflicting change on the client. |
306 | 313 |
// |
307 | 314 |
|
308 |
//Not enough to compare only the local hashes (MD5), also have to compare the tophashes |
|
309 |
//If any of the hashes match, we are done |
|
310 |
if ((cloudHash == localHash || cloudHash == topHash)) |
|
315 |
//If the hashes match, we are done |
|
316 |
if (cloudHash == localHash) |
|
311 | 317 |
{ |
312 | 318 |
Log.InfoFormat("Skipping {0}, hashes match", downloadPath); |
313 | 319 |
return; |
... | ... | |
351 | 357 |
Contract.EndContractBlock(); |
352 | 358 |
|
353 | 359 |
_deleteAgent.ProceedEvent.Wait(); |
360 |
/* |
|
354 | 361 |
|
355 | 362 |
//If the action targets a local file, add a treehash calculation |
356 | 363 |
if (!(cloudAction is CloudDeleteAction) && cloudAction.LocalFile as FileInfo != null) |
357 | 364 |
{ |
358 | 365 |
var accountInfo = cloudAction.AccountInfo; |
359 | 366 |
var localFile = (FileInfo) cloudAction.LocalFile; |
367 |
|
|
360 | 368 |
if (localFile.Length > accountInfo.BlockSize) |
361 | 369 |
cloudAction.TopHash = |
362 | 370 |
new Lazy<string>(() => Signature.CalculateTreeHashAsync(localFile, |
... | ... | |
367 | 375 |
{ |
368 | 376 |
cloudAction.TopHash = new Lazy<string>(() => cloudAction.LocalHash.Value); |
369 | 377 |
} |
378 |
|
|
370 | 379 |
} |
371 | 380 |
else |
372 | 381 |
{ |
373 | 382 |
//The hash for a directory is the empty string |
374 | 383 |
cloudAction.TopHash = new Lazy<string>(() => String.Empty); |
375 | 384 |
} |
385 |
*/ |
|
376 | 386 |
|
377 | 387 |
if (cloudAction is CloudDeleteAction) |
378 | 388 |
_deleteAgent.Post((CloudDeleteAction)cloudAction); |
... | ... | |
497 | 507 |
} |
498 | 508 |
else |
499 | 509 |
{ |
500 |
//Retrieve the hashmap from the server |
|
501 |
var serverHash = await client.GetHashMap(account, container, url); |
|
502 |
//If it's a small file |
|
503 |
if (serverHash.Hashes.Count == 1) |
|
504 |
//Download it in one go |
|
505 |
await |
|
506 |
DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath, |
|
507 |
serverHash); |
|
508 |
//Otherwise download it block by block |
|
509 |
else |
|
510 |
await DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath, serverHash); |
|
511 |
|
|
512 |
if (cloudFile.AllowedTo == "read") |
|
510 |
var isChanged = IsObjectChanged(cloudFile, localPath); |
|
511 |
if (isChanged) |
|
513 | 512 |
{ |
514 |
var attributes = File.GetAttributes(localPath); |
|
515 |
File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly); |
|
513 |
//Retrieve the hashmap from the server |
|
514 |
var serverHash = await client.GetHashMap(account, container, url); |
|
515 |
//If it's a small file |
|
516 |
if (serverHash.Hashes.Count == 1) |
|
517 |
//Download it in one go |
|
518 |
await |
|
519 |
DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath, |
|
520 |
serverHash); |
|
521 |
//Otherwise download it block by block |
|
522 |
else |
|
523 |
await |
|
524 |
DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath, |
|
525 |
serverHash); |
|
526 |
|
|
527 |
if (cloudFile.AllowedTo == "read") |
|
528 |
{ |
|
529 |
var attributes = File.GetAttributes(localPath); |
|
530 |
File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly); |
|
531 |
} |
|
516 | 532 |
} |
517 | 533 |
} |
518 | 534 |
|
... | ... | |
523 | 539 |
} |
524 | 540 |
} |
525 | 541 |
|
542 |
private bool IsObjectChanged(ObjectInfo cloudFile, string localPath) |
|
543 |
{ |
|
544 |
//If the target is a directory, there are no changes to download |
|
545 |
if (Directory.Exists(localPath)) |
|
546 |
return false; |
|
547 |
//If the file doesn't exist, we have a chagne |
|
548 |
if (!File.Exists(localPath)) |
|
549 |
return true; |
|
550 |
//If there is no stored state, we have a change |
|
551 |
var localState = StatusKeeper.GetStateByFilePath(localPath); |
|
552 |
if (localState == null) |
|
553 |
return true; |
|
554 |
|
|
555 |
var info = new FileInfo(localPath); |
|
556 |
var shortHash = info.ComputeShortHash(); |
|
557 |
//If the file is different from the stored state, we have a change |
|
558 |
if (localState.ShortHash != shortHash) |
|
559 |
return true; |
|
560 |
//If the top hashes differ, we have a change |
|
561 |
return (localState.Checksum != cloudFile.Hash); |
|
562 |
} |
|
563 |
|
|
526 | 564 |
//Download a small file with a single GET operation |
527 | 565 |
private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath,TreeHash serverHash) |
528 | 566 |
{ |
... | ... | |
539 | 577 |
Contract.EndContractBlock(); |
540 | 578 |
|
541 | 579 |
var localPath = Pithos.Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath); |
542 |
//If the file already exists |
|
543 |
if (File.Exists(localPath)) |
|
544 |
{ |
|
545 |
//First check with MD5 as this is a small file |
|
546 |
var localMD5 = Signature.CalculateMD5(localPath); |
|
547 |
var cloudHash=serverHash.TopHash.ToHashString(); |
|
548 |
if (localMD5==cloudHash) |
|
549 |
return; |
|
550 |
//Then check with a treehash |
|
551 |
var localTreeHash = Signature.CalculateTreeHash(localPath, serverHash.BlockSize, serverHash.BlockHash); |
|
552 |
var localHash = localTreeHash.TopHash.ToHashString(); |
|
553 |
if (localHash==cloudHash) |
|
554 |
return; |
|
555 |
} |
|
580 |
StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Downloading {0}",Path.GetFileName(localPath))); |
|
556 | 581 |
StatusNotification.Notify(new CloudNotification { Data = cloudFile }); |
557 | 582 |
|
558 | 583 |
var fileAgent = GetFileAgent(accountInfo); |
... | ... | |
608 | 633 |
var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash); |
609 | 634 |
|
610 | 635 |
|
611 |
|
|
636 |
StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Calculating hashmap for {0} before download",Path.GetFileName(localPath))); |
|
612 | 637 |
//Calculate the file's treehash |
613 | 638 |
var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash, 2); |
614 | 639 |
|
... | ... | |
676 | 701 |
|
677 | 702 |
if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)) |
678 | 703 |
return; |
704 |
|
|
705 |
// |
|
706 |
if (action.FileState == null) |
|
707 |
action.FileState = StatusKeeper.GetStateByFilePath(fileInfo.FullName); |
|
679 | 708 |
//Do not upload files in conflict |
680 | 709 |
if (action.FileState.FileStatus == FileStatus.Conflict ) |
681 | 710 |
{ |
... | ... | |
724 | 753 |
|
725 | 754 |
var client = new CloudFilesClient(accountInfo); |
726 | 755 |
//Even if GetObjectInfo times out, we can proceed with the upload |
727 |
var info = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
|
|
756 |
var cloudInfo = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
|
|
728 | 757 |
|
729 | 758 |
//If this is a read-only file, do not upload changes |
730 |
if (info.AllowedTo == "read")
|
|
759 |
if (cloudInfo.AllowedTo == "read")
|
|
731 | 760 |
return; |
732 | 761 |
|
733 | 762 |
//TODO: Check how a directory hash is calculated -> All dirs seem to have the same hash |
734 | 763 |
if (fileInfo is DirectoryInfo) |
735 | 764 |
{ |
736 | 765 |
//If the directory doesn't exist the Hash property will be empty |
737 |
if (String.IsNullOrWhiteSpace(info.Hash))
|
|
766 |
if (String.IsNullOrWhiteSpace(cloudInfo.Hash))
|
|
738 | 767 |
//Go on and create the directory |
739 | 768 |
await |
740 | 769 |
client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName, |
... | ... | |
743 | 772 |
else |
744 | 773 |
{ |
745 | 774 |
|
746 |
var cloudHash = info.Hash.ToLower(); |
|
775 |
var cloudHash = cloudInfo.Hash.ToLower(); |
|
776 |
|
|
777 |
StatusNotification.Notify(new StatusNotification(String.Format("Hashing {0} for Upload",fileInfo.Name))); |
|
778 |
|
|
779 |
//TODO: This is the same as the calculation for the Local Hash! |
|
780 |
//First, calculate the tree hash |
|
781 |
/* |
|
782 |
var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize, |
|
783 |
accountInfo.BlockHash, 2); |
|
784 |
*/ |
|
747 | 785 |
|
748 |
var hash = action.LocalHash.Value; |
|
749 |
var topHash = action.TopHash.Value; |
|
786 |
|
|
787 |
var treeHash = action.TreeHash.Value; |
|
788 |
var topHash = treeHash.TopHash.ToHashString(); |
|
789 |
|
|
790 |
//var topHash = action.TopHash.Value; |
|
750 | 791 |
|
751 | 792 |
//If the file hashes match, abort the upload |
752 |
if (hash == cloudHash || topHash == cloudHash)
|
|
793 |
if (topHash == cloudHash /*|| topHash == cloudHash*/)
|
|
753 | 794 |
{ |
754 | 795 |
//but store any metadata changes |
755 |
StatusKeeper.StoreInfo(fullFileName, info);
|
|
796 |
StatusKeeper.StoreInfo(fullFileName, cloudInfo);
|
|
756 | 797 |
Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName); |
757 | 798 |
return; |
758 | 799 |
} |
... | ... | |
765 | 806 |
//Upload even small files using the Hashmap. The server may already contain |
766 | 807 |
//the relevant block |
767 | 808 |
|
768 |
//First, calculate the tree hash |
|
769 |
var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize, |
|
770 |
accountInfo.BlockHash, 2); |
|
771 |
|
|
772 | 809 |
//TODO: If the upload fails with a 403, abort it and mark conflict |
773 | 810 |
|
774 | 811 |
await |
... | ... | |
847 | 884 |
throw new ArgumentException("Invalid container","cloudFile"); |
848 | 885 |
Contract.EndContractBlock(); |
849 | 886 |
|
887 |
StatusNotification.Notify(new StatusNotification(String.Format("Uploading {0} for Upload", fileInfo.Name))); |
|
888 |
|
|
850 | 889 |
var fullFileName = fileInfo.GetProperCapitalization(); |
851 | 890 |
|
852 | 891 |
var account = cloudFile.Account ?? accountInfo.UserName; |
... | ... | |
886 | 925 |
//Repeat until there are no more missing hashes |
887 | 926 |
missingHashes = await client.PutHashMap(account, container, url, treeHash); |
888 | 927 |
} |
928 |
|
|
889 | 929 |
ReportUploadProgress(fileInfo.Name, missingHashes.Count, missingHashes.Count, fileInfo.Length); |
890 | 930 |
} |
891 | 931 |
|
892 | 932 |
private void ReportUploadProgress(string fileName,int block, int totalBlocks, long fileSize) |
893 | 933 |
{ |
894 |
StatusNotification.Notify(new ProgressNotification(fileName,"Uploading",block,totalBlocks,fileSize) ); |
|
934 |
StatusNotification.Notify(totalBlocks == 0 |
|
935 |
? new ProgressNotification(fileName, "Uploading", 1, 1, fileSize) |
|
936 |
: new ProgressNotification(fileName, "Uploading", block, totalBlocks, fileSize)); |
|
895 | 937 |
} |
938 |
|
|
896 | 939 |
private void ReportDownloadProgress(string fileName,int block, int totalBlocks, long fileSize) |
897 | 940 |
{ |
898 |
StatusNotification.Notify(new ProgressNotification(fileName,"Downloading",block,totalBlocks,fileSize) ); |
|
941 |
StatusNotification.Notify(totalBlocks == 0 |
|
942 |
? new ProgressNotification(fileName, "Downloading", 1, 1, fileSize) |
|
943 |
: new ProgressNotification(fileName, "Downloading", block, totalBlocks, fileSize)); |
|
899 | 944 |
} |
900 | 945 |
} |
901 | 946 |
|
Also available in: Unified diff