Revision 174bbb6e trunk/Pithos.Core/Agents/NetworkAgent.cs

b/trunk/Pithos.Core/Agents/NetworkAgent.cs
89 89

  
90 90
        public void Start()
91 91
        {
92
            if (_agent != null)
93
                return;
94

  
95
            if (Log.IsDebugEnabled)
96
                Log.Debug("Starting Network Agent");
97

  
92 98
            _agent = Agent<CloudAction>.Start(inbox =>
93 99
            {
94 100
                Action loop = null;
......
123 129

  
124 130
                try
125 131
                {
132
                    StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Processing");
126 133
                    _proceedEvent.Reset();
127 134
                    //UpdateStatus(PithosStatus.Syncing);
128 135
                    var accountInfo = action.AccountInfo;
......
204 211
                {
205 212
                    if (_agent.IsEmpty)
206 213
                        _proceedEvent.Set();
207
                    UpdateStatus(PithosStatus.InSynch);                                        
214
                    UpdateStatus(PithosStatus.LocalComplete);                                        
208 215
                }
209 216
            }
210 217
        }
......
212 219

  
213 220
        private void UpdateStatus(PithosStatus status)
214 221
        {
215
            StatusKeeper.SetPithosStatus(status);
216
            StatusNotification.Notify(new Notification());
222
            StatusNotification.SetPithosStatus(status);
223
            //StatusNotification.Notify(new Notification());
217 224
        }
218 225

  
219 226
        private void RenameLocalFile(AccountInfo accountInfo, CloudAction action)
......
297 304

  
298 305
                var cloudHash = cloudFile.Hash.ToLower();
299 306
                var previousCloudHash = cloudFile.PreviousHash.ToLower();
300
                var localHash = action.LocalHash.Value.ToLower();
301
                var topHash = action.TopHash.Value.ToLower();
307
                var localHash = action.TreeHash.Value.TopHash.ToHashString();// LocalHash.Value.ToLower();
308
                //var topHash = action.TopHash.Value.ToLower();
302 309

  
303 310
                //At this point we know that an object has changed on the server and that a local
304 311
                //file already exists. We need to decide whether the file has only changed on 
305 312
                //the server or there is a conflicting change on the client.
306 313
                //
307 314

  
308
                //Not enough to compare only the local hashes (MD5), also have to compare the tophashes            
309
                //If any of the hashes match, we are done
310
                if ((cloudHash == localHash || cloudHash == topHash))
315
                //If the hashes match, we are done
316
                if (cloudHash == localHash)
311 317
                {
312 318
                    Log.InfoFormat("Skipping {0}, hashes match", downloadPath);
313 319
                    return;
......
351 357
            Contract.EndContractBlock();
352 358

  
353 359
            _deleteAgent.ProceedEvent.Wait();
360
/*
354 361

  
355 362
            //If the action targets a local file, add a treehash calculation
356 363
            if (!(cloudAction is CloudDeleteAction) && cloudAction.LocalFile as FileInfo != null)
357 364
            {
358 365
                var accountInfo = cloudAction.AccountInfo;
359 366
                var localFile = (FileInfo) cloudAction.LocalFile;
367

  
360 368
                if (localFile.Length > accountInfo.BlockSize)
361 369
                    cloudAction.TopHash =
362 370
                        new Lazy<string>(() => Signature.CalculateTreeHashAsync(localFile,
......
367 375
                {
368 376
                    cloudAction.TopHash = new Lazy<string>(() => cloudAction.LocalHash.Value);
369 377
                }
378

  
370 379
            }
371 380
            else
372 381
            {
373 382
                //The hash for a directory is the empty string
374 383
                cloudAction.TopHash = new Lazy<string>(() => String.Empty);
375 384
            }
385
*/
376 386
            
377 387
            if (cloudAction is CloudDeleteAction)
378 388
                _deleteAgent.Post((CloudDeleteAction)cloudAction);
......
497 507
                    }
498 508
                    else
499 509
                    {
500
                        //Retrieve the hashmap from the server
501
                        var serverHash = await client.GetHashMap(account, container, url);
502
                        //If it's a small file
503
                        if (serverHash.Hashes.Count == 1)
504
                            //Download it in one go
505
                            await
506
                                DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath,
507
                                                        serverHash);
508
                            //Otherwise download it block by block
509
                        else
510
                            await DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath, serverHash);
511

  
512
                        if (cloudFile.AllowedTo == "read")
510
                        var isChanged = IsObjectChanged(cloudFile, localPath);
511
                        if (isChanged)
513 512
                        {
514
                            var attributes = File.GetAttributes(localPath);
515
                            File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly);
513
                            //Retrieve the hashmap from the server
514
                            var serverHash = await client.GetHashMap(account, container, url);
515
                            //If it's a small file
516
                            if (serverHash.Hashes.Count == 1)
517
                                //Download it in one go
518
                                await
519
                                    DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath,
520
                                                            serverHash);
521
                                //Otherwise download it block by block
522
                            else
523
                                await
524
                                    DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath,
525
                                                       serverHash);
526

  
527
                            if (cloudFile.AllowedTo == "read")
528
                            {
529
                                var attributes = File.GetAttributes(localPath);
530
                                File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly);
531
                            }
516 532
                        }
517 533
                    }
518 534

  
......
523 539
            }
524 540
        }
525 541

  
542
        private bool IsObjectChanged(ObjectInfo cloudFile, string localPath)
543
        {
544
            //If the target is a directory, there are no changes to download
545
            if (Directory.Exists(localPath))
546
                return false;
547
            //If the file doesn't exist, we have a chagne
548
            if (!File.Exists(localPath)) 
549
                return true;
550
            //If there is no stored state, we have a change
551
            var localState = StatusKeeper.GetStateByFilePath(localPath);
552
            if (localState == null)
553
                return true;
554

  
555
            var info = new FileInfo(localPath);
556
            var shortHash = info.ComputeShortHash();
557
            //If the file is different from the stored state, we have a change
558
            if (localState.ShortHash != shortHash)
559
                return true;
560
            //If the top hashes differ, we have a change
561
            return (localState.Checksum != cloudFile.Hash);
562
        }
563

  
526 564
        //Download a small file with a single GET operation
527 565
        private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath,TreeHash serverHash)
528 566
        {
......
539 577
            Contract.EndContractBlock();
540 578

  
541 579
            var localPath = Pithos.Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath);
542
            //If the file already exists
543
            if (File.Exists(localPath))
544
            {
545
                //First check with MD5 as this is a small file
546
                var localMD5 = Signature.CalculateMD5(localPath);
547
                var cloudHash=serverHash.TopHash.ToHashString();
548
                if (localMD5==cloudHash)
549
                    return;
550
                //Then check with a treehash
551
                var localTreeHash = Signature.CalculateTreeHash(localPath, serverHash.BlockSize, serverHash.BlockHash);
552
                var localHash = localTreeHash.TopHash.ToHashString();
553
                if (localHash==cloudHash)
554
                    return;
555
            }
580
            StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Downloading {0}",Path.GetFileName(localPath)));
556 581
            StatusNotification.Notify(new CloudNotification { Data = cloudFile });
557 582

  
558 583
            var fileAgent = GetFileAgent(accountInfo);
......
608 633
            var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash);
609 634

  
610 635
            
611
                        
636
            StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Calculating hashmap for {0} before download",Path.GetFileName(localPath)));
612 637
            //Calculate the file's treehash
613 638
            var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash, 2);
614 639
                
......
676 701

  
677 702
                    if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
678 703
                        return;
704
                    
705
                    //
706
                    if (action.FileState == null)
707
                        action.FileState = StatusKeeper.GetStateByFilePath(fileInfo.FullName);
679 708
                    //Do not upload files in conflict
680 709
                    if (action.FileState.FileStatus == FileStatus.Conflict )
681 710
                    {
......
724 753

  
725 754
                        var client = new CloudFilesClient(accountInfo);
726 755
                        //Even if GetObjectInfo times out, we can proceed with the upload            
727
                        var info = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
756
                        var cloudInfo = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
728 757

  
729 758
                        //If this is a read-only file, do not upload changes
730
                        if (info.AllowedTo == "read")
759
                        if (cloudInfo.AllowedTo == "read")
731 760
                            return;
732 761

  
733 762
                        //TODO: Check how a directory hash is calculated -> All dirs seem to have the same hash
734 763
                            if (fileInfo is DirectoryInfo)
735 764
                            {
736 765
                                //If the directory doesn't exist the Hash property will be empty
737
                                if (String.IsNullOrWhiteSpace(info.Hash))
766
                                if (String.IsNullOrWhiteSpace(cloudInfo.Hash))
738 767
                                    //Go on and create the directory
739 768
                                    await
740 769
                                        client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,
......
743 772
                            else
744 773
                            {
745 774

  
746
                                var cloudHash = info.Hash.ToLower();
775
                                var cloudHash = cloudInfo.Hash.ToLower();
776

  
777
                                StatusNotification.Notify(new StatusNotification(String.Format("Hashing {0} for Upload",fileInfo.Name)));
778

  
779
                                //TODO: This is the same as the calculation for the Local Hash!
780
                                //First, calculate the tree hash
781
/*
782
                                var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize,
783
                                                                                      accountInfo.BlockHash, 2);
784
*/
747 785

  
748
                                var hash = action.LocalHash.Value;
749
                                var topHash = action.TopHash.Value;
786

  
787
                                var treeHash = action.TreeHash.Value;
788
                                var topHash = treeHash.TopHash.ToHashString();
789
                                
790
                                //var topHash = action.TopHash.Value;
750 791

  
751 792
                                //If the file hashes match, abort the upload
752
                                if (hash == cloudHash || topHash == cloudHash)
793
                                if (topHash == cloudHash /*|| topHash == cloudHash*/)
753 794
                                {
754 795
                                    //but store any metadata changes 
755
                                    StatusKeeper.StoreInfo(fullFileName, info);
796
                                    StatusKeeper.StoreInfo(fullFileName, cloudInfo);
756 797
                                    Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
757 798
                                    return;
758 799
                                }
......
765 806
                                //Upload even small files using the Hashmap. The server may already contain
766 807
                                //the relevant block
767 808

  
768
                                //First, calculate the tree hash
769
                                var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize,
770
                                                                                      accountInfo.BlockHash, 2);
771

  
772 809
                                //TODO: If the upload fails with a 403, abort it and mark conflict
773 810

  
774 811
                                await
......
847 884
                throw new ArgumentException("Invalid container","cloudFile");
848 885
            Contract.EndContractBlock();
849 886

  
887
            StatusNotification.Notify(new StatusNotification(String.Format("Uploading {0} for Upload", fileInfo.Name)));
888

  
850 889
            var fullFileName = fileInfo.GetProperCapitalization();
851 890

  
852 891
            var account = cloudFile.Account ?? accountInfo.UserName;
......
886 925
                //Repeat until there are no more missing hashes                
887 926
                missingHashes = await client.PutHashMap(account, container, url, treeHash);
888 927
            }
928

  
889 929
            ReportUploadProgress(fileInfo.Name, missingHashes.Count, missingHashes.Count, fileInfo.Length);
890 930
        }
891 931

  
892 932
        private void ReportUploadProgress(string fileName,int block, int totalBlocks, long fileSize)
893 933
        {
894
            StatusNotification.Notify(new ProgressNotification(fileName,"Uploading",block,totalBlocks,fileSize) );            
934
            StatusNotification.Notify(totalBlocks == 0
935
                                          ? new ProgressNotification(fileName, "Uploading", 1, 1, fileSize)
936
                                          : new ProgressNotification(fileName, "Uploading", block, totalBlocks, fileSize));
895 937
        }
938

  
896 939
        private void ReportDownloadProgress(string fileName,int block, int totalBlocks, long fileSize)
897 940
        {
898
            StatusNotification.Notify(new ProgressNotification(fileName,"Downloading",block,totalBlocks,fileSize) );            
941
            StatusNotification.Notify(totalBlocks == 0
942
                                          ? new ProgressNotification(fileName, "Downloading", 1, 1, fileSize)
943
                                          : new ProgressNotification(fileName, "Downloading", block, totalBlocks, fileSize));
899 944
        }
900 945
    }
901 946

  

Also available in: Unified diff