Revision 540b8cf8 trunk/Pithos.Core/Agents/NetworkAgent.cs

b/trunk/Pithos.Core/Agents/NetworkAgent.cs
76 76

  
77 77
        private readonly ConcurrentBag<AccountInfo> _accounts = new ConcurrentBag<AccountInfo>();
78 78

  
79

  
80
        private bool _firstPoll = true;
79 81
        public void Start()
80 82
        {
81

  
83
            _firstPoll = true;
82 84
            _agent = Agent<CloudAction>.Start(inbox =>
83 85
            {
84 86
                Action loop = null;
......
447 449

  
448 450
                    await TaskEx.WhenAll(tasks.ToList());
449 451

  
452
                    _firstPoll = false;
450 453
                    ProcessRemoteFiles(nextSince);
451 454
                }
452 455
                catch (Exception ex)
......
480 483
                try
481 484
                {
482 485
                    _pauseAgent.Wait();
486
                    //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted
487
                    //than delete a file that was created while we were executing the poll
488
                    var pollTime = DateTime.Now;
489
                    
483 490
                    //Get the list of server objects changed since the last check
484 491
                    //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step
485 492
                    var listObjects = from container in containers
......
499 506
                                            from obj in objectList.Result
500 507
                                            select obj;
501 508

  
509
                        //TODO: Change the way deleted objects are detected.
510
                        //The list operation returns all existing objects so we could detect deleted remote objects
511
                        //by detecting objects that exist only locally. There are several cases where this is NOT the case:
512
                        //1.    The first time the application runs, as there may be files that were added while 
513
                        //      the application was down.
514
                        //2.    An object that is currently being uploaded will not appear in the remote list
515
                        //      until the upload finishes.
516
                        //      SOLUTION 1: Check the upload/download queue for the file
517
                        //      SOLUTION 2: Check the SQLite states for the file's entry. If it is being uploaded, 
518
                        //          or its last modification was after the current poll, don't delete it. This way we don't
519
                        //          delete objects whose upload finished too late to be included in the list.
520
                        //We need to detect and protect against such situations
521
                        //TODO: Does FileState have a LastModification field?
522
                        //TODO: How do we update the LastModification field? Do we need to add SQLite triggers?
523
                        //      Do we need to use a proper SQLite schema?
524
                        //      We can create a trigger with 
525
                        // CREATE TRIGGER IF NOT EXISTS update_last_modified UPDATE ON FileState FOR EACH ROW
526
                        //  BEGIN
527
                        //      UPDATE FileState SET LastModification=datetime('now')  WHERE Id=old.Id;
528
                        //  END;
529
                        //
530
                        //NOTE: Some files may have been deleted remotely while the application was down. 
531
                        //  We DO have to delete those files. Checking the trash makes it easy to detect them,
532
                        //  Otherwise, we can't be really sure whether we need to upload or delete 
533
                        //  the local-only files.
534
                        //  SOLUTION 1: Ask the user when such a local-only file is detected during the first poll.
535
                        //  SOLUTION 2: Mark conflict and ask the user as in #1
536

  
502 537
                        var trashObjects = dict["trash"].Result;
503 538
                        //var sharedObjects = ((Task<IList<ObjectInfo>>) task.Result[2]).Result;
504 539

  
......
509 544
                                            !remoteObjects.Any(
510 545
                                                info => info.Name == trash.Name && info.Hash == trash.Hash)
511 546
                                        select trash;
512
                        ProcessDeletedFiles(accountInfo, realTrash);
547
                        ProcessTrashedFiles(accountInfo, realTrash);
513 548

  
514 549

  
515
                        var remote = from info in remoteObjects
550
                        var cleanRemotes = from info in remoteObjects
516 551
                                     //.Union(sharedObjects)
517 552
                                     let name = info.Name
518 553
                                     where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&
......
520 555
                                                            StringComparison.InvariantCultureIgnoreCase)
521 556
                                     select info;
522 557

  
558

  
559
                        
560

  
561

  
523 562
                        //Create a list of actions from the remote files
524
                        var allActions = ObjectsToActions(accountInfo, remote);
563
                        var allActions = ObjectsToActions(accountInfo, cleanRemotes);
564

  
565
                        ProcessDeletedFiles(accountInfo, cleanRemotes, pollTime);
566
                        //var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);
525 567

  
526 568
                        //And remove those that are already being processed by the agent
527 569
                        var distinctActions = allActions
......
548 590
            }
549 591
        }
550 592

  
593
        /// <summary>
594
        /// Deletes local files that are not found in the list of cloud files
595
        /// </summary>
596
        /// <param name="accountInfo"></param>
597
        /// <param name="cloudFiles"></param>
598
        /// <param name="pollTime"></param>
599
        private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles, DateTime pollTime)
600
        {
601
            if (accountInfo == null)
602
                throw new ArgumentNullException("accountInfo");
603
            if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))
604
                throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");
605
            if (cloudFiles == null)
606
                throw new ArgumentNullException("cloudFiles");
607
            Contract.EndContractBlock();
608

  
609
            if (_firstPoll) return;
610

  
611
            var deleteCandidates = from state in FileState.Queryable
612
                                   let stateUrl = FileInfoExtensions.FromPath(state.FilePath)
613
                                       .AsRelativeUrlTo(accountInfo.AccountPath)
614
                                   where state.Modified <= pollTime &&
615
                                         !cloudFiles.Any(r => r.Name == stateUrl)
616
                                   select state;
617

  
618
            foreach (var deleteCandidate in deleteCandidates)
619
            {
620
                File.Delete(deleteCandidate.FilePath);
621
                StatusKeeper.ClearFileStatus(deleteCandidate.FilePath);
622
            }
623
        }
624

  
551 625
        private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)
552 626
        {
553 627
            var containerPaths = from container in containers
......
613 687
            return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);
614 688
        }
615 689

  
616
        private void ProcessDeletedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects)
690
        private void ProcessTrashedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects)
617 691
        {
618 692
            var fileAgent = GetFileAgent(accountInfo);
619 693
            foreach (var trashObject in trashObjects)

Also available in: Unified diff