Revision 540b8cf8 trunk/Pithos.Core/Agents/NetworkAgent.cs
b/trunk/Pithos.Core/Agents/NetworkAgent.cs | ||
---|---|---|
76 | 76 |
|
77 | 77 |
private readonly ConcurrentBag<AccountInfo> _accounts = new ConcurrentBag<AccountInfo>(); |
78 | 78 |
|
79 |
|
|
80 |
private bool _firstPoll = true; |
|
79 | 81 |
public void Start() |
80 | 82 |
{ |
81 |
|
|
83 |
_firstPoll = true; |
|
82 | 84 |
_agent = Agent<CloudAction>.Start(inbox => |
83 | 85 |
{ |
84 | 86 |
Action loop = null; |
... | ... | |
447 | 449 |
|
448 | 450 |
await TaskEx.WhenAll(tasks.ToList()); |
449 | 451 |
|
452 |
_firstPoll = false; |
|
450 | 453 |
ProcessRemoteFiles(nextSince); |
451 | 454 |
} |
452 | 455 |
catch (Exception ex) |
... | ... | |
480 | 483 |
try |
481 | 484 |
{ |
482 | 485 |
_pauseAgent.Wait(); |
486 |
//Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted |
|
487 |
//than delete a file that was created while we were executing the poll |
|
488 |
var pollTime = DateTime.Now; |
|
489 |
|
|
483 | 490 |
//Get the list of server objects changed since the last check |
484 | 491 |
//The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step |
485 | 492 |
var listObjects = from container in containers |
... | ... | |
499 | 506 |
from obj in objectList.Result |
500 | 507 |
select obj; |
501 | 508 |
|
509 |
//TODO: Change the way deleted objects are detected. |
|
510 |
//The list operation returns all existing objects so we could detect deleted remote objects |
|
511 |
//by detecting objects that exist only locally. There are several cases where this is NOT the case: |
|
512 |
//1. The first time the application runs, as there may be files that were added while |
|
513 |
// the application was down. |
|
514 |
//2. An object that is currently being uploaded will not appear in the remote list |
|
515 |
// until the upload finishes. |
|
516 |
// SOLUTION 1: Check the upload/download queue for the file |
|
517 |
// SOLUTION 2: Check the SQLite states for the file's entry. If it is being uploaded, |
|
518 |
// or its last modification was after the current poll, don't delete it. This way we don't |
|
519 |
// delete objects whose upload finished too late to be included in the list. |
|
520 |
//We need to detect and protect against such situations |
|
521 |
//TODO: Does FileState have a LastModification field? |
|
522 |
//TODO: How do we update the LastModification field? Do we need to add SQLite triggers? |
|
523 |
// Do we need to use a proper SQLite schema? |
|
524 |
// We can create a trigger with |
|
525 |
// CREATE TRIGGER IF NOT EXISTS update_last_modified UPDATE ON FileState FOR EACH ROW |
|
526 |
// BEGIN |
|
527 |
// UPDATE FileState SET LastModification=datetime('now') WHERE Id=old.Id; |
|
528 |
// END; |
|
529 |
// |
|
530 |
//NOTE: Some files may have been deleted remotely while the application was down. |
|
531 |
// We DO have to delete those files. Checking the trash makes it easy to detect them, |
|
532 |
// Otherwise, we can't be really sure whether we need to upload or delete |
|
533 |
// the local-only files. |
|
534 |
// SOLUTION 1: Ask the user when such a local-only file is detected during the first poll. |
|
535 |
// SOLUTION 2: Mark conflict and ask the user as in #1 |
|
536 |
|
|
502 | 537 |
var trashObjects = dict["trash"].Result; |
503 | 538 |
//var sharedObjects = ((Task<IList<ObjectInfo>>) task.Result[2]).Result; |
504 | 539 |
|
... | ... | |
509 | 544 |
!remoteObjects.Any( |
510 | 545 |
info => info.Name == trash.Name && info.Hash == trash.Hash) |
511 | 546 |
select trash; |
512 |
ProcessDeletedFiles(accountInfo, realTrash);
|
|
547 |
ProcessTrashedFiles(accountInfo, realTrash);
|
|
513 | 548 |
|
514 | 549 |
|
515 |
var remote = from info in remoteObjects
|
|
550 |
var cleanRemotes = from info in remoteObjects
|
|
516 | 551 |
//.Union(sharedObjects) |
517 | 552 |
let name = info.Name |
518 | 553 |
where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) && |
... | ... | |
520 | 555 |
StringComparison.InvariantCultureIgnoreCase) |
521 | 556 |
select info; |
522 | 557 |
|
558 |
|
|
559 |
|
|
560 |
|
|
561 |
|
|
523 | 562 |
//Create a list of actions from the remote files |
524 |
var allActions = ObjectsToActions(accountInfo, remote); |
|
563 |
var allActions = ObjectsToActions(accountInfo, cleanRemotes); |
|
564 |
|
|
565 |
ProcessDeletedFiles(accountInfo, cleanRemotes, pollTime); |
|
566 |
//var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName); |
|
525 | 567 |
|
526 | 568 |
//And remove those that are already being processed by the agent |
527 | 569 |
var distinctActions = allActions |
... | ... | |
548 | 590 |
} |
549 | 591 |
} |
550 | 592 |
|
593 |
/// <summary> |
|
594 |
/// Deletes local files that are not found in the list of cloud files |
|
595 |
/// </summary> |
|
596 |
/// <param name="accountInfo"></param> |
|
597 |
/// <param name="cloudFiles"></param> |
|
598 |
/// <param name="pollTime"></param> |
|
599 |
private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles, DateTime pollTime) |
|
600 |
{ |
|
601 |
if (accountInfo == null) |
|
602 |
throw new ArgumentNullException("accountInfo"); |
|
603 |
if (String.IsNullOrWhiteSpace(accountInfo.AccountPath)) |
|
604 |
throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo"); |
|
605 |
if (cloudFiles == null) |
|
606 |
throw new ArgumentNullException("cloudFiles"); |
|
607 |
Contract.EndContractBlock(); |
|
608 |
|
|
609 |
if (_firstPoll) return; |
|
610 |
|
|
611 |
var deleteCandidates = from state in FileState.Queryable |
|
612 |
let stateUrl = FileInfoExtensions.FromPath(state.FilePath) |
|
613 |
.AsRelativeUrlTo(accountInfo.AccountPath) |
|
614 |
where state.Modified <= pollTime && |
|
615 |
!cloudFiles.Any(r => r.Name == stateUrl) |
|
616 |
select state; |
|
617 |
|
|
618 |
foreach (var deleteCandidate in deleteCandidates) |
|
619 |
{ |
|
620 |
File.Delete(deleteCandidate.FilePath); |
|
621 |
StatusKeeper.ClearFileStatus(deleteCandidate.FilePath); |
|
622 |
} |
|
623 |
} |
|
624 |
|
|
551 | 625 |
private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers) |
552 | 626 |
{ |
553 | 627 |
var containerPaths = from container in containers |
... | ... | |
613 | 687 |
return AgentLocator<FileAgent>.Get(accountInfo.AccountPath); |
614 | 688 |
} |
615 | 689 |
|
616 |
private void ProcessDeletedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects)
|
|
690 |
private void ProcessTrashedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects)
|
|
617 | 691 |
{ |
618 | 692 |
var fileAgent = GetFileAgent(accountInfo); |
619 | 693 |
foreach (var trashObject in trashObjects) |
Also available in: Unified diff