Minor fix in Selective sync to ignore 403 errors the first time new share listings...
[pithos-ms-client] / trunk / Pithos.Core / Agents / PollAgent.cs
index 89635e9..d1974bf 100644 (file)
@@ -45,9 +45,9 @@ using System.ComponentModel.Composition;
 using System.Diagnostics;\r
 using System.Diagnostics.Contracts;\r
 using System.IO;\r
+using System.Reflection;\r
 using System.Threading;\r
 using System.Threading.Tasks;\r
-using System.Threading.Tasks.Dataflow;\r
 using Castle.ActiveRecord;\r
 using Pithos.Interfaces;\r
 using Pithos.Network;\r
@@ -58,7 +58,6 @@ namespace Pithos.Core.Agents
     using System;\r
     using System.Collections.Generic;\r
     using System.Linq;\r
-    using System.Text;\r
 \r
     /// <summary>\r
     /// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all\r
@@ -69,7 +68,7 @@ namespace Pithos.Core.Agents
     [Export]\r
     public class PollAgent\r
     {\r
-        private static readonly ILog Log = LogManager.GetLogger("PollAgent");\r
+        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
 \r
         [System.ComponentModel.Composition.Import]\r
         public IStatusKeeper StatusKeeper { get; set; }\r
@@ -80,15 +79,36 @@ namespace Pithos.Core.Agents
         [System.ComponentModel.Composition.Import]\r
         public NetworkAgent NetworkAgent { get; set; }\r
 \r
+        [System.ComponentModel.Composition.Import]\r
+        public Selectives Selectives { get; set; }\r
+\r
         public IStatusNotification StatusNotification { get; set; }\r
 \r
+        public bool Pause\r
+        {\r
+            get {\r
+                return _pause;\r
+            }\r
+            set {\r
+                _pause = value;                \r
+                if (!_pause)\r
+                    _unPauseEvent.Set();\r
+                else\r
+                {\r
+                    _unPauseEvent.Reset();\r
+                }\r
+            }\r
+        }\r
+\r
         private bool _firstPoll = true;\r
 \r
         //The Sync Event signals a manual synchronisation\r
         private readonly AsyncManualResetEvent _syncEvent = new AsyncManualResetEvent();\r
 \r
-        private ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>();\r
-        private readonly ConcurrentBag<AccountInfo> _accounts = new ConcurrentBag<AccountInfo>();\r
+        private readonly AsyncManualResetEvent _unPauseEvent = new AsyncManualResetEvent(true);\r
+\r
+        private readonly ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>();\r
+        private readonly ConcurrentDictionary<Uri, AccountInfo> _accounts = new ConcurrentDictionary<Uri,AccountInfo>();\r
 \r
 \r
         /// <summary>\r
@@ -99,32 +119,40 @@ namespace Pithos.Core.Agents
             _syncEvent.Set();\r
         }\r
 \r
-        //Remote files are polled periodically. Any changes are processed\r
+        /// <summary>\r
+        /// Remote files are polled periodically. Any changes are processed\r
+        /// </summary>\r
+        /// <param name="since"></param>\r
+        /// <returns></returns>\r
         public async Task PollRemoteFiles(DateTime? since = null)\r
         {\r
-            Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
+            if (Log.IsDebugEnabled)\r
+                Log.DebugFormat("Polling changes after [{0}]",since);\r
 \r
-            UpdateStatus(PithosStatus.Syncing);\r
-            StatusNotification.Notify(new PollNotification());\r
+            Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
+            \r
 \r
-            using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
+            using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
             {\r
                 //If this poll fails, we will retry with the same since value\r
                 var nextSince = since;\r
                 try\r
                 {\r
-                    //Next time we will check for all changes since the current check minus 1 second\r
-                    //This is done to ensure there are no discrepancies due to clock differences\r
-                    var current = DateTime.Now.AddSeconds(-1);\r
+                    await _unPauseEvent.WaitAsync();\r
+                    UpdateStatus(PithosStatus.PollSyncing);\r
 \r
-                    var tasks = from accountInfo in _accounts\r
+                    var tasks = from accountInfo in _accounts.Values\r
                                 select ProcessAccountFiles(accountInfo, since);\r
 \r
-                    await TaskEx.WhenAll(tasks.ToList());\r
+                    var nextTimes=await TaskEx.WhenAll(tasks.ToList());\r
 \r
                     _firstPoll = false;\r
                     //Reschedule the poll with the current timestamp as a "since" value\r
-                    nextSince = current;\r
+\r
+                    if (nextTimes.Length>0)\r
+                        nextSince = nextTimes.Min();\r
+                    if (Log.IsDebugEnabled)\r
+                        Log.DebugFormat("Next Poll at [{0}]",nextSince);\r
                 }\r
                 catch (Exception ex)\r
                 {\r
@@ -132,12 +160,20 @@ namespace Pithos.Core.Agents
                     //In case of failure retry with the same "since" value\r
                 }\r
 \r
-                UpdateStatus(PithosStatus.InSynch);\r
-                //Wait for the polling interval to pass or the Sync event to be signalled\r
-                nextSince = await WaitForScheduledOrManualPoll(nextSince);\r
-\r
-                TaskEx.Run(()=>PollRemoteFiles(nextSince));\r
-\r
+                UpdateStatus(PithosStatus.PollComplete);\r
+                //The multiple try blocks are required because we can't have an await call\r
+                //inside a finally block\r
+                //TODO: Find a more elegant solution for reschedulling in the event of an exception\r
+                try\r
+                {\r
+                    //Wait for the polling interval to pass or the Sync event to be signalled\r
+                    nextSince = await WaitForScheduledOrManualPoll(nextSince);\r
+                }\r
+                finally\r
+                {\r
+                    //Ensure polling is scheduled even in case of error\r
+                    TaskEx.Run(() => PollRemoteFiles(nextSince));                        \r
+                }\r
             }\r
         }\r
 \r
@@ -150,8 +186,12 @@ namespace Pithos.Core.Agents
         {\r
             var sync = _syncEvent.WaitAsync();\r
             var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval), NetworkAgent.CancellationToken);\r
+            \r
             var signaledTask = await TaskEx.WhenAny(sync, wait);\r
-\r
+            \r
+            //Pausing takes precedence over manual sync or awaiting\r
+            _unPauseEvent.Wait();\r
+            \r
             //Wait for network processing to finish before polling\r
             var pauseTask=NetworkAgent.ProceedEvent.WaitAsync();\r
             await TaskEx.WhenAll(signaledTask, pauseTask);\r
@@ -166,7 +206,7 @@ namespace Pithos.Core.Agents
             return since;\r
         }\r
 \r
-        public async Task ProcessAccountFiles(AccountInfo accountInfo, DateTime? since = null)\r
+        public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, DateTime? since = null)\r
         {\r
             if (accountInfo == null)\r
                 throw new ArgumentNullException("accountInfo");\r
@@ -175,24 +215,33 @@ namespace Pithos.Core.Agents
             Contract.EndContractBlock();\r
 \r
 \r
-            using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))\r
+            using (ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))\r
             {\r
+\r
                 await NetworkAgent.GetDeleteAwaiter();\r
 \r
                 Log.Info("Scheduled");\r
                 var client = new CloudFilesClient(accountInfo);\r
 \r
-                var containers = client.ListContainers(accountInfo.UserName);\r
+                //We don't need to check the trash container\r
+                var containers = client.ListContainers(accountInfo.UserName)\r
+                    .Where(c=>c.Name!="trash")\r
+                    .ToList();\r
 \r
 \r
                 CreateContainerFolders(accountInfo, containers);\r
 \r
+                //The nextSince time fallback time is the same as the current.\r
+                //If polling succeeds, the next Since time will be the smallest of the maximum modification times\r
+                //of the shared and account objects\r
+                var nextSince = since;\r
+\r
                 try\r
                 {\r
+                    //Wait for any deletions to finish\r
                     await NetworkAgent.GetDeleteAwaiter();\r
                     //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
                     //than delete a file that was created while we were executing the poll                    \r
-                    var pollTime = DateTime.Now;\r
 \r
                     //Get the list of server objects changed since the last check\r
                     //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
@@ -200,22 +249,27 @@ namespace Pithos.Core.Agents
                                        select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
                                              client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
 \r
-                    var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => client.ListSharedObjects(since), "shared");\r
+                    var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
+                        client.ListSharedObjects(since), "shared");\r
                     listObjects.Add(listShared);\r
                     var listTasks = await Task.Factory.WhenAll(listObjects.ToArray());\r
 \r
-                    using (log4net.ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
+                    using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
                     {\r
                         var dict = listTasks.ToDictionary(t => t.AsyncState);\r
 \r
                         //Get all non-trash objects. Remember, the container name is stored in AsyncState\r
-                        var remoteObjects = from objectList in listTasks\r
+                        var remoteObjects = (from objectList in listTasks\r
                                             where (string)objectList.AsyncState != "trash"\r
                                             from obj in objectList.Result\r
-                                            select obj;\r
+                                            select obj).ToList();\r
+                        \r
+                        //Get the latest remote object modification date, only if it is after\r
+                        //the original since date\r
+                        nextSince = GetLatestDateAfter(nextSince, remoteObjects);\r
 \r
-                        var trashObjects = dict["trash"].Result;\r
                         var sharedObjects = dict["shared"].Result;\r
+                        nextSince = GetLatestDateBefore(nextSince, sharedObjects);\r
 \r
                         //DON'T process trashed files\r
                         //If some files are deleted and added again to a folder, they will be deleted\r
@@ -234,26 +288,40 @@ namespace Pithos.Core.Agents
 */\r
 \r
                         var cleanRemotes = (from info in remoteObjects.Union(sharedObjects)\r
-                                            let name = info.Name\r
+                                            let name = info.Name??""\r
                                             where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&\r
                                                   !name.StartsWith(FolderConstants.CacheFolder + "/",\r
                                                                    StringComparison.InvariantCultureIgnoreCase)\r
                                             select info).ToList();\r
 \r
+                        if (_firstPoll)\r
+                            StatusKeeper.CleanupOrphanStates();\r
+                        StatusKeeper.CleanupStaleStates(accountInfo, cleanRemotes);\r
+                        \r
                         var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
 \r
-                        ProcessDeletedFiles(accountInfo, differencer.Deleted, pollTime);\r
+                        var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
+\r
+                        ProcessDeletedFiles(accountInfo, differencer.Deleted.FilterDirectlyBelow(filterUris));\r
+\r
+                        // @@@ NEED To add previous state here as well, To compare with previous hash\r
+\r
+                        \r
 \r
                         //Create a list of actions from the remote files\r
-                        var allActions = ChangesToActions(accountInfo, differencer.Changed)\r
+                        \r
+                        var allActions = MovesToActions(accountInfo,differencer.Moved.FilterDirectlyBelow(filterUris))\r
                                         .Union(\r
-                                        CreatesToActions(accountInfo, differencer.Created));\r
+                                        ChangesToActions(accountInfo, differencer.Changed.FilterDirectlyBelow(filterUris)))\r
+                                        .Union(\r
+                                        CreatesToActions(accountInfo, differencer.Created.FilterDirectlyBelow(filterUris)));\r
 \r
                         //And remove those that are already being processed by the agent\r
                         var distinctActions = allActions\r
-                            .Except(NetworkAgent.GetEnumerable(), new PithosMonitor.LocalFileComparer())\r
+                            .Except(NetworkAgent.GetEnumerable(), new LocalFileComparer())\r
                             .ToList();\r
 \r
+                        await _unPauseEvent.WaitAsync();\r
                         //Queue all the actions\r
                         foreach (var message in distinctActions)\r
                         {\r
@@ -266,23 +334,62 @@ namespace Pithos.Core.Agents
                 catch (Exception ex)\r
                 {\r
                     Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);\r
-                    return;\r
+                    return nextSince;\r
                 }\r
 \r
                 Log.Info("[LISTENER] Finished");\r
-\r
+                return nextSince;\r
             }\r
         }\r
 \r
-        AccountsDifferencer _differencer = new AccountsDifferencer();\r
+        /// <summary>\r
+        /// Returns the latest LastModified date from the list of objects, but only if it is before\r
+        /// than the threshold value\r
+        /// </summary>\r
+        /// <param name="threshold"></param>\r
+        /// <param name="cloudObjects"></param>\r
+        /// <returns></returns>\r
+        private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
+        {\r
+            DateTime? maxDate = null;\r
+            if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
+                maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
+            if (maxDate == null || maxDate == DateTime.MinValue)\r
+                return threshold;\r
+            if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate)\r
+                return maxDate;\r
+            return threshold;\r
+        }\r
+\r
+        /// <summary>\r
+        /// Returns the latest LastModified date from the list of objects, but only if it is after\r
+        /// the threshold value\r
+        /// </summary>\r
+        /// <param name="threshold"></param>\r
+        /// <param name="cloudObjects"></param>\r
+        /// <returns></returns>\r
+        private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
+        {\r
+            DateTime? maxDate = null;\r
+            if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
+                maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
+            if (maxDate == null || maxDate == DateTime.MinValue)\r
+                return threshold;\r
+            if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate)\r
+                return maxDate;\r
+            return threshold;\r
+        }\r
+\r
+        readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
+        private Dictionary<Uri, List<Uri>> _selectiveUris = new Dictionary<Uri, List<Uri>>();\r
+        private bool _pause;\r
 \r
         /// <summary>\r
         /// Deletes local files that are not found in the list of cloud files\r
         /// </summary>\r
         /// <param name="accountInfo"></param>\r
         /// <param name="cloudFiles"></param>\r
-        /// <param name="pollTime"></param>\r
-        private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles, DateTime pollTime)\r
+        private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles)\r
         {\r
             if (accountInfo == null)\r
                 throw new ArgumentNullException("accountInfo");\r
@@ -320,7 +427,7 @@ namespace Pithos.Core.Agents
                     {\r
                         if (gate.Failed)\r
                             continue;\r
-                        StatusKeeper.SetFileState(item.FullName, FileStatus.Conflict, FileOverlayStatus.Deleted);\r
+                        StatusKeeper.SetFileState(item.FullName, FileStatus.Conflict, FileOverlayStatus.Deleted,"Local file missing from server");\r
                     }\r
                 }\r
                 UpdateStatus(PithosStatus.HasConflicts);\r
@@ -332,8 +439,12 @@ namespace Pithos.Core.Agents
                 var deletedFiles = new List<FileSystemInfo>();\r
                 foreach (var objectInfo in cloudFiles)\r
                 {\r
+                    if (Log.IsDebugEnabled)\r
+                        Log.DebugFormat("Handle deleted [{0}]",objectInfo.Uri);\r
                     var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
-                    var item = GetFileAgent(accountInfo).GetFileSystemInfo(relativePath);\r
+                    var item = FileAgent.GetFileAgent(accountInfo).GetFileSystemInfo(relativePath);\r
+                    if (Log.IsDebugEnabled)\r
+                        Log.DebugFormat("Will delete [{0}] for [{1}]", item.FullName,objectInfo.Uri);\r
                     if (item.Exists)\r
                     {\r
                         if ((item.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)\r
@@ -341,37 +452,52 @@ namespace Pithos.Core.Agents
                             item.Attributes = item.Attributes & ~FileAttributes.ReadOnly;\r
 \r
                         }\r
-                        item.Delete();\r
+                        \r
+                        \r
+                        Log.DebugFormat("Deleting {0}", item.FullName);\r
+\r
+                        var directory = item as DirectoryInfo;\r
+                        if (directory!=null)\r
+                            directory.Delete(true);\r
+                        else\r
+                            item.Delete();\r
+                        Log.DebugFormat("Deleted [{0}] for [{1}]", item.FullName, objectInfo.Uri);\r
                         DateTime lastDate;\r
                         _lastSeen.TryRemove(item.FullName, out lastDate);\r
                         deletedFiles.Add(item);\r
                     }\r
-                    StatusKeeper.SetFileState(item.FullName, FileStatus.Deleted, FileOverlayStatus.Deleted);\r
+                    StatusKeeper.SetFileState(item.FullName, FileStatus.Deleted, FileOverlayStatus.Deleted, "File Deleted");\r
                 }\r
+                Log.InfoFormat("[{0}] files were deleted",deletedFiles.Count);\r
                 StatusNotification.NotifyForFiles(deletedFiles, String.Format("{0} files were deleted", deletedFiles.Count), TraceLevel.Info);\r
             }\r
 \r
         }\r
 \r
-        //Creates an appropriate action for each server file\r
+        /// <summary>\r
+        /// Creates a Sync action for each changed server file\r
+        /// </summary>\r
+        /// <param name="accountInfo"></param>\r
+        /// <param name="changes"></param>\r
+        /// <returns></returns>\r
         private IEnumerable<CloudAction> ChangesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> changes)\r
         {\r
             if (changes == null)\r
                 throw new ArgumentNullException();\r
             Contract.EndContractBlock();\r
-            var fileAgent = GetFileAgent(accountInfo);\r
+            var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
 \r
             //In order to avoid multiple iterations over the files, we iterate only once\r
             //over the remote files\r
             foreach (var objectInfo in changes)\r
             {\r
                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
-                //and remove any matching objects from the list, adding them to the commonObjects list\r
+                //If a directory object already exists, we may need to sync it\r
                 if (fileAgent.Exists(relativePath))\r
                 {\r
-                    //If a directory object already exists, we don't need to perform any other action                    \r
                     var localFile = fileAgent.GetFileSystemInfo(relativePath);\r
-                    if (objectInfo.Content_Type == @"application/directory" && localFile is DirectoryInfo)\r
+                    //We don't need to sync directories\r
+                    if (objectInfo.IsDirectory && localFile is DirectoryInfo)\r
                         continue;\r
                     using (new SessionScope(FlushAction.Never))\r
                     {\r
@@ -392,26 +518,43 @@ namespace Pithos.Core.Agents
             }\r
         }\r
 \r
-        private IEnumerable<CloudAction> CreatesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> creates)\r
+        /// <summary>\r
+        /// Creates a Local Move action for each moved server file\r
+        /// </summary>\r
+        /// <param name="accountInfo"></param>\r
+        /// <param name="moves"></param>\r
+        /// <returns></returns>\r
+        private IEnumerable<CloudAction> MovesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> moves)\r
         {\r
-            if (creates == null)\r
+            if (moves == null)\r
                 throw new ArgumentNullException();\r
             Contract.EndContractBlock();\r
-            var fileAgent = GetFileAgent(accountInfo);\r
+            var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
 \r
             //In order to avoid multiple iterations over the files, we iterate only once\r
             //over the remote files\r
-            foreach (var objectInfo in creates)\r
+            foreach (var objectInfo in moves)\r
             {\r
-                var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
-                //and remove any matching objects from the list, adding them to the commonObjects list\r
-                if (fileAgent.Exists(relativePath))\r
+                var previousRelativepath = objectInfo.Previous.RelativeUrlToFilePath(accountInfo.UserName);\r
+                //If the previous file already exists, we can execute a Move operation\r
+                if (fileAgent.Exists(previousRelativepath))\r
                 {\r
-                    //If the object already exists, we probably have a conflict\r
-                    //If a directory object already exists, we don't need to perform any other action                    \r
-                    var localFile = fileAgent.GetFileSystemInfo(relativePath);\r
-                    StatusKeeper.SetFileState(localFile.FullName, FileStatus.Conflict, FileOverlayStatus.Conflict);\r
+                    var previousFile = fileAgent.GetFileSystemInfo(previousRelativepath);\r
+                    using (new SessionScope(FlushAction.Never))\r
+                    {\r
+                        var state = StatusKeeper.GetStateByFilePath(previousFile.FullName);\r
+                        _lastSeen[previousFile.FullName] = DateTime.Now;\r
+\r
+                        //For each moved object we need to move both the local file and update                                                \r
+                        yield return new CloudAction(accountInfo, CloudActionType.RenameLocal,\r
+                                                     previousFile, objectInfo, state, accountInfo.BlockSize,\r
+                                                     accountInfo.BlockHash);\r
+                        //For modified files, we need to download the changes as well\r
+                        if (objectInfo.Hash!=objectInfo.PreviousHash)\r
+                            yield return new CloudDownloadAction(accountInfo,objectInfo);\r
+                    }\r
                 }\r
+                //If the previous file does not exist, we need to download it in the new location\r
                 else\r
                 {\r
                     //Remote files should be downloaded\r
@@ -420,28 +563,63 @@ namespace Pithos.Core.Agents
             }\r
         }\r
 \r
-        private static FileAgent GetFileAgent(AccountInfo accountInfo)\r
-        {\r
-            return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);\r
-        }\r
 \r
-        private void ProcessTrashedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> trashObjects)\r
+        /// <summary>\r
+        /// Creates a download action for each new server file\r
+        /// </summary>\r
+        /// <param name="accountInfo"></param>\r
+        /// <param name="creates"></param>\r
+        /// <returns></returns>\r
+        private IEnumerable<CloudAction> CreatesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> creates)\r
         {\r
-            var fileAgent = GetFileAgent(accountInfo);\r
-            foreach (var trashObject in trashObjects)\r
+            if (creates == null)\r
+                throw new ArgumentNullException();\r
+            Contract.EndContractBlock();\r
+            var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
+\r
+            //In order to avoid multiple iterations over the files, we iterate only once\r
+            //over the remote files\r
+            foreach (var objectInfo in creates)\r
             {\r
-                var barePath = trashObject.RelativeUrlToFilePath(accountInfo.UserName);\r
-                //HACK: Assume only the "pithos" container is used. Must find out what happens when\r
-                //deleting a file from a different container\r
-                var relativePath = Path.Combine("pithos", barePath);\r
-                fileAgent.Delete(relativePath);\r
+                if (Log.IsDebugEnabled)\r
+                    Log.DebugFormat("[NEW INFO] {0}",objectInfo.Uri);\r
+\r
+                var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
+\r
+                //If the object already exists, we should check before uploading or downloading\r
+                if (fileAgent.Exists(relativePath))\r
+                {\r
+                    var localFile= fileAgent.GetFileSystemInfo(relativePath);\r
+                    var state = StatusKeeper.GetStateByFilePath(localFile.WithProperCapitalization().FullName);\r
+                    yield return new CloudAction(accountInfo, CloudActionType.MustSynch,\r
+                                                     localFile, objectInfo, state, accountInfo.BlockSize,\r
+                                                     accountInfo.BlockHash);                    \r
+                }\r
+                else\r
+                {\r
+                    //Remote files should be downloaded\r
+                    yield return new CloudDownloadAction(accountInfo, objectInfo);\r
+                }\r
+\r
             }\r
         }\r
 \r
+        /// <summary>\r
+        /// Notify the UI to update the visual status\r
+        /// </summary>\r
+        /// <param name="status"></param>\r
         private void UpdateStatus(PithosStatus status)\r
         {\r
-            StatusKeeper.SetPithosStatus(status);\r
-            StatusNotification.Notify(new Notification());\r
+            try\r
+            {\r
+                StatusNotification.SetPithosStatus(status);\r
+                //StatusNotification.Notify(new Notification());\r
+            }\r
+            catch (Exception exc)\r
+            {\r
+                //Failure is not critical, just log it\r
+                Log.Warn("Error while updating status", exc);\r
+            }\r
         }\r
 \r
         private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)\r
@@ -457,5 +635,80 @@ namespace Pithos.Core.Agents
             }\r
         }\r
 \r
+        public void AddAccount(AccountInfo accountInfo)\r
+        {\r
+            //Avoid adding a duplicate accountInfo\r
+            _accounts.TryAdd(accountInfo.AccountKey, accountInfo);\r
+        }\r
+\r
+        public void RemoveAccount(AccountInfo accountInfo)\r
+        {\r
+            AccountInfo account;\r
+            _accounts.TryRemove(accountInfo.AccountKey, out account);\r
+            SnapshotDifferencer differencer;\r
+            _differencer.Differencers.TryRemove(accountInfo.AccountKey, out differencer);\r
+        }\r
+\r
+        public void SetSelectivePaths(AccountInfo accountInfo,Uri[] added, Uri[] removed)\r
+        {\r
+            AbortRemovedPaths(accountInfo,removed);\r
+            DownloadNewPaths(accountInfo,added);\r
+        }\r
+\r
+        private void DownloadNewPaths(AccountInfo accountInfo, Uri[] added)\r
+        {\r
+            var client = new CloudFilesClient(accountInfo);\r
+            foreach (var folderUri in added)\r
+            {\r
+                try\r
+                {\r
+\r
+                    string account;\r
+                    string container;\r
+                    var segmentsCount = folderUri.Segments.Length;\r
+                    if (segmentsCount < 3)\r
+                        continue;\r
+                    if (segmentsCount == 3)\r
+                    {\r
+                        account = folderUri.Segments[1].TrimEnd('/');\r
+                        container = folderUri.Segments[2].TrimEnd('/');\r
+                    }\r
+                    else\r
+                    {\r
+                        account = folderUri.Segments[2].TrimEnd('/');\r
+                        container = folderUri.Segments[3].TrimEnd('/');\r
+                    }\r
+                    IList<ObjectInfo> items;\r
+                    if (segmentsCount > 3)\r
+                    {\r
+                        var folder = String.Join("", folderUri.Segments.Splice(4));\r
+                        items = client.ListObjects(account, container, folder);\r
+                    }\r
+                    else\r
+                    {\r
+                        items = client.ListObjects(account, container);\r
+                    }\r
+                    var actions = CreatesToActions(accountInfo, items);\r
+                    foreach (var action in actions)\r
+                    {\r
+                        NetworkAgent.Post(action);\r
+                    }\r
+                }\r
+                catch (Exception exc)\r
+                {\r
+                    Log.WarnFormat("Listing of new selective path [{0}] failed with \r\n{1}", folderUri, exc);\r
+                }\r
+            }\r
+\r
+            //Need to get a listing of each of the URLs, then post them to the NetworkAgent\r
+            //CreatesToActions(accountInfo,)\r
+\r
+/*            NetworkAgent.Post();*/\r
+        }\r
+\r
+        private void AbortRemovedPaths(AccountInfo accountInfo, Uri[] removed)\r
+        {\r
+            /*this.NetworkAgent.*/\r
+        }\r
     }\r
 }\r