[System.ComponentModel.Composition.Import]
public IPithosSettings Settings { get; set; }
- private bool _firstPoll = true;
-
- //The Sync Event signals a manual synchronisation
- private readonly AsyncManualResetEvent _syncEvent=new AsyncManualResetEvent();
+ //The Pause event stops the poll agent to give priority to the network agent
+ //Initially the event is signalled because we don't need to pause
+ private readonly AsyncManualResetEvent _pauseEvent = new AsyncManualResetEvent(true);
+
+ public AsyncManualResetEvent PauseEvent
+ {
+ get { return _pauseEvent; }
+ }
- private ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>();
public void Start()
{
- _firstPoll = true;
_agent = Agent<CloudAction>.Start(inbox =>
{
Action loop = null;
throw new ArgumentException("The action.AccountInfo is empty","action");
Contract.EndContractBlock();
- UpdateStatus(PithosStatus.Syncing);
- var accountInfo = action.AccountInfo;
+
+
using (log4net.ThreadContext.Stacks["NETWORK"].Push("PROCESS"))
{
var downloadPath = action.GetDownloadPath();
try
- {
+ {
+ _pauseEvent.Reset();
+ UpdateStatus(PithosStatus.Syncing);
+ var accountInfo = action.AccountInfo;
+
if (action.Action == CloudActionType.DeleteCloud)
{
//Redirect deletes to the delete agent
}
finally
{
- UpdateStatus(PithosStatus.InSynch);
+ if (_agent.IsEmpty)
+ _pauseEvent.Set();
+ UpdateStatus(PithosStatus.InSynch);
}
}
}
}
- /// <summary>
- /// Start a manual synchronization
- /// </summary>
- public void SynchNow()
- {
- _syncEvent.Set();
- }
-
- //Remote files are polled periodically. Any changes are processed
- public async Task PollRemoteFiles(DateTime? since = null)
+ public IEnumerable<CloudAction> GetEnumerable()
{
- Debug.Assert(Thread.CurrentThread.IsBackground,"Polling Ended up in the main thread!");
-
- UpdateStatus(PithosStatus.Syncing);
- StatusNotification.Notify(new PollNotification());
-
- using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))
- {
- //If this poll fails, we will retry with the same since value
- var nextSince = since;
- try
- {
- //Next time we will check for all changes since the current check minus 1 second
- //This is done to ensure there are no discrepancies due to clock differences
- var current = DateTime.Now.AddSeconds(-1);
-
- var tasks = from accountInfo in _accounts
- select ProcessAccountFiles(accountInfo, since);
-
- await TaskEx.WhenAll(tasks.ToList());
-
- _firstPoll = false;
- //Reschedule the poll with the current timestamp as a "since" value
- nextSince = current;
- }
- catch (Exception ex)
- {
- Log.ErrorFormat("Error while processing accounts\r\n{0}",ex);
- //In case of failure retry with the same "since" value
- }
-
- UpdateStatus(PithosStatus.InSynch);
- //Wait for the polling interval to pass or the Sync event to be signalled
- nextSince = await WaitForScheduledOrManualPoll(nextSince);
-
- PollRemoteFiles(nextSince);
-
- }
+ return _agent.GetEnumerable();
}
- /// <summary>
- /// Wait for the polling period to expire or a manual sync request
- /// </summary>
- /// <param name="since"></param>
- /// <returns></returns>
- private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)
+ public Task GetDeleteAwaiter()
{
- var sync=_syncEvent.WaitAsync();
- var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval), _agent.CancellationToken);
- var signaledTask = await TaskEx.WhenAny(sync, wait);
-
- //If polling is signalled by SynchNow, ignore the since tag
- if (signaledTask is Task<bool>)
- return null;
- return since;
+ return _deleteAgent.PauseEvent.WaitAsync();
}
-
- public async Task ProcessAccountFiles(AccountInfo accountInfo,DateTime? since=null)
- {
- if (accountInfo==null)
- throw new ArgumentNullException("accountInfo");
- if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))
- throw new ArgumentException("The AccountInfo.AccountPath is empty","accountInfo");
- Contract.EndContractBlock();
-
-
- using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))
- {
- await _deleteAgent.PauseEvent.WaitAsync();
-
- Log.Info("Scheduled");
- var client = new CloudFilesClient(accountInfo);
-
- var containers = client.ListContainers(accountInfo.UserName);
-
-
- CreateContainerFolders(accountInfo, containers);
-
- try
- {
- await _deleteAgent.PauseEvent.WaitAsync();
- //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted
- //than delete a file that was created while we were executing the poll
- var pollTime = DateTime.Now;
-
- //Get the list of server objects changed since the last check
- //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step
- var listObjects = (from container in containers
- select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>
- client.ListObjects(accountInfo.UserName,container.Name, since),container.Name)).ToList();
-
- var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => client.ListSharedObjects(since), "shared");
- listObjects.Add(listShared);
- var listTasks = await Task.Factory.WhenAll(listObjects.ToArray());
-
- using (log4net.ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))
- {
- var dict = listTasks.ToDictionary(t => t.AsyncState);
-
- //Get all non-trash objects. Remember, the container name is stored in AsyncState
- var remoteObjects = from objectList in listTasks
- where (string) objectList.AsyncState != "trash"
- from obj in objectList.Result
- select obj;
-
- var trashObjects = dict["trash"].Result;
- var sharedObjects = dict["shared"].Result;
-
- //Items with the same name, hash may be both in the container and the trash
- //Don't delete items that exist in the container
- var realTrash = from trash in trashObjects
- where
- !remoteObjects.Any(
- info => info.Name == trash.Name && info.Hash == trash.Hash)
- select trash;
- ProcessTrashedFiles(accountInfo, realTrash);
-
- var cleanRemotes = (from info in remoteObjects.Union(sharedObjects)
- let name = info.Name
- where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&
- !name.StartsWith(FolderConstants.CacheFolder + "/",
- StringComparison.InvariantCultureIgnoreCase)
- select info).ToList();
-
- var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);
-
- ProcessDeletedFiles(accountInfo, differencer.Deleted, pollTime);
-
- //Create a list of actions from the remote files
- var allActions = ChangesToActions(accountInfo, differencer.Changed)
- .Union(
- CreatesToActions(accountInfo,differencer.Created));
-
- //And remove those that are already being processed by the agent
- var distinctActions = allActions
- .Except(_agent.GetEnumerable(), new PithosMonitor.LocalFileComparer())
- .ToList();
-
- //Queue all the actions
- foreach (var message in distinctActions)
- {
- Post(message);
- }
-
- Log.Info("[LISTENER] End Processing");
- }
- }
- catch (Exception ex)
- {
- Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);
- return;
- }
-
- Log.Info("[LISTENER] Finished");
-
- }
- }
-
- AccountsDifferencer _differencer= new AccountsDifferencer();
-
- /// <summary>
- /// Deletes local files that are not found in the list of cloud files
- /// </summary>
- /// <param name="accountInfo"></param>
- /// <param name="cloudFiles"></param>
- /// <param name="pollTime"></param>
- private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles, DateTime pollTime)
+ public CancellationToken CancellationToken
{
- if (accountInfo == null)
- throw new ArgumentNullException("accountInfo");
- if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))
- throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");
- if (cloudFiles == null)
- throw new ArgumentNullException("cloudFiles");
- Contract.EndContractBlock();
-
- //On the first run
- if (_firstPoll)
- {
- //Only consider files that are not being modified, ie they are in the Unchanged state
- var deleteCandidates = FileState.Queryable.Where(state =>
- state.FilePath.StartsWith(accountInfo.AccountPath)
- && state.FileStatus == FileStatus.Unchanged).ToList();
-
-
- //TODO: filesToDelete must take into account the Others container
- var filesToDelete = (from deleteCandidate in deleteCandidates
- let localFile = FileInfoExtensions.FromPath(deleteCandidate.FilePath)
- let relativeFilePath = localFile.AsRelativeTo(accountInfo.AccountPath)
- where
- !cloudFiles.Any(r => r.RelativeUrlToFilePath(accountInfo.UserName) == relativeFilePath)
- select localFile).ToList();
-
-
-
- //Set the status of missing files to Conflict
- foreach (var item in filesToDelete)
- {
- //Try to acquire a gate on the file, to take into account files that have been dequeued
- //and are being processed
- using (var gate = NetworkGate.Acquire(item.FullName, NetworkOperation.Deleting))
- {
- if (gate.Failed)
- continue;
- StatusKeeper.SetFileState(item.FullName, FileStatus.Conflict, FileOverlayStatus.Deleted);
- }
- }
- UpdateStatus(PithosStatus.HasConflicts);
- StatusNotification.NotifyConflicts(filesToDelete, String.Format("{0} local files are missing from Pithos, possibly because they were deleted",filesToDelete.Count));
- StatusNotification.NotifyForFiles(filesToDelete, String.Format("{0} files were deleted", filesToDelete.Count), TraceLevel.Info);
- }
- else
- {
- var deletedFiles = new List<FileSystemInfo>();
- foreach (var objectInfo in cloudFiles)
- {
- var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);
- var item = GetFileAgent(accountInfo).GetFileSystemInfo(relativePath);
- if (item.Exists)
- {
- if ((item.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
- {
- item.Attributes = item.Attributes & ~FileAttributes.ReadOnly;
-
- }
- item.Delete();
- DateTime lastDate;
- _lastSeen.TryRemove(item.FullName, out lastDate);
- deletedFiles.Add(item);
- }
- StatusKeeper.SetFileState(item.FullName, FileStatus.Deleted, FileOverlayStatus.Deleted);
- }
- StatusNotification.NotifyForFiles(deletedFiles, String.Format("{0} files were deleted", deletedFiles.Count), TraceLevel.Info);
- }
-
- }
-
- private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)
- {
- var containerPaths = from container in containers
- let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)
- where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)
- select containerPath;
-
- foreach (var path in containerPaths)
- {
- Directory.CreateDirectory(path);
- }
- }
-
- //Creates an appropriate action for each server file
- private IEnumerable<CloudAction> ChangesToActions(AccountInfo accountInfo,IEnumerable<ObjectInfo> changes)
- {
- if (changes==null)
- throw new ArgumentNullException();
- Contract.EndContractBlock();
- var fileAgent = GetFileAgent(accountInfo);
-
- //In order to avoid multiple iterations over the files, we iterate only once
- //over the remote files
- foreach (var objectInfo in changes)
- {
- var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);
- //and remove any matching objects from the list, adding them to the commonObjects list
- if (fileAgent.Exists(relativePath))
- {
- //If a directory object already exists, we don't need to perform any other action
- var localFile = fileAgent.GetFileSystemInfo(relativePath);
- if (objectInfo.Content_Type == @"application/directory" && localFile is DirectoryInfo)
- continue;
- using (new SessionScope(FlushAction.Never))
- {
- var state = StatusKeeper.GetStateByFilePath(localFile.FullName);
- _lastSeen[localFile.FullName] = DateTime.Now;
- //Common files should be checked on a per-case basis to detect differences, which is newer
-
- yield return new CloudAction(accountInfo, CloudActionType.MustSynch,
- localFile, objectInfo, state, accountInfo.BlockSize,
- accountInfo.BlockHash);
- }
- }
- else
- {
- //Remote files should be downloaded
- yield return new CloudDownloadAction(accountInfo,objectInfo);
- }
- }
+ get { return _agent.CancellationToken; }
}
- private IEnumerable<CloudAction> CreatesToActions(AccountInfo accountInfo,IEnumerable<ObjectInfo> creates)
- {
- if (creates==null)
- throw new ArgumentNullException();
- Contract.EndContractBlock();
- var fileAgent = GetFileAgent(accountInfo);
-
- //In order to avoid multiple iterations over the files, we iterate only once
- //over the remote files
- foreach (var objectInfo in creates)
- {
- var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);
- //and remove any matching objects from the list, adding them to the commonObjects list
- if (fileAgent.Exists(relativePath))
- {
- //If the object already exists, we probably have a conflict
- //If a directory object already exists, we don't need to perform any other action
- var localFile = fileAgent.GetFileSystemInfo(relativePath);
- StatusKeeper.SetFileState(localFile.FullName,FileStatus.Conflict,FileOverlayStatus.Conflict);
- }
- else
- {
- //Remote files should be downloaded
- yield return new CloudDownloadAction(accountInfo,objectInfo);
- }
- }
- }
-
-
private static FileAgent GetFileAgent(AccountInfo accountInfo)
{
return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);
}
- private void ProcessTrashedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects)
- {
- var fileAgent = GetFileAgent(accountInfo);
- foreach (var trashObject in trashObjects)
- {
- var barePath = trashObject.RelativeUrlToFilePath(accountInfo.UserName);
- //HACK: Assume only the "pithos" container is used. Must find out what happens when
- //deleting a file from a different container
- var relativePath = Path.Combine("pithos", barePath);
- fileAgent.Delete(relativePath);
- }
- }
private void RenameCloudFile(AccountInfo accountInfo,CloudMoveAction action)
--- /dev/null
+#region\r
+/* -----------------------------------------------------------------------\r
+ * <copyright file="PollAgent.cs" company="GRNet">\r
+ * \r
+ * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * 1. Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * 2. Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ *\r
+ * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
+ * POSSIBILITY OF SUCH DAMAGE.\r
+ *\r
+ * The views and conclusions contained in the software and\r
+ * documentation are those of the authors and should not be\r
+ * interpreted as representing official policies, either expressed\r
+ * or implied, of GRNET S.A.\r
+ * </copyright>\r
+ * -----------------------------------------------------------------------\r
+ */\r
+#endregion\r
+\r
+using System.Collections.Concurrent;\r
+using System.ComponentModel.Composition;\r
+using System.Diagnostics;\r
+using System.Diagnostics.Contracts;\r
+using System.IO;\r
+using System.Threading;\r
+using System.Threading.Tasks;\r
+using System.Threading.Tasks.Dataflow;\r
+using Castle.ActiveRecord;\r
+using Pithos.Interfaces;\r
+using Pithos.Network;\r
+using log4net;\r
+\r
+namespace Pithos.Core.Agents\r
+{\r
+ using System;\r
+ using System.Collections.Generic;\r
+ using System.Linq;\r
+ using System.Text;\r
+\r
+ /// <summary>\r
+ /// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all\r
+ /// objects and compares it with a previously cached version to detect differences. \r
+ /// New files are downloaded, missing files are deleted from the local file system and common files are compared\r
+ /// to determine the appropriate action\r
+ /// </summary>\r
+ [Export]\r
+ public class PollAgent\r
+ {\r
+ private static readonly ILog Log = LogManager.GetLogger("PollAgent");\r
+\r
+ [System.ComponentModel.Composition.Import]\r
+ public IStatusKeeper StatusKeeper { get; set; }\r
+\r
+ [System.ComponentModel.Composition.Import]\r
+ public IPithosSettings Settings { get; set; }\r
+\r
+ [System.ComponentModel.Composition.Import]\r
+ public NetworkAgent NetworkAgent { get; set; }\r
+\r
+ public IStatusNotification StatusNotification { get; set; }\r
+\r
+ private bool _firstPoll = true;\r
+\r
+ //The Sync Event signals a manual synchronisation\r
+ private readonly AsyncManualResetEvent _syncEvent = new AsyncManualResetEvent();\r
+\r
+ private ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>();\r
+ private readonly ConcurrentBag<AccountInfo> _accounts = new ConcurrentBag<AccountInfo>();\r
+\r
+\r
+ /// <summary>\r
+ /// Start a manual synchronization\r
+ /// </summary>\r
+ public void SynchNow()\r
+ { \r
+ _syncEvent.Set();\r
+ }\r
+\r
+ //Remote files are polled periodically. Any changes are processed\r
+ public async Task PollRemoteFiles(DateTime? since = null)\r
+ {\r
+ Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
+\r
+ UpdateStatus(PithosStatus.Syncing);\r
+ StatusNotification.Notify(new PollNotification());\r
+\r
+ using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
+ {\r
+ //If this poll fails, we will retry with the same since value\r
+ var nextSince = since;\r
+ try\r
+ {\r
+ //Next time we will check for all changes since the current check minus 1 second\r
+ //This is done to ensure there are no discrepancies due to clock differences\r
+ var current = DateTime.Now.AddSeconds(-1);\r
+\r
+ var tasks = from accountInfo in _accounts\r
+ select ProcessAccountFiles(accountInfo, since);\r
+\r
+ await TaskEx.WhenAll(tasks.ToList());\r
+\r
+ _firstPoll = false;\r
+ //Reschedule the poll with the current timestamp as a "since" value\r
+ nextSince = current;\r
+ }\r
+ catch (Exception ex)\r
+ {\r
+ Log.ErrorFormat("Error while processing accounts\r\n{0}", ex);\r
+ //In case of failure retry with the same "since" value\r
+ }\r
+\r
+ UpdateStatus(PithosStatus.InSynch);\r
+ //Wait for the polling interval to pass or the Sync event to be signalled\r
+ nextSince = await WaitForScheduledOrManualPoll(nextSince);\r
+\r
+ TaskEx.Run(()=>PollRemoteFiles(nextSince));\r
+\r
+ }\r
+ }\r
+\r
+ /// <summary>\r
+ /// Wait for the polling period to expire or a manual sync request\r
+ /// </summary>\r
+ /// <param name="since"></param>\r
+ /// <returns></returns>\r
+ private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
+ {\r
+ var sync = _syncEvent.WaitAsync();\r
+ var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval), NetworkAgent.CancellationToken);\r
+ var signaledTask = await TaskEx.WhenAny(sync, wait);\r
+\r
+ //Wait for network processing to finish before polling\r
+ var pauseTask=NetworkAgent.PauseEvent.WaitAsync();\r
+ await TaskEx.WhenAll(signaledTask, pauseTask);\r
+\r
+ //If polling is signalled by SynchNow, ignore the since tag\r
+ if (sync.IsCompleted)\r
+ {\r
+ //TODO: Must convert to AutoReset\r
+ _syncEvent.Reset();\r
+ return null;\r
+ }\r
+ return since;\r
+ }\r
+\r
+ public async Task ProcessAccountFiles(AccountInfo accountInfo, DateTime? since = null)\r
+ {\r
+ if (accountInfo == null)\r
+ throw new ArgumentNullException("accountInfo");\r
+ if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
+ throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
+ Contract.EndContractBlock();\r
+\r
+\r
+ using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))\r
+ {\r
+ await NetworkAgent.GetDeleteAwaiter();\r
+\r
+ Log.Info("Scheduled");\r
+ var client = new CloudFilesClient(accountInfo);\r
+\r
+ var containers = client.ListContainers(accountInfo.UserName);\r
+\r
+\r
+ CreateContainerFolders(accountInfo, containers);\r
+\r
+ try\r
+ {\r
+ await NetworkAgent.GetDeleteAwaiter();\r
+ //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
+ //than delete a file that was created while we were executing the poll \r
+ var pollTime = DateTime.Now;\r
+\r
+ //Get the list of server objects changed since the last check\r
+ //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
+ var listObjects = (from container in containers\r
+ select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
+ client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
+\r
+ var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => client.ListSharedObjects(since), "shared");\r
+ listObjects.Add(listShared);\r
+ var listTasks = await Task.Factory.WhenAll(listObjects.ToArray());\r
+\r
+ using (log4net.ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
+ {\r
+ var dict = listTasks.ToDictionary(t => t.AsyncState);\r
+\r
+ //Get all non-trash objects. Remember, the container name is stored in AsyncState\r
+ var remoteObjects = from objectList in listTasks\r
+ where (string)objectList.AsyncState != "trash"\r
+ from obj in objectList.Result\r
+ select obj;\r
+\r
+ var trashObjects = dict["trash"].Result;\r
+ var sharedObjects = dict["shared"].Result;\r
+\r
+ //DON'T process trashed files\r
+ //If some files are deleted and added again to a folder, they will be deleted\r
+ //even though they are new.\r
+ //We would have to check file dates and hashes to ensure that a trashed file\r
+ //can be deleted safely from the local hard drive.\r
+ /*\r
+ //Items with the same name, hash may be both in the container and the trash\r
+ //Don't delete items that exist in the container\r
+ var realTrash = from trash in trashObjects\r
+ where\r
+ !remoteObjects.Any(\r
+ info => info.Name == trash.Name && info.Hash == trash.Hash)\r
+ select trash;\r
+ ProcessTrashedFiles(accountInfo, realTrash);\r
+*/\r
+\r
+ var cleanRemotes = (from info in remoteObjects.Union(sharedObjects)\r
+ let name = info.Name\r
+ where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&\r
+ !name.StartsWith(FolderConstants.CacheFolder + "/",\r
+ StringComparison.InvariantCultureIgnoreCase)\r
+ select info).ToList();\r
+\r
+ var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
+\r
+ ProcessDeletedFiles(accountInfo, differencer.Deleted, pollTime);\r
+\r
+ //Create a list of actions from the remote files\r
+ var allActions = ChangesToActions(accountInfo, differencer.Changed)\r
+ .Union(\r
+ CreatesToActions(accountInfo, differencer.Created));\r
+\r
+ //And remove those that are already being processed by the agent\r
+ var distinctActions = allActions\r
+ .Except(NetworkAgent.GetEnumerable(), new PithosMonitor.LocalFileComparer())\r
+ .ToList();\r
+\r
+ //Queue all the actions\r
+ foreach (var message in distinctActions)\r
+ {\r
+ NetworkAgent.Post(message);\r
+ }\r
+\r
+ Log.Info("[LISTENER] End Processing");\r
+ }\r
+ }\r
+ catch (Exception ex)\r
+ {\r
+ Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);\r
+ return;\r
+ }\r
+\r
+ Log.Info("[LISTENER] Finished");\r
+\r
+ }\r
+ }\r
+\r
+ AccountsDifferencer _differencer = new AccountsDifferencer();\r
+\r
+ /// <summary>\r
+ /// Deletes local files that are not found in the list of cloud files\r
+ /// </summary>\r
+ /// <param name="accountInfo"></param>\r
+ /// <param name="cloudFiles"></param>\r
+ /// <param name="pollTime"></param>\r
+ private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles, DateTime pollTime)\r
+ {\r
+ if (accountInfo == null)\r
+ throw new ArgumentNullException("accountInfo");\r
+ if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
+ throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
+ if (cloudFiles == null)\r
+ throw new ArgumentNullException("cloudFiles");\r
+ Contract.EndContractBlock();\r
+\r
+ //On the first run\r
+ if (_firstPoll)\r
+ {\r
+ //Only consider files that are not being modified, ie they are in the Unchanged state \r
+ var deleteCandidates = FileState.Queryable.Where(state =>\r
+ state.FilePath.StartsWith(accountInfo.AccountPath)\r
+ && state.FileStatus == FileStatus.Unchanged).ToList();\r
+\r
+\r
+ //TODO: filesToDelete must take into account the Others container \r
+ var filesToDelete = (from deleteCandidate in deleteCandidates\r
+ let localFile = FileInfoExtensions.FromPath(deleteCandidate.FilePath)\r
+ let relativeFilePath = localFile.AsRelativeTo(accountInfo.AccountPath)\r
+ where\r
+ !cloudFiles.Any(r => r.RelativeUrlToFilePath(accountInfo.UserName) == relativeFilePath)\r
+ select localFile).ToList();\r
+\r
+\r
+\r
+ //Set the status of missing files to Conflict\r
+ foreach (var item in filesToDelete)\r
+ {\r
+ //Try to acquire a gate on the file, to take into account files that have been dequeued\r
+ //and are being processed\r
+ using (var gate = NetworkGate.Acquire(item.FullName, NetworkOperation.Deleting))\r
+ {\r
+ if (gate.Failed)\r
+ continue;\r
+ StatusKeeper.SetFileState(item.FullName, FileStatus.Conflict, FileOverlayStatus.Deleted);\r
+ }\r
+ }\r
+ UpdateStatus(PithosStatus.HasConflicts);\r
+ StatusNotification.NotifyConflicts(filesToDelete, String.Format("{0} local files are missing from Pithos, possibly because they were deleted", filesToDelete.Count));\r
+ StatusNotification.NotifyForFiles(filesToDelete, String.Format("{0} files were deleted", filesToDelete.Count), TraceLevel.Info);\r
+ }\r
+ else\r
+ {\r
+ var deletedFiles = new List<FileSystemInfo>();\r
+ foreach (var objectInfo in cloudFiles)\r
+ {\r
+ var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
+ var item = GetFileAgent(accountInfo).GetFileSystemInfo(relativePath);\r
+ if (item.Exists)\r
+ {\r
+ if ((item.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)\r
+ {\r
+ item.Attributes = item.Attributes & ~FileAttributes.ReadOnly;\r
+\r
+ }\r
+ item.Delete();\r
+ DateTime lastDate;\r
+ _lastSeen.TryRemove(item.FullName, out lastDate);\r
+ deletedFiles.Add(item);\r
+ }\r
+ StatusKeeper.SetFileState(item.FullName, FileStatus.Deleted, FileOverlayStatus.Deleted);\r
+ }\r
+ StatusNotification.NotifyForFiles(deletedFiles, String.Format("{0} files were deleted", deletedFiles.Count), TraceLevel.Info);\r
+ }\r
+\r
+ }\r
+\r
+ //Creates an appropriate action for each server file\r
+ private IEnumerable<CloudAction> ChangesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> changes)\r
+ {\r
+ if (changes == null)\r
+ throw new ArgumentNullException();\r
+ Contract.EndContractBlock();\r
+ var fileAgent = GetFileAgent(accountInfo);\r
+\r
+ //In order to avoid multiple iterations over the files, we iterate only once\r
+ //over the remote files\r
+ foreach (var objectInfo in changes)\r
+ {\r
+ var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
+ //and remove any matching objects from the list, adding them to the commonObjects list\r
+ if (fileAgent.Exists(relativePath))\r
+ {\r
+ //If a directory object already exists, we don't need to perform any other action \r
+ var localFile = fileAgent.GetFileSystemInfo(relativePath);\r
+ if (objectInfo.Content_Type == @"application/directory" && localFile is DirectoryInfo)\r
+ continue;\r
+ using (new SessionScope(FlushAction.Never))\r
+ {\r
+ var state = StatusKeeper.GetStateByFilePath(localFile.FullName);\r
+ _lastSeen[localFile.FullName] = DateTime.Now;\r
+ //Common files should be checked on a per-case basis to detect differences, which is newer\r
+\r
+ yield return new CloudAction(accountInfo, CloudActionType.MustSynch,\r
+ localFile, objectInfo, state, accountInfo.BlockSize,\r
+ accountInfo.BlockHash);\r
+ }\r
+ }\r
+ else\r
+ {\r
+ //Remote files should be downloaded\r
+ yield return new CloudDownloadAction(accountInfo, objectInfo);\r
+ }\r
+ }\r
+ }\r
+\r
+ private IEnumerable<CloudAction> CreatesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> creates)\r
+ {\r
+ if (creates == null)\r
+ throw new ArgumentNullException();\r
+ Contract.EndContractBlock();\r
+ var fileAgent = GetFileAgent(accountInfo);\r
+\r
+ //In order to avoid multiple iterations over the files, we iterate only once\r
+ //over the remote files\r
+ foreach (var objectInfo in creates)\r
+ {\r
+ var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
+ //and remove any matching objects from the list, adding them to the commonObjects list\r
+ if (fileAgent.Exists(relativePath))\r
+ {\r
+ //If the object already exists, we probably have a conflict\r
+ //If a directory object already exists, we don't need to perform any other action \r
+ var localFile = fileAgent.GetFileSystemInfo(relativePath);\r
+ StatusKeeper.SetFileState(localFile.FullName, FileStatus.Conflict, FileOverlayStatus.Conflict);\r
+ }\r
+ else\r
+ {\r
+ //Remote files should be downloaded\r
+ yield return new CloudDownloadAction(accountInfo, objectInfo);\r
+ }\r
+ }\r
+ }\r
+\r
+ private static FileAgent GetFileAgent(AccountInfo accountInfo)\r
+ {\r
+ return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);\r
+ }\r
+\r
+ private void ProcessTrashedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> trashObjects)\r
+ {\r
+ var fileAgent = GetFileAgent(accountInfo);\r
+ foreach (var trashObject in trashObjects)\r
+ {\r
+ var barePath = trashObject.RelativeUrlToFilePath(accountInfo.UserName);\r
+ //HACK: Assume only the "pithos" container is used. Must find out what happens when\r
+ //deleting a file from a different container\r
+ var relativePath = Path.Combine("pithos", barePath);\r
+ fileAgent.Delete(relativePath);\r
+ }\r
+ }\r
+\r
+ private void UpdateStatus(PithosStatus status)\r
+ {\r
+ StatusKeeper.SetPithosStatus(status);\r
+ StatusNotification.Notify(new Notification());\r
+ }\r
+\r
+ private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)\r
+ {\r
+ var containerPaths = from container in containers\r
+ let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)\r
+ where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)\r
+ select containerPath;\r
+\r
+ foreach (var path in containerPaths)\r
+ {\r
+ Directory.CreateDirectory(path);\r
+ }\r
+ }\r
+\r
+ }\r
+}\r
projects / pithos-ms-client / commitdiff