using System.Reflection;\r
using System.Threading;\r
using System.Threading.Tasks;\r
-using Castle.ActiveRecord;\r
using Pithos.Interfaces;\r
using Pithos.Network;\r
using log4net;\r
using System.Collections.Generic;\r
using System.Linq;\r
\r
+ /*public class PollRequest\r
+ {\r
+ public DateTime? Since { get; set; }\r
+ public IEnumerable<string> Batch { get; set; }\r
+ }*/\r
+\r
+\r
/// <summary>\r
/// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all\r
/// objects and compares it with a previously cached version to detect differences. \r
\r
public IStatusNotification StatusNotification { get; set; }\r
\r
+ private CancellationTokenSource _currentOperationCancellation = new CancellationTokenSource();\r
+\r
+ public void CancelCurrentOperation()\r
+ {\r
+ //What does it mean to cancel the current upload/download?\r
+ //Obviously, the current operation will be cancelled by throwing\r
+ //a cancellation exception.\r
+ //\r
+ //The default behavior is to retry any operations that throw.\r
+ //Obviously this is not what we want in this situation.\r
+ //The cancelled operation should NOT bea retried. \r
+ //\r
+ //This can be done by catching the cancellation exception\r
+ //and avoiding the retry.\r
+ //\r
+\r
+ //Have to reset the cancellation source - it is not possible to reset the source\r
+ //Have to prevent a case where an operation requests a token from the old source\r
+ var oldSource = Interlocked.Exchange(ref _currentOperationCancellation, new CancellationTokenSource());\r
+ oldSource.Cancel();\r
+\r
+ }\r
+\r
public bool Pause\r
{\r
get {\r
}\r
}\r
\r
+ public CancellationToken CancellationToken\r
+ {\r
+ get { return _currentOperationCancellation.Token; }\r
+ }\r
+\r
private bool _firstPoll = true;\r
\r
//The Sync Event signals a manual synchronisation\r
private readonly ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>();\r
private readonly ConcurrentDictionary<Uri, AccountInfo> _accounts = new ConcurrentDictionary<Uri,AccountInfo>();\r
\r
+ //private readonly ActionBlock<PollRequest> _pollAction;\r
+\r
+ readonly HashSet<string> _knownContainers = new HashSet<string>();\r
+\r
+ \r
+ /// <summary>\r
+ /// Start a manual synchronization\r
+ /// </summary>\r
+ public void SynchNow(IEnumerable<string> paths=null)\r
+ {\r
+ _batchQueue.Enqueue(paths);\r
+ _syncEvent.SetAsync(); \r
+\r
+ //_pollAction.Post(new PollRequest {Batch = paths});\r
+ }\r
\r
/// <summary>\r
/// Start a manual synchronization\r
/// </summary>\r
- public void SynchNow()\r
- { \r
- _syncEvent.Set();\r
+ public Task SynchNowAsync(IEnumerable<string> paths=null)\r
+ {\r
+ _batchQueue.Enqueue(paths);\r
+ return _syncEvent.SetAsync(); \r
+\r
+ //_pollAction.Post(new PollRequest {Batch = paths});\r
+ }\r
+\r
+ readonly ConcurrentQueue<IEnumerable<string>> _batchQueue=new ConcurrentQueue<IEnumerable<string>>();\r
+\r
+ ConcurrentDictionary<string,MovedEventArgs> _moves=new ConcurrentDictionary<string, MovedEventArgs>(); \r
+\r
+ public void PostMove(MovedEventArgs args)\r
+ {\r
+ TaskEx.Run(() => _moves.AddOrUpdate(args.OldFullPath, args,(s,e)=>e)); \r
}\r
\r
+\r
+ private bool _hasConnection;\r
+\r
/// <summary>\r
/// Remote files are polled periodically. Any changes are processed\r
/// </summary>\r
/// <param name="since"></param>\r
/// <returns></returns>\r
- public async Task PollRemoteFiles(DateTime? since = null)\r
+ public async Task PollRemoteFiles(DateTimeOffset? since = null)\r
{\r
if (Log.IsDebugEnabled)\r
Log.DebugFormat("Polling changes after [{0}]",since);\r
\r
Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
- \r
\r
+ //GC.Collect();\r
+\r
+ \r
using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
{\r
//If this poll fails, we will retry with the same since value\r
- var nextSince = since;\r
+ DateTimeOffset? nextSince = since;\r
try\r
{\r
- await _unPauseEvent.WaitAsync();\r
+ _unPauseEvent.Wait();\r
UpdateStatus(PithosStatus.PollSyncing);\r
\r
- var tasks = from accountInfo in _accounts.Values\r
- select ProcessAccountFiles(accountInfo, since);\r
+ if (!NetworkAgent.IsConnectedToInternet)\r
+ {\r
+ if (_hasConnection)\r
+ {\r
+ StatusNotification.Notify(new Notification\r
+ {\r
+ Level = TraceLevel.Error,\r
+ Title = "Internet Connection problem",\r
+ Message ="Internet connectivity was lost. Synchronization will continue when connectivity is restored"\r
+ });\r
+ }\r
+ _hasConnection = false;\r
+ }\r
+ else\r
+ {\r
+ if (!_hasConnection)\r
+ {\r
+ StatusNotification.Notify(new Notification\r
+ {\r
+ Level = TraceLevel.Info,\r
+ Title = "Internet Connection",\r
+ Message = "Internet connectivity restored."\r
+ });\r
+ }\r
+ _hasConnection = true;\r
+\r
+ var accountBatches = new Dictionary<Uri, IEnumerable<string>>();\r
+ IEnumerable<string> batch = null;\r
+ if (_batchQueue.TryDequeue(out batch) && batch != null)\r
+ foreach (var account in _accounts.Values)\r
+ {\r
+ var accountBatch = batch.Where(path => path.IsAtOrBelow(account.AccountPath));\r
+ accountBatches[account.AccountKey] = accountBatch;\r
+ }\r
+\r
+ var moves = Interlocked.Exchange(ref _moves, new ConcurrentDictionary<string, MovedEventArgs>());\r
+\r
+ var tasks = new List<Task<DateTimeOffset?>>();\r
+ foreach (var accountInfo in _accounts.Values)\r
+ {\r
+ IEnumerable<string> accountBatch;\r
+ accountBatches.TryGetValue(accountInfo.AccountKey, out accountBatch);\r
+ var t = ProcessAccountFiles(accountInfo, accountBatch, moves, since);\r
+ tasks.Add(t);\r
+ }\r
\r
- var nextTimes=await TaskEx.WhenAll(tasks.ToList());\r
+ var taskList = tasks.ToList();\r
+ var nextTimes = await TaskEx.WhenAll(taskList).ConfigureAwait(false);\r
\r
- _firstPoll = false;\r
- //Reschedule the poll with the current timestamp as a "since" value\r
+ _firstPoll = false;\r
+ //Reschedule the poll with the current timestamp as a "since" value\r
\r
- if (nextTimes.Length>0)\r
- nextSince = nextTimes.Min();\r
- if (Log.IsDebugEnabled)\r
- Log.DebugFormat("Next Poll at [{0}]",nextSince);\r
+ if (nextTimes.Length > 0)\r
+ nextSince = nextTimes.Min();\r
+ if (Log.IsDebugEnabled)\r
+ Log.DebugFormat("Next Poll for changes since [{0}]", nextSince);\r
+ }\r
}\r
catch (Exception ex)\r
{\r
try\r
{\r
//Wait for the polling interval to pass or the Sync event to be signalled\r
- nextSince = await WaitForScheduledOrManualPoll(nextSince);\r
+ nextSince = await WaitForScheduledOrManualPoll(nextSince).ConfigureAwait(false);\r
}\r
finally\r
{\r
//Ensure polling is scheduled even in case of error\r
- TaskEx.Run(() => PollRemoteFiles(nextSince)); \r
+#pragma warning disable 4014\r
+ TaskEx.Run(()=>PollRemoteFiles(nextSince));\r
+#pragma warning restore 4014\r
+ //_pollAction.Post(new PollRequest {Since = nextSince});\r
}\r
}\r
}\r
/// </summary>\r
/// <param name="since"></param>\r
/// <returns></returns>\r
- private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
+ private async Task<DateTimeOffset?> WaitForScheduledOrManualPoll(DateTimeOffset? since)\r
{\r
var sync = _syncEvent.WaitAsync();\r
- var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval), NetworkAgent.CancellationToken);\r
- \r
- var signaledTask = await TaskEx.WhenAny(sync, wait);\r
+ var delay = TimeSpan.FromSeconds(Settings.PollingInterval);\r
+ if (Log.IsDebugEnabled)\r
+ Log.DebugFormat("Next Poll at [{0}]", DateTime.Now.Add(delay));\r
+ var wait = TaskEx.Delay(delay);\r
+\r
+ var signaledTask = await TaskEx.WhenAny(sync, wait).ConfigureAwait(false);\r
\r
//Pausing takes precedence over manual sync or awaiting\r
_unPauseEvent.Wait();\r
\r
//Wait for network processing to finish before polling\r
var pauseTask=NetworkAgent.ProceedEvent.WaitAsync();\r
- await TaskEx.WhenAll(signaledTask, pauseTask);\r
+ await TaskEx.WhenAll(signaledTask, pauseTask).ConfigureAwait(false);\r
\r
//If polling is signalled by SynchNow, ignore the since tag\r
if (sync.IsCompleted)\r
- {\r
- //TODO: Must convert to AutoReset\r
+ { \r
_syncEvent.Reset();\r
return null;\r
}\r
return since;\r
}\r
\r
- public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, DateTime? since = null)\r
+ \r
+\r
+ public async Task<DateTimeOffset?> ProcessAccountFiles(AccountInfo accountInfo, IEnumerable<string> accountBatch, ConcurrentDictionary<string, MovedEventArgs> moves, DateTimeOffset? since = null)\r
{\r
if (accountInfo == null)\r
throw new ArgumentNullException("accountInfo");\r
using (ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))\r
{\r
\r
- await NetworkAgent.GetDeleteAwaiter();\r
+ await NetworkAgent.GetDeleteAwaiter().ConfigureAwait(false);\r
\r
Log.Info("Scheduled");\r
var client = new CloudFilesClient(accountInfo);\r
\r
//We don't need to check the trash container\r
- var containers = client.ListContainers(accountInfo.UserName)\r
- .Where(c=>c.Name!="trash")\r
+ var allContainers=await client.ListContainers(accountInfo.UserName).ConfigureAwait(false);\r
+ var containers = allContainers\r
+ .Where(c=>c.Name.ToString()!="trash")\r
.ToList();\r
\r
\r
//The nextSince time fallback time is the same as the current.\r
//If polling succeeds, the next Since time will be the smallest of the maximum modification times\r
//of the shared and account objects\r
- var nextSince = since;\r
+ DateTimeOffset? nextSince = since;\r
\r
try\r
{\r
//Wait for any deletions to finish\r
- await NetworkAgent.GetDeleteAwaiter();\r
+ await NetworkAgent.GetDeleteAwaiter().ConfigureAwait(false);\r
//Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
//than delete a file that was created while we were executing the poll \r
\r
+ var token = _currentOperationCancellation.Token;\r
+\r
//Get the list of server objects changed since the last check\r
//The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
var listObjects = (from container in containers\r
select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
- client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
-\r
- var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
- client.ListSharedObjects(since), "shared");\r
+ client.ListObjects(accountInfo.UserName, container.Name, since), container.Name,token)).ToList();\r
+\r
+ var selectiveEnabled = Selectives.IsSelectiveEnabled(accountInfo.AccountKey);\r
+ var listShared = selectiveEnabled?\r
+ Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
+ client.ListSharedObjects(_knownContainers,since), "shared",token)\r
+ :Task.Factory.FromResult((IList<ObjectInfo>) new List<ObjectInfo>(),"shared");\r
+ \r
listObjects.Add(listShared);\r
- var listTasks = await Task.Factory.WhenAll(listObjects.ToArray());\r
+ var listTasks = await Task.Factory.WhenAll(listObjects.ToArray()).ConfigureAwait(false);\r
\r
using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
{\r
+\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
var dict = listTasks.ToDictionary(t => t.AsyncState);\r
\r
//Get all non-trash objects. Remember, the container name is stored in AsyncState\r
var remoteObjects = (from objectList in listTasks\r
- where (string)objectList.AsyncState != "trash"\r
+ where objectList.AsyncState.ToString() != "trash"\r
from obj in objectList.Result\r
+ orderby obj.Bytes ascending \r
select obj).ToList();\r
\r
//Get the latest remote object modification date, only if it is after\r
- //the original since date\r
+ //the original since date \r
nextSince = GetLatestDateAfter(nextSince, remoteObjects);\r
\r
var sharedObjects = dict["shared"].Result;\r
- nextSince = GetLatestDateBefore(nextSince, sharedObjects);\r
\r
//DON'T process trashed files\r
//If some files are deleted and added again to a folder, they will be deleted\r
where\r
!remoteObjects.Any(\r
info => info.Name == trash.Name && info.Hash == trash.Hash)\r
- select trash;\r
+ 8 select trash;\r
ProcessTrashedFiles(accountInfo, realTrash);\r
*/\r
\r
var cleanRemotes = (from info in remoteObjects.Union(sharedObjects)\r
- let name = info.Name??""\r
+ let name = info.Name.ToUnescapedString()??""\r
where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&\r
!name.StartsWith(FolderConstants.CacheFolder + "/",\r
StringComparison.InvariantCultureIgnoreCase)\r
select info).ToList();\r
\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
if (_firstPoll)\r
StatusKeeper.CleanupOrphanStates();\r
- StatusKeeper.CleanupStaleStates(accountInfo, cleanRemotes);\r
\r
var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
+ var currentRemotes = differencer.Current.ToList();\r
\r
- var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+ \r
+ StatusKeeper.CleanupStaleStates(accountInfo, currentRemotes);\r
\r
+ //var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
\r
- //On the first run\r
- if (_firstPoll)\r
+ //May have to wait if the FileAgent has asked for a Pause, due to local changes\r
+ await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
+\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
+ //Get the local files here \r
+ var agent = AgentLocator<FileAgent>.Get(accountInfo.AccountPath); \r
+ var files = LoadLocalFileTuples(accountInfo, accountBatch);\r
+\r
+\r
+ \r
+ //WARNING: GetFileSystemInfo may create state entries.\r
+ //TODO: Find a different way to create the tuples and block long filenames\r
+ var infos = (from remote in currentRemotes\r
+ let path = remote.RelativeUrlToFilePath(accountInfo.UserName)\r
+ let info=agent.GetFileSystemInfo(path)\r
+ where info != null\r
+ select Tuple.Create(info.FullName,remote))\r
+ .ToList();\r
+\r
+ var states = StatusKeeper.GetAllStates();\r
+\r
+ var tupleBuilder = new TupleBuilder(CancellationToken,StatusKeeper,StatusNotification,Settings);\r
+\r
+ var tuples = tupleBuilder.MergeSources(infos, files, states,moves).ToList();\r
+\r
+ var processedPaths = new HashSet<string>();\r
+ //Process only the changes in the batch file, if one exists\r
+ var stateTuples = accountBatch==null?tuples:tuples.Where(t => accountBatch.Contains(t.FilePath));\r
+ foreach (var tuple in stateTuples.Where(s=>!s.Locked))\r
{\r
- MarkSuspectedDeletes(accountInfo, cleanRemotes);\r
- }\r
- ProcessDeletedFiles(accountInfo, differencer.Deleted.FilterDirectlyBelow(filterUris));\r
+ await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
\r
- // @@@ NEED To add previous state here as well, To compare with previous hash\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
\r
- \r
+ //Set the Merkle Hash\r
+ //SetMerkleHash(accountInfo, tuple);\r
\r
- //Create a list of actions from the remote files\r
- \r
- var allActions = MovesToActions(accountInfo,differencer.Moved.FilterDirectlyBelow(filterUris))\r
- .Union(\r
- ChangesToActions(accountInfo, differencer.Changed.FilterDirectlyBelow(filterUris)))\r
- .Union(\r
- CreatesToActions(accountInfo, differencer.Created.FilterDirectlyBelow(filterUris)));\r
-\r
- //And remove those that are already being processed by the agent\r
- var distinctActions = allActions\r
- .Except(NetworkAgent.GetEnumerable(), new LocalFileComparer())\r
- .ToList();\r
-\r
- await _unPauseEvent.WaitAsync();\r
- //Queue all the actions\r
- foreach (var message in distinctActions)\r
+ await SyncSingleItem(accountInfo, tuple, agent, moves,processedPaths,token).ConfigureAwait(false);\r
+\r
+ }\r
+\r
+\r
+ //On the first run\r
+/*\r
+ if (_firstPoll)\r
{\r
- NetworkAgent.Post(message);\r
+ MarkSuspectedDeletes(accountInfo, cleanRemotes);\r
}\r
+*/\r
+\r
\r
Log.Info("[LISTENER] End Processing");\r
}\r
}\r
catch (Exception ex)\r
{\r
- Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);\r
+ Log.ErrorFormat("[FAIL] ListObjects for {0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);\r
return nextSince;\r
}\r
\r
return nextSince;\r
}\r
}\r
+/*\r
\r
- /// <summary>\r
- /// Returns the latest LastModified date from the list of objects, but only if it is before\r
- /// than the threshold value\r
- /// </summary>\r
- /// <param name="threshold"></param>\r
- /// <param name="cloudObjects"></param>\r
- /// <returns></returns>\r
- private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
+ private static void SetMerkleHash(AccountInfo accountInfo, StateTuple tuple)\r
{\r
- DateTime? maxDate = null;\r
- if (cloudObjects!=null && cloudObjects.Count > 0)\r
- maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
- if (maxDate == null || maxDate == DateTime.MinValue)\r
- return threshold;\r
- if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate)\r
- return maxDate;\r
- return threshold;\r
+ //The Merkle hash for directories is that of an empty buffer\r
+ if (tuple.FileInfo is DirectoryInfo)\r
+ tuple.C = MERKLE_EMPTY;\r
+ else if (tuple.FileState != null && tuple.MD5 == tuple.FileState.ETag)\r
+ {\r
+ //If there is a state whose MD5 matches, load the merkle hash from the file state\r
+ //insteaf of calculating it\r
+ tuple.C = tuple.FileState.Checksum; \r
+ }\r
+ else\r
+ {\r
+ tuple.Merkle = Signature.CalculateTreeHashAsync((FileInfo)tuple.FileInfo, accountInfo.BlockSize, accountInfo.BlockHash,1,progress);\r
+ //tuple.C=tuple.Merkle.TopHash.ToHashString(); \r
+ }\r
+ }\r
+*/\r
+\r
+ private IEnumerable<FileSystemInfo> LoadLocalFileTuples(AccountInfo accountInfo,IEnumerable<string> batch )\r
+ {\r
+ using (ThreadContext.Stacks["Account Files Hashing"].Push(accountInfo.UserName))\r
+ {\r
+ var batchPaths = (batch==null)?new List<string>():batch.ToList();\r
+ IEnumerable<FileSystemInfo> localInfos=AgentLocator<FileAgent>.Get(accountInfo.AccountPath)\r
+ .EnumerateFileSystemInfos();\r
+ if (batchPaths.Count>0)\r
+ localInfos= localInfos.Where(fi => batchPaths.Contains(fi.FullName));\r
+\r
+ return localInfos;\r
+ }\r
}\r
\r
/// <summary>\r
- /// Returns the latest LastModified date from the list of objects, but only if it is after\r
- /// the threshold value\r
+ /// Wait and Pause the agent while waiting\r
/// </summary>\r
- /// <param name="threshold"></param>\r
- /// <param name="cloudObjects"></param>\r
+ /// <param name="backoff"></param>\r
/// <returns></returns>\r
- private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
+ private async Task PauseFor(int backoff)\r
{\r
- DateTime? maxDate = null;\r
- if (cloudObjects!=null && cloudObjects.Count > 0)\r
- maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
- if (maxDate == null || maxDate == DateTime.MinValue)\r
- return threshold;\r
- if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate)\r
- return maxDate;\r
- return threshold;\r
- }\r
\r
- readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
- private Dictionary<Uri, List<Uri>> _selectiveUris = new Dictionary<Uri, List<Uri>>();\r
- private bool _pause;\r
+ Pause = true;\r
+ await TaskEx.Delay(backoff).ConfigureAwait(false);\r
+ Pause = false;\r
+ }\r
\r
- /// <summary>\r
- /// Deletes local files that are not found in the list of cloud files\r
- /// </summary>\r
- /// <param name="accountInfo"></param>\r
- /// <param name="cloudFiles"></param>\r
- private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles)\r
+ private async Task SyncSingleItem(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves,HashSet<string> processedPaths, CancellationToken token)\r
{\r
- if (accountInfo == null)\r
- throw new ArgumentNullException("accountInfo");\r
- if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
- throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
- if (cloudFiles == null)\r
- throw new ArgumentNullException("cloudFiles");\r
- Contract.EndContractBlock();\r
+ Log.DebugFormat("Sync [{0}] C:[{1}] L:[{2}] S:[{3}]", tuple.FilePath, tuple.C, tuple.L, tuple.S);\r
\r
- var deletedFiles = new List<FileSystemInfo>();\r
- foreach (var objectInfo in cloudFiles)\r
+ //If the processed paths already contain the current path, exit\r
+ if (!processedPaths.Add(tuple.FilePath))\r
+ return;\r
+\r
+ try\r
{\r
- if (Log.IsDebugEnabled)\r
- Log.DebugFormat("Handle deleted [{0}]", objectInfo.Uri);\r
- var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
- var item = FileAgent.GetFileAgent(accountInfo).GetFileSystemInfo(relativePath);\r
- if (Log.IsDebugEnabled)\r
- Log.DebugFormat("Will delete [{0}] for [{1}]", item.FullName, objectInfo.Uri);\r
- if (item.Exists)\r
+ bool isInferredParent = tuple.ObjectInfo != null && tuple.ObjectInfo.UUID.StartsWith("00000000-0000-0000");\r
+\r
+ var localFilePath = tuple.FilePath;\r
+ //Don't use the tuple info, it may have been deleted\r
+ var localInfo = FileInfoExtensions.FromPath(localFilePath);\r
+\r
+\r
+ var isUnselectedRootFolder = agent.IsUnselectedRootFolder(tuple.FilePath);\r
+\r
+ //Unselected root folders that have not yet been uploaded should be uploaded and added to the \r
+ //selective folders\r
+\r
+ if (!Selectives.IsSelected(accountInfo, localFilePath) &&\r
+ !(isUnselectedRootFolder && tuple.ObjectInfo == null))\r
+ return;\r
+\r
+ // Local file unchanged? If both C and L are null, make sure it's because \r
+ //both the file is missing and the state checksum is not missing\r
+ if (tuple.C == tuple.L /*&& (localInfo.Exists || tuple.FileState == null)*/)\r
{\r
- if ((item.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)\r
+ //No local changes\r
+ //Server unchanged?\r
+ if (tuple.S == tuple.L)\r
{\r
- item.Attributes = item.Attributes & ~FileAttributes.ReadOnly;\r
-\r
+ // No server changes\r
+ //Has the file been renamed locally?\r
+ if (!await MoveForLocalMove(accountInfo,tuple))\r
+ //Has the file been renamed on the server?\r
+ MoveForServerMove(accountInfo, tuple);\r
}\r
+ else\r
+ {\r
+ //Different from server\r
+ //Does the server file exist?\r
+ if (tuple.S == null)\r
+ {\r
+ //Server file doesn't exist\r
+ //deleteObjectFromLocal()\r
+ using (\r
+ StatusNotification.GetNotifier("Deleting local {0}", "Deleted local {0}",true,\r
+ localInfo.Name))\r
+ {\r
+ DeleteLocalFile(agent, localFilePath);\r
+ }\r
+ }\r
+ else\r
+ {\r
+ //Server file exists\r
+ //downloadServerObject() // Result: L = S\r
+ //If the file has moved on the server, move it locally before downloading\r
+ using (\r
+ StatusNotification.GetNotifier("Downloading {0}", "Downloaded {0}",true,\r
+ localInfo.Name))\r
+ {\r
+ var targetPath = MoveForServerMove(accountInfo, tuple);\r
+ if (targetPath != null)\r
+ {\r
+\r
+ await DownloadCloudFile(accountInfo, tuple, token, targetPath).ConfigureAwait(false);\r
+\r
+ AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
+ }\r
+ }\r
+ }\r
+ }\r
+ }\r
+ else\r
+ { \r
\r
+ //Local changes found\r
\r
- Log.DebugFormat("Deleting {0}", item.FullName);\r
-\r
- var directory = item as DirectoryInfo;\r
- if (directory != null)\r
- directory.Delete(true);\r
+ //Server unchanged?\r
+ if (tuple.S == tuple.L)\r
+ {\r
+ //The FileAgent selective sync checks for new root folder files\r
+ if (!agent.Ignore(localFilePath))\r
+ {\r
+ if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
+ {\r
+ //deleteObjectFromServer()\r
+ DeleteCloudFile(accountInfo, tuple);\r
+ //updateRecord( Remove L, S) \r
+ }\r
+ else\r
+ {\r
+ //uploadLocalObject() // Result: S = C, L = S \r
+ var progress = new Progress<HashProgress>(d =>\r
+ StatusNotification.Notify(new StatusNotification(String.Format("Merkle Hashing for Upload {0:p} of {1}", d.Percentage, localInfo.Name))));\r
+\r
+ //Is it an unselected root folder\r
+ var isCreation = isUnselectedRootFolder ||//or a new folder under a selected parent?\r
+ (localInfo is DirectoryInfo && Selectives.IsSelected(accountInfo, localInfo) && tuple.FileState == null && tuple.ObjectInfo == null);\r
+\r
+\r
+ //Is this a result of a FILE move with no modifications? Then try to move it,\r
+ //to avoid an expensive hash\r
+ if (!await MoveForLocalMove(accountInfo, tuple))\r
+ {\r
+ await UploadLocalFile(accountInfo, tuple, token, isCreation, localInfo,processedPaths, progress).ConfigureAwait(false);\r
+ }\r
+\r
+ //updateRecord( S = C )\r
+ //State updated by the uploader\r
+ \r
+ if (isCreation )\r
+ { \r
+ ProcessChildren(accountInfo, tuple, agent, moves,processedPaths,token);\r
+ }\r
+ }\r
+ }\r
+ }\r
else\r
- item.Delete();\r
- Log.DebugFormat("Deleted [{0}] for [{1}]", item.FullName, objectInfo.Uri);\r
- DateTime lastDate;\r
- _lastSeen.TryRemove(item.FullName, out lastDate);\r
- deletedFiles.Add(item);\r
+ {\r
+ if (tuple.C == tuple.S)\r
+ {\r
+ // (Identical Changes) Result: L = S\r
+ //doNothing()\r
+ \r
+ //Don't update anything for nonexistend server files\r
+ if (tuple.S != null)\r
+ {\r
+ //Detect server moves\r
+ var targetPath = MoveForServerMove(accountInfo, tuple);\r
+ if (targetPath != null)\r
+ {\r
+ Debug.Assert(tuple.Merkle != null);\r
+ StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo, tuple.Merkle);\r
+\r
+ AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
+ }\r
+ }\r
+ else\r
+ {\r
+ //At this point, C==S==NULL and we have a stale state (L)\r
+ //Log the stale tuple for investigation\r
+ Log.WarnFormat("Stale tuple detected FilePathPath:[{0}], State:[{1}], LocalFile:[{2}]", tuple.FilePath, tuple.FileState, tuple.FileInfo);\r
+\r
+ //And remove it\r
+ if (!String.IsNullOrWhiteSpace(tuple.FilePath))\r
+ StatusKeeper.ClearFileStatus(tuple.FilePath);\r
+ }\r
+ }\r
+ else\r
+ {\r
+ if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
+ {\r
+ //deleteObjectFromServer()\r
+ DeleteCloudFile(accountInfo, tuple);\r
+ //updateRecord(Remove L, S) \r
+ }\r
+ //If both the local and server files are missing, the state is stale\r
+ else if (!localInfo.Exists && (tuple.S == null || tuple.ObjectInfo == null))\r
+ {\r
+ StatusKeeper.ClearFileStatus(localInfo.FullName);\r
+ }\r
+ else\r
+ {\r
+ ReportConflictForMismatch(localFilePath);\r
+ //identifyAsConflict() // Manual action required\r
+ }\r
+ }\r
+ }\r
}\r
- StatusKeeper.SetFileState(item.FullName, FileStatus.Deleted, FileOverlayStatus.Deleted, "File Deleted");\r
}\r
- Log.InfoFormat("[{0}] files were deleted", deletedFiles.Count);\r
- StatusNotification.NotifyForFiles(deletedFiles, String.Format("{0} files were deleted", deletedFiles.Count),\r
- TraceLevel.Info);\r
-\r
+ catch (Exception exc)\r
+ {\r
+ //In case of error log and retry with the next poll\r
+ Log.ErrorFormat("[SYNC] Failed for file {0}. Will Retry.\r\n{1}",tuple.FilePath,exc);\r
+ }\r
}\r
\r
- private void MarkSuspectedDeletes(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles)\r
+ private void DeleteLocalFile(FileAgent agent, string localFilePath)\r
{\r
-//Only consider files that are not being modified, ie they are in the Unchanged state \r
- var deleteCandidates = FileState.Queryable.Where(state =>\r
- state.FilePath.StartsWith(accountInfo.AccountPath)\r
- && state.FileStatus == FileStatus.Unchanged).ToList();\r
+ StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
+ FileOverlayStatus.Deleted, "");\r
+ using (NetworkGate.Acquire(localFilePath, NetworkOperation.Deleting))\r
+ {\r
+ agent.Delete(localFilePath);\r
+ }\r
+ //updateRecord(Remove C, L)\r
+ StatusKeeper.ClearFileStatus(localFilePath);\r
+ }\r
\r
+ private async Task DownloadCloudFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token, string targetPath)\r
+ { \r
+ //Don't create a new state for non-existent files\r
+ if (File.Exists(targetPath) || Directory.Exists(targetPath))\r
+ StatusKeeper.SetFileState(targetPath, FileStatus.Modified, FileOverlayStatus.Modified,"");\r
+\r
+ var finalHash=await\r
+ NetworkAgent.Downloader.DownloadCloudFile(accountInfo, tuple.ObjectInfo, targetPath,\r
+ token)\r
+ .ConfigureAwait(false);\r
+ //updateRecord( L = S )\r
+ StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
+ finalHash);\r
+\r
+ StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo,finalHash);\r
+ }\r
\r
- //TODO: filesToDelete must take into account the Others container \r
- var filesToDelete = (from deleteCandidate in deleteCandidates\r
- let localFile = FileInfoExtensions.FromPath(deleteCandidate.FilePath)\r
- let relativeFilePath = localFile.AsRelativeTo(accountInfo.AccountPath)\r
- where\r
- !cloudFiles.Any(r => r.RelativeUrlToFilePath(accountInfo.UserName) == relativeFilePath)\r
- select localFile).ToList();\r
+ private async Task UploadLocalFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token,\r
+ bool isUnselectedRootFolder, FileSystemInfo localInfo, HashSet<string> processedPaths, IProgress<HashProgress> progress)\r
+ {\r
+ var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
+ accountInfo.BlockSize, accountInfo.BlockHash,\r
+ "Poll", isUnselectedRootFolder, token, progress,tuple.Merkle); \r
\r
+ using (StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}",true,\r
+ localInfo.Name))\r
+ {\r
+ await NetworkAgent.Uploader.UploadCloudFile(action, token).ConfigureAwait(false);\r
+ }\r
\r
- //Set the status of missing files to Conflict\r
- foreach (var item in filesToDelete)\r
+ if (isUnselectedRootFolder)\r
{\r
- //Try to acquire a gate on the file, to take into account files that have been dequeued\r
- //and are being processed\r
- using (var gate = NetworkGate.Acquire(item.FullName, NetworkOperation.Deleting))\r
+ var dirActions =(\r
+ from dir in ((DirectoryInfo) localInfo).EnumerateDirectories("*", SearchOption.AllDirectories)\r
+ let subAction = new CloudUploadAction(accountInfo, dir, null,\r
+ accountInfo.BlockSize, accountInfo.BlockHash,\r
+ "Poll", true, token, progress)\r
+ select subAction).ToList();\r
+ foreach (var dirAction in dirActions)\r
{\r
- if (gate.Failed)\r
- continue;\r
- StatusKeeper.SetFileState(item.FullName, FileStatus.Conflict, FileOverlayStatus.Deleted,\r
- "Local file missing from server");\r
+ processedPaths.Add(dirAction.LocalFile.FullName);\r
}\r
+ \r
+ await TaskEx.WhenAll(dirActions.Select(a=>NetworkAgent.Uploader.UploadCloudFile(a,token)).ToArray());\r
}\r
- UpdateStatus(PithosStatus.HasConflicts);\r
- StatusNotification.NotifyConflicts(filesToDelete,\r
- String.Format(\r
- "{0} local files are missing from Pithos, possibly because they were deleted",\r
- filesToDelete.Count));\r
- StatusNotification.NotifyForFiles(filesToDelete, String.Format("{0} files were deleted", filesToDelete.Count),\r
- TraceLevel.Info);\r
}\r
\r
- /// <summary>\r
- /// Creates a Sync action for each changed server file\r
- /// </summary>\r
- /// <param name="accountInfo"></param>\r
- /// <param name="changes"></param>\r
- /// <returns></returns>\r
- private IEnumerable<CloudAction> ChangesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> changes)\r
+ private async Task<bool> MoveForLocalMove(AccountInfo accountInfo, StateTuple tuple)\r
{\r
- if (changes == null)\r
- throw new ArgumentNullException();\r
- Contract.EndContractBlock();\r
- var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
+ //Is the previous path missing?\r
+ if (String.IsNullOrWhiteSpace(tuple.OldFullPath))\r
+ return false;\r
+ //Has the file locally, in which case it should be uploaded rather than moved?\r
+ if (tuple.OldChecksum != tuple.Merkle.TopHash.ToHashString())\r
+ return false;\r
+\r
+ var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
+ var serverPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
+ //Has the file been renamed on the server?\r
+ if (!tuple.OldFullPath.Equals(serverPath))\r
+ {\r
+ ReportConflictForDoubleRename(tuple.FilePath);\r
+ return false;\r
+ }\r
\r
- //In order to avoid multiple iterations over the files, we iterate only once\r
- //over the remote files\r
- foreach (var objectInfo in changes)\r
+ try\r
{\r
- var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
- //If a directory object already exists, we may need to sync it\r
- if (fileAgent.Exists(relativePath))\r
- {\r
- var localFile = fileAgent.GetFileSystemInfo(relativePath);\r
- //We don't need to sync directories\r
- if (objectInfo.IsDirectory && localFile is DirectoryInfo)\r
- continue;\r
- using (new SessionScope(FlushAction.Never))\r
- {\r
- var state = StatusKeeper.GetStateByFilePath(localFile.FullName);\r
- _lastSeen[localFile.FullName] = DateTime.Now;\r
- //Common files should be checked on a per-case basis to detect differences, which is newer\r
\r
- yield return new CloudAction(accountInfo, CloudActionType.MustSynch,\r
- localFile, objectInfo, state, accountInfo.BlockSize,\r
- accountInfo.BlockHash,"Poll Changes");\r
- }\r
+ var client = new CloudFilesClient(accountInfo);\r
+ var objectInfo = CloudAction.CreateObjectInfoFor(accountInfo, tuple.FileInfo);\r
+ objectInfo.X_Object_Hash = tuple.Merkle.TopHash.ToHashString();\r
+ var containerPath = Path.Combine(accountInfo.AccountPath, objectInfo.Container.ToUnescapedString());\r
+ //TODO: SImplify these multiple conversions from and to Uris\r
+ var oldName = tuple.OldFullPath.AsRelativeTo(containerPath);\r
+ //Then execute a move instead of an upload\r
+ using (StatusNotification.GetNotifier("Moving {0}", "Moved {0}", true,tuple.FileInfo.Name))\r
+ {\r
+ await client.MoveObject(objectInfo.Account, objectInfo.Container, oldName.Replace('\\','/').ToEscapedUri(),\r
+ objectInfo.Container, objectInfo.Name).ConfigureAwait(false);\r
+ StatusKeeper.MoveFileState(tuple.OldFullPath, tuple.FilePath, objectInfo, tuple.Merkle);\r
+ //StatusKeeper.StoreInfo(tuple.FilePath,objectInfo,tuple.Merkle);\r
+ //StatusKeeper.ClearFolderStatus(tuple.FilePath);\r
}\r
- else\r
+ return true;\r
+ }\r
+ catch (Exception exc)\r
+ {\r
+ Log.ErrorFormat("[MOVE] Failed for [{0}],:\r\n{1}", tuple.FilePath, exc);\r
+ //Return false to force an upload of the file\r
+ return false;\r
+ }\r
+\r
+ }\r
+\r
+ private void AddOwnFolderToSelectives(AccountInfo accountInfo, StateTuple tuple, string targetPath)\r
+ {\r
+ //Not for shared folders\r
+ if (tuple.ObjectInfo.IsShared==true)\r
+ return;\r
+ //Also ensure that any newly created folders are added to the selectives, if the original folder was selected\r
+ var containerPath = Path.Combine(accountInfo.AccountPath, tuple.ObjectInfo.Container.ToUnescapedString());\r
+\r
+ //If this is a root folder encountered for the first time\r
+ if (tuple.L == null && Directory.Exists(tuple.FileInfo.FullName) \r
+ && (tuple.FileInfo.FullName.IsAtOrBelow(containerPath)))\r
+ {\r
+ \r
+ var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
+ var initialPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
+\r
+ //var hasMoved = true;// !initialPath.Equals(targetPath);\r
+ //If the new path is under a selected folder, add it to the selectives as well\r
+ if (Selectives.IsSelected(accountInfo, initialPath))\r
{\r
- //Remote files should be downloaded\r
- yield return new CloudDownloadAction(accountInfo, objectInfo,"Poll Changes");\r
+ Selectives.AddUri(accountInfo, tuple.ObjectInfo.Uri);\r
+ Selectives.Save(accountInfo);\r
}\r
}\r
}\r
\r
- /// <summary>\r
- /// Creates a Local Move action for each moved server file\r
- /// </summary>\r
- /// <param name="accountInfo"></param>\r
- /// <param name="moves"></param>\r
- /// <returns></returns>\r
- private IEnumerable<CloudAction> MovesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> moves)\r
+ private string MoveForServerMove(AccountInfo accountInfo, StateTuple tuple)\r
{\r
- if (moves == null)\r
- throw new ArgumentNullException();\r
- Contract.EndContractBlock();\r
- var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
+ if (tuple.ObjectInfo == null)\r
+ return null;\r
+ var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
+ var serverPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
+ \r
+ //Compare Case Insensitive\r
+ if (String.Equals(tuple.FilePath ,serverPath,StringComparison.InvariantCultureIgnoreCase)) \r
+ return serverPath;\r
+\r
+ //Has the file been renamed locally?\r
+ if (!String.IsNullOrWhiteSpace(tuple.OldFullPath) && !tuple.OldFullPath.Equals(tuple.FilePath))\r
+ {\r
+ ReportConflictForDoubleRename(tuple.FilePath);\r
+ return null;\r
+ }\r
\r
- //In order to avoid multiple iterations over the files, we iterate only once\r
- //over the remote files\r
- foreach (var objectInfo in moves)\r
+ tuple.FileInfo.Refresh();\r
+ //The file/folder may not exist if it was moved because its parent moved\r
+ if (!tuple.FileInfo.Exists)\r
{\r
- var previousRelativepath = objectInfo.Previous.RelativeUrlToFilePath(accountInfo.UserName);\r
- //If the previous file already exists, we can execute a Move operation\r
- if (fileAgent.Exists(previousRelativepath))\r
+ var target=FileInfoExtensions.FromPath(serverPath);\r
+ if (!target.Exists)\r
{\r
- var previousFile = fileAgent.GetFileSystemInfo(previousRelativepath);\r
- using (new SessionScope(FlushAction.Never))\r
- {\r
- var state = StatusKeeper.GetStateByFilePath(previousFile.FullName);\r
- _lastSeen[previousFile.FullName] = DateTime.Now;\r
-\r
- //For each moved object we need to move both the local file and update \r
- yield return new CloudAction(accountInfo, CloudActionType.RenameLocal,\r
- previousFile, objectInfo, state, accountInfo.BlockSize,\r
- accountInfo.BlockHash,"Poll Moves");\r
- //For modified files, we need to download the changes as well\r
- if (objectInfo.Hash!=objectInfo.PreviousHash)\r
- yield return new CloudDownloadAction(accountInfo,objectInfo, "Poll Moves");\r
- }\r
+ Log.ErrorFormat("No source or target found while trying to move {0} to {1}", tuple.FileInfo.FullName, serverPath);\r
}\r
- //If the previous file does not exist, we need to download it in the new location\r
- else\r
+ return serverPath;\r
+ }\r
+\r
+ using (StatusNotification.GetNotifier("Moving local {0}", "Moved local {0}", true,Path.GetFileName(tuple.FilePath)))\r
+ using(NetworkGate.Acquire(tuple.FilePath,NetworkOperation.Renaming))\r
+ {\r
+ \r
+ var fi = tuple.FileInfo as FileInfo;\r
+ if (fi != null)\r
{\r
- //Remote files should be downloaded\r
- yield return new CloudDownloadAction(accountInfo, objectInfo, "Poll Moves");\r
+ var targetFile = new FileInfo(serverPath);\r
+ if (!targetFile.Directory.Exists)\r
+ targetFile.Directory.Create();\r
+ fi.MoveTo(serverPath);\r
}\r
+ var di = tuple.FileInfo as DirectoryInfo;\r
+ if (di != null)\r
+ {\r
+ var targetDir = new DirectoryInfo(serverPath);\r
+ if (!targetDir.Parent.Exists)\r
+ targetDir.Parent.Create();\r
+ di.MoveTo(serverPath);\r
+ }\r
+ }\r
+\r
+ StatusKeeper.StoreInfo(serverPath, tuple.ObjectInfo);\r
+\r
+ return serverPath;\r
+ }\r
+\r
+ private void DeleteCloudFile(AccountInfo accountInfo, StateTuple tuple)\r
+ {\r
+ using (StatusNotification.GetNotifier("Deleting server {0}", "Deleted server {0}", true,Path.GetFileName(tuple.FilePath)))\r
+ {\r
+\r
+ StatusKeeper.SetFileState(tuple.FilePath, FileStatus.Deleted,\r
+ FileOverlayStatus.Deleted, "");\r
+ NetworkAgent.DeleteAgent.DeleteCloudFile(accountInfo, tuple.ObjectInfo);\r
+ StatusKeeper.ClearFileStatus(tuple.FilePath);\r
}\r
}\r
\r
+ private void ProcessChildren(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves,HashSet<string> processedPaths,CancellationToken token)\r
+ {\r
+\r
+ var dirInfo = tuple.FileInfo as DirectoryInfo;\r
+ var folderTuples = from folder in dirInfo.EnumerateDirectories("*", SearchOption.AllDirectories)\r
+ select new StateTuple(folder){C=Signature.MERKLE_EMPTY};\r
+ \r
+ var fileTuples = from file in dirInfo.EnumerateFiles("*", SearchOption.AllDirectories)\r
+ let state=StatusKeeper.GetStateByFilePath(file.FullName)\r
+ select new StateTuple(file){\r
+ Merkle=StatusAgent.CalculateTreeHash(file,accountInfo,state,\r
+ Settings.HashingParallelism,token,null)\r
+ };\r
+ \r
+ //Process folders first, to ensure folders appear on the sever as soon as possible\r
+ folderTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, moves, processedPaths,token).Wait());\r
+ \r
+ fileTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, moves,processedPaths, token).Wait());\r
+ }\r
+\r
+\r
+\r
+\r
+ \r
+\r
\r
/// <summary>\r
- /// Creates a download action for each new server file\r
+ /// Returns the latest LastModified date from the list of objects, but only if it is before\r
+ /// than the threshold value\r
/// </summary>\r
- /// <param name="accountInfo"></param>\r
- /// <param name="creates"></param>\r
+ /// <param name="threshold"></param>\r
+ /// <param name="cloudObjects"></param>\r
/// <returns></returns>\r
- private IEnumerable<CloudAction> CreatesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> creates)\r
+ private static DateTimeOffset? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
{\r
- if (creates == null)\r
- throw new ArgumentNullException();\r
- Contract.EndContractBlock();\r
- var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
+ DateTimeOffset? maxDate = null;\r
+ if (cloudObjects!=null && cloudObjects.Count > 0)\r
+ maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
+ if (!maxDate.HasValue)\r
+ return threshold;\r
+ if (!threshold.HasValue|| threshold > maxDate)\r
+ return maxDate;\r
+ return threshold;\r
+ }\r
\r
- //In order to avoid multiple iterations over the files, we iterate only once\r
- //over the remote files\r
- foreach (var objectInfo in creates)\r
- {\r
- if (Log.IsDebugEnabled)\r
- Log.DebugFormat("[NEW INFO] {0}",objectInfo.Uri);\r
+ /// <summary>\r
+ /// Returns the latest LastModified date from the list of objects, but only if it is after\r
+ /// the threshold value\r
+ /// </summary>\r
+ /// <param name="threshold"></param>\r
+ /// <param name="cloudObjects"></param>\r
+ /// <returns></returns>\r
+ private static DateTimeOffset? GetLatestDateAfter(DateTimeOffset? threshold, IList<ObjectInfo> cloudObjects)\r
+ {\r
+ DateTimeOffset? maxDate = null;\r
+ if (cloudObjects!=null && cloudObjects.Count > 0)\r
+ maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
+ if (!maxDate.HasValue)\r
+ return threshold;\r
+ if (!threshold.HasValue|| threshold < maxDate)\r
+ return maxDate;\r
+ return threshold;\r
+ }\r
\r
- var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
+ readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
+ private bool _pause;\r
+ \r
\r
- //If the object already exists, we should check before uploading or downloading\r
- if (fileAgent.Exists(relativePath))\r
- {\r
- var localFile= fileAgent.GetFileSystemInfo(relativePath);\r
- var state = StatusKeeper.GetStateByFilePath(localFile.WithProperCapitalization().FullName);\r
- yield return new CloudAction(accountInfo, CloudActionType.MustSynch,\r
- localFile, objectInfo, state, accountInfo.BlockSize,\r
- accountInfo.BlockHash,"Poll Creates"); \r
- }\r
- else\r
- {\r
- //Remote files should be downloaded\r
- yield return new CloudDownloadAction(accountInfo, objectInfo,"Poll Creates");\r
- }\r
\r
- }\r
+\r
+ private void ReportConflictForMismatch(string localFilePath)\r
+ {\r
+ if (String.IsNullOrWhiteSpace(localFilePath))\r
+ throw new ArgumentNullException("localFilePath");\r
+ Contract.EndContractBlock();\r
+\r
+ StatusKeeper.SetFileState(localFilePath, FileStatus.Conflict, FileOverlayStatus.Conflict, "File changed at the server");\r
+ UpdateStatus(PithosStatus.HasConflicts);\r
+ var message = String.Format("Conflict detected for file {0}", localFilePath);\r
+ Log.Warn(message);\r
+ StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
}\r
\r
+ private void ReportConflictForDoubleRename(string localFilePath)\r
+ {\r
+ if (String.IsNullOrWhiteSpace(localFilePath))\r
+ throw new ArgumentNullException("localFilePath");\r
+ Contract.EndContractBlock();\r
+\r
+ StatusKeeper.SetFileState(localFilePath, FileStatus.Conflict, FileOverlayStatus.Conflict, "File renamed both locally and on the server");\r
+ UpdateStatus(PithosStatus.HasConflicts);\r
+ var message = String.Format("Double rename conflict detected for file {0}", localFilePath);\r
+ Log.Warn(message);\r
+ StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
+ }\r
+\r
+\r
/// <summary>\r
/// Notify the UI to update the visual status\r
/// </summary>\r
private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)\r
{\r
var containerPaths = from container in containers\r
- let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)\r
- where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)\r
+ let containerPath = Path.Combine(accountInfo.AccountPath, container.Name.ToUnescapedString())\r
+ where container.Name.ToString() != FolderConstants.TrashContainer && !Directory.Exists(containerPath)\r
select containerPath;\r
\r
foreach (var path in containerPaths)\r
\r
public void RemoveAccount(AccountInfo accountInfo)\r
{\r
+ if (accountInfo == null)\r
+ return;\r
+\r
AccountInfo account;\r
_accounts.TryRemove(accountInfo.AccountKey, out account);\r
+\r
SnapshotDifferencer differencer;\r
_differencer.Differencers.TryRemove(accountInfo.AccountKey, out differencer);\r
}\r
-\r
- public void SetSelectivePaths(AccountInfo accountInfo,Uri[] added, Uri[] removed)\r
- {\r
- AbortRemovedPaths(accountInfo,removed);\r
- DownloadNewPaths(accountInfo,added);\r
- }\r
-\r
- private void DownloadNewPaths(AccountInfo accountInfo, Uri[] added)\r
- {\r
- var client = new CloudFilesClient(accountInfo);\r
- foreach (var folderUri in added)\r
- {\r
- try\r
- {\r
-\r
- string account;\r
- string container;\r
- var segmentsCount = folderUri.Segments.Length;\r
- //Is this an account URL?\r
- if (segmentsCount < 3)\r
- continue;\r
- //Is this a container or folder URL?\r
- if (segmentsCount == 3)\r
- {\r
- account = folderUri.Segments[1].TrimEnd('/');\r
- container = folderUri.Segments[2].TrimEnd('/');\r
- }\r
- else\r
- {\r
- account = folderUri.Segments[2].TrimEnd('/');\r
- container = folderUri.Segments[3].TrimEnd('/');\r
- }\r
- IList<ObjectInfo> items;\r
- if (segmentsCount > 3)\r
- {\r
- //List folder\r
- var folder = String.Join("", folderUri.Segments.Splice(4));\r
- items = client.ListObjects(account, container, folder);\r
- }\r
- else\r
- {\r
- //List container\r
- items = client.ListObjects(account, container);\r
- }\r
- var actions = CreatesToActions(accountInfo, items);\r
- foreach (var action in actions)\r
- {\r
- NetworkAgent.Post(action);\r
- }\r
- }\r
- catch (Exception exc)\r
- {\r
- Log.WarnFormat("Listing of new selective path [{0}] failed with \r\n{1}", folderUri, exc);\r
- }\r
- }\r
-\r
- //Need to get a listing of each of the URLs, then post them to the NetworkAgent\r
- //CreatesToActions(accountInfo,)\r
-\r
-/* NetworkAgent.Post();*/\r
- }\r
-\r
- private void AbortRemovedPaths(AccountInfo accountInfo, Uri[] removed)\r
- {\r
- /*this.NetworkAgent.*/\r
- }\r
+ \r
}\r
}\r
projects / pithos-ms-client / blobdiff