using System.Diagnostics;\r
using System.Diagnostics.Contracts;\r
using System.IO;\r
-using System.Linq.Expressions;\r
using System.Reflection;\r
-using System.Security.Cryptography;\r
using System.Threading;\r
using System.Threading.Tasks;\r
-using System.Threading.Tasks.Dataflow;\r
-using Castle.ActiveRecord;\r
using Pithos.Interfaces;\r
using Pithos.Network;\r
using log4net;\r
public void SynchNow(IEnumerable<string> paths=null)\r
{\r
_batchQueue.Enqueue(paths);\r
- _syncEvent.Set(); \r
+ _syncEvent.SetAsync(); \r
+\r
+ //_pollAction.Post(new PollRequest {Batch = paths});\r
+ }\r
+\r
+ /// <summary>\r
+ /// Start a manual synchronization\r
+ /// </summary>\r
+ public Task SynchNowAsync(IEnumerable<string> paths=null)\r
+ {\r
+ _batchQueue.Enqueue(paths);\r
+ return _syncEvent.SetAsync(); \r
\r
//_pollAction.Post(new PollRequest {Batch = paths});\r
}\r
TaskEx.Run(() => _moves.AddOrUpdate(args.OldFullPath, args,(s,e)=>e)); \r
}\r
\r
+\r
+ private bool _hasConnection;\r
+\r
/// <summary>\r
/// Remote files are polled periodically. Any changes are processed\r
/// </summary>\r
/// <param name="since"></param>\r
/// <returns></returns>\r
- public void PollRemoteFiles(DateTime? since = null)\r
+ public async Task PollRemoteFiles(DateTimeOffset? since = null)\r
{\r
if (Log.IsDebugEnabled)\r
Log.DebugFormat("Polling changes after [{0}]",since);\r
\r
//GC.Collect();\r
\r
+ \r
using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
{\r
//If this poll fails, we will retry with the same since value\r
- var nextSince = since;\r
+ DateTimeOffset? nextSince = since;\r
try\r
{\r
_unPauseEvent.Wait();\r
UpdateStatus(PithosStatus.PollSyncing);\r
\r
- var accountBatches=new Dictionary<Uri, IEnumerable<string>>();\r
- IEnumerable<string> batch = null;\r
- if (_batchQueue.TryDequeue(out batch) && batch != null)\r
- foreach (var account in _accounts.Values)\r
+ if (!NetworkAgent.IsConnectedToInternet)\r
+ {\r
+ if (_hasConnection)\r
{\r
- var accountBatch = batch.Where(path => path.IsAtOrBelow(account.AccountPath));\r
- accountBatches[account.AccountKey] = accountBatch;\r
+ StatusNotification.Notify(new Notification\r
+ {\r
+ Level = TraceLevel.Error,\r
+ Title = "Internet Connection problem",\r
+ Message ="Internet connectivity was lost. Synchronization will continue when connectivity is restored"\r
+ });\r
}\r
- \r
- var moves=Interlocked.Exchange(ref _moves, new ConcurrentDictionary<string, MovedEventArgs>());\r
-\r
- var tasks = new List<Task<DateTime?>>();\r
- foreach(var accountInfo in _accounts.Values)\r
- {\r
- IEnumerable<string> accountBatch ;\r
- accountBatches.TryGetValue(accountInfo.AccountKey,out accountBatch);\r
- var t=ProcessAccountFiles (accountInfo, accountBatch, moves,since);\r
- tasks.Add(t);\r
+ _hasConnection = false;\r
}\r
+ else\r
+ {\r
+ if (!_hasConnection)\r
+ {\r
+ StatusNotification.Notify(new Notification\r
+ {\r
+ Level = TraceLevel.Info,\r
+ Title = "Internet Connection",\r
+ Message = "Internet connectivity restored."\r
+ });\r
+ }\r
+ _hasConnection = true;\r
\r
- var nextTimes=TaskEx.WhenAll(tasks.ToList()).Result;\r
+ var accountBatches = new Dictionary<Uri, IEnumerable<string>>();\r
+ IEnumerable<string> batch = null;\r
+ if (_batchQueue.TryDequeue(out batch) && batch != null)\r
+ foreach (var account in _accounts.Values)\r
+ {\r
+ var accountBatch = batch.Where(path => path.IsAtOrBelow(account.AccountPath));\r
+ accountBatches[account.AccountKey] = accountBatch;\r
+ }\r
\r
- _firstPoll = false;\r
- //Reschedule the poll with the current timestamp as a "since" value\r
+ var moves = Interlocked.Exchange(ref _moves, new ConcurrentDictionary<string, MovedEventArgs>());\r
\r
- if (nextTimes.Length>0)\r
- nextSince = nextTimes.Min();\r
- if (Log.IsDebugEnabled)\r
- Log.DebugFormat("Next Poll at [{0}]",nextSince);\r
+ var tasks = new List<Task<DateTimeOffset?>>();\r
+ foreach (var accountInfo in _accounts.Values)\r
+ {\r
+ IEnumerable<string> accountBatch;\r
+ accountBatches.TryGetValue(accountInfo.AccountKey, out accountBatch);\r
+ var t = ProcessAccountFiles(accountInfo, accountBatch, moves, since);\r
+ tasks.Add(t);\r
+ }\r
+\r
+ var taskList = tasks.ToList();\r
+ var nextTimes = await TaskEx.WhenAll(taskList).ConfigureAwait(false);\r
+\r
+ _firstPoll = false;\r
+ //Reschedule the poll with the current timestamp as a "since" value\r
+\r
+ if (nextTimes.Length > 0)\r
+ nextSince = nextTimes.Min();\r
+ if (Log.IsDebugEnabled)\r
+ Log.DebugFormat("Next Poll for changes since [{0}]", nextSince);\r
+ }\r
}\r
catch (Exception ex)\r
{\r
try\r
{\r
//Wait for the polling interval to pass or the Sync event to be signalled\r
- nextSince = WaitForScheduledOrManualPoll(nextSince).Result;\r
+ nextSince = await WaitForScheduledOrManualPoll(nextSince).ConfigureAwait(false);\r
}\r
finally\r
{\r
//Ensure polling is scheduled even in case of error\r
+#pragma warning disable 4014\r
TaskEx.Run(()=>PollRemoteFiles(nextSince));\r
+#pragma warning restore 4014\r
//_pollAction.Post(new PollRequest {Since = nextSince});\r
}\r
}\r
/// </summary>\r
/// <param name="since"></param>\r
/// <returns></returns>\r
- private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
+ private async Task<DateTimeOffset?> WaitForScheduledOrManualPoll(DateTimeOffset? since)\r
{\r
var sync = _syncEvent.WaitAsync();\r
- var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval));\r
+ var delay = TimeSpan.FromSeconds(Settings.PollingInterval);\r
+ if (Log.IsDebugEnabled)\r
+ Log.DebugFormat("Next Poll at [{0}]", DateTime.Now.Add(delay));\r
+ var wait = TaskEx.Delay(delay);\r
\r
var signaledTask = await TaskEx.WhenAny(sync, wait).ConfigureAwait(false);\r
\r
\r
\r
\r
- public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, IEnumerable<string> accountBatch, ConcurrentDictionary<string, MovedEventArgs> moves, DateTime? since = null)\r
+ public async Task<DateTimeOffset?> ProcessAccountFiles(AccountInfo accountInfo, IEnumerable<string> accountBatch, ConcurrentDictionary<string, MovedEventArgs> moves, DateTimeOffset? since = null)\r
{\r
if (accountInfo == null)\r
throw new ArgumentNullException("accountInfo");\r
var client = new CloudFilesClient(accountInfo);\r
\r
//We don't need to check the trash container\r
- var containers = client.ListContainers(accountInfo.UserName)\r
- .Where(c=>c.Name!="trash")\r
+ var allContainers=await client.ListContainers(accountInfo.UserName).ConfigureAwait(false);\r
+ var containers = allContainers\r
+ .Where(c=>c.Name.ToString()!="trash")\r
.ToList();\r
\r
\r
//The nextSince time fallback time is the same as the current.\r
//If polling succeeds, the next Since time will be the smallest of the maximum modification times\r
//of the shared and account objects\r
- var nextSince = since;\r
+ DateTimeOffset? nextSince = since;\r
\r
try\r
{\r
//Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
//than delete a file that was created while we were executing the poll \r
\r
+ var token = _currentOperationCancellation.Token;\r
+\r
//Get the list of server objects changed since the last check\r
//The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
var listObjects = (from container in containers\r
select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
- client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
+ client.ListObjects(accountInfo.UserName, container.Name, since), container.Name,token)).ToList();\r
\r
- var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
- client.ListSharedObjects(_knownContainers,since), "shared");\r
+ var selectiveEnabled = Selectives.IsSelectiveEnabled(accountInfo.AccountKey);\r
+ var listShared = selectiveEnabled?\r
+ Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
+ client.ListSharedObjects(_knownContainers,since), "shared",token)\r
+ :Task.Factory.FromResult((IList<ObjectInfo>) new List<ObjectInfo>(),"shared");\r
+ \r
listObjects.Add(listShared);\r
var listTasks = await Task.Factory.WhenAll(listObjects.ToArray()).ConfigureAwait(false);\r
\r
using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
{\r
+\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
var dict = listTasks.ToDictionary(t => t.AsyncState);\r
\r
//Get all non-trash objects. Remember, the container name is stored in AsyncState\r
var remoteObjects = (from objectList in listTasks\r
- where (string)objectList.AsyncState != "trash"\r
+ where objectList.AsyncState.ToString() != "trash"\r
from obj in objectList.Result\r
orderby obj.Bytes ascending \r
select obj).ToList();\r
*/\r
\r
var cleanRemotes = (from info in remoteObjects.Union(sharedObjects)\r
- let name = info.Name??""\r
+ let name = info.Name.ToUnescapedString()??""\r
where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&\r
!name.StartsWith(FolderConstants.CacheFolder + "/",\r
StringComparison.InvariantCultureIgnoreCase)\r
select info).ToList();\r
\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
if (_firstPoll)\r
StatusKeeper.CleanupOrphanStates();\r
\r
var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
var currentRemotes = differencer.Current.ToList();\r
+\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+ \r
StatusKeeper.CleanupStaleStates(accountInfo, currentRemotes);\r
\r
//var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
//May have to wait if the FileAgent has asked for a Pause, due to local changes\r
await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
//Get the local files here \r
var agent = AgentLocator<FileAgent>.Get(accountInfo.AccountPath); \r
var files = LoadLocalFileTuples(accountInfo, accountBatch);\r
\r
- var states = FileState.Queryable.ToList(); \r
- \r
+\r
+ \r
+ //WARNING: GetFileSystemInfo may create state entries.\r
+ //TODO: Find a different way to create the tuples and block long filenames\r
var infos = (from remote in currentRemotes\r
let path = remote.RelativeUrlToFilePath(accountInfo.UserName)\r
let info=agent.GetFileSystemInfo(path)\r
+ where info != null\r
select Tuple.Create(info.FullName,remote))\r
.ToList();\r
\r
- var token = _currentOperationCancellation.Token;\r
+ var states = StatusKeeper.GetAllStates();\r
+\r
+ var tupleBuilder = new TupleBuilder(CancellationToken,StatusKeeper,StatusNotification,Settings);\r
\r
- var tuples = MergeSources(infos, files, states,moves).ToList();\r
+ var tuples = tupleBuilder.MergeSources(infos, files, states,moves).ToList();\r
\r
+ var processedPaths = new HashSet<string>();\r
//Process only the changes in the batch file, if one exists\r
var stateTuples = accountBatch==null?tuples:tuples.Where(t => accountBatch.Contains(t.FilePath));\r
foreach (var tuple in stateTuples.Where(s=>!s.Locked))\r
{\r
await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
//Set the Merkle Hash\r
//SetMerkleHash(accountInfo, tuple);\r
\r
- await SyncSingleItem(accountInfo, tuple, agent, moves,token).ConfigureAwait(false);\r
+ await SyncSingleItem(accountInfo, tuple, agent, moves,processedPaths,token).ConfigureAwait(false);\r
\r
}\r
\r
Pause = false;\r
}\r
\r
- private async Task SyncSingleItem(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves, CancellationToken token)\r
+ private async Task SyncSingleItem(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves,HashSet<string> processedPaths, CancellationToken token)\r
{\r
Log.DebugFormat("Sync [{0}] C:[{1}] L:[{2}] S:[{3}]", tuple.FilePath, tuple.C, tuple.L, tuple.S);\r
\r
+ //If the processed paths already contain the current path, exit\r
+ if (!processedPaths.Add(tuple.FilePath))\r
+ return;\r
+\r
try\r
{\r
bool isInferredParent = tuple.ObjectInfo != null && tuple.ObjectInfo.UUID.StartsWith("00000000-0000-0000");\r
if (tuple.S == tuple.L)\r
{\r
// No server changes\r
- //Has the file been renamed on the server?\r
- MoveForServerMove(accountInfo, tuple);\r
+ //Has the file been renamed locally?\r
+ if (!await MoveForLocalMove(accountInfo,tuple))\r
+ //Has the file been renamed on the server?\r
+ MoveForServerMove(accountInfo, tuple);\r
}\r
else\r
{\r
//Server file doesn't exist\r
//deleteObjectFromLocal()\r
using (\r
- StatusNotification.GetNotifier("Deleting local {0}", "Deleted local {0}",\r
- Path.GetFileName(localFilePath)))\r
+ StatusNotification.GetNotifier("Deleting local {0}", "Deleted local {0}",true,\r
+ localInfo.Name))\r
{\r
- StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
- FileOverlayStatus.Deleted, "");\r
- using (NetworkGate.Acquire(localFilePath, NetworkOperation.Deleting))\r
- {\r
- agent.Delete(localFilePath);\r
- }\r
- //updateRecord(Remove C, L)\r
- StatusKeeper.ClearFileStatus(localFilePath);\r
+ DeleteLocalFile(agent, localFilePath);\r
}\r
}\r
else\r
//downloadServerObject() // Result: L = S\r
//If the file has moved on the server, move it locally before downloading\r
using (\r
- StatusNotification.GetNotifier("Downloading {0}", "Downloaded {0}",\r
- Path.GetFileName(localFilePath)))\r
+ StatusNotification.GetNotifier("Downloading {0}", "Downloaded {0}",true,\r
+ localInfo.Name))\r
{\r
var targetPath = MoveForServerMove(accountInfo, tuple);\r
+ if (targetPath != null)\r
+ {\r
\r
- StatusKeeper.SetFileState(targetPath, FileStatus.Modified, FileOverlayStatus.Modified,\r
- "");\r
-\r
- await\r
- NetworkAgent.Downloader.DownloadCloudFile(accountInfo, tuple.ObjectInfo, targetPath,\r
- token)\r
- .ConfigureAwait(false);\r
- //updateRecord( L = S )\r
- StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
- tuple.ObjectInfo.X_Object_Hash);\r
-\r
- StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
+ await DownloadCloudFile(accountInfo, tuple, token, targetPath).ConfigureAwait(false);\r
\r
- AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
+ AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
+ }\r
}\r
-\r
- /*\r
- StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
- FileOverlayStatus.Normal, "");\r
- */\r
}\r
}\r
}\r
else\r
{\r
//uploadLocalObject() // Result: S = C, L = S \r
- var progress = new Progress<double>(d =>\r
- StatusNotification.Notify(new StatusNotification(String.Format("Merkle Hashing for Upload {0:p} of {1}", d, localInfo.Name))));\r
-\r
- //Debug.Assert(tuple.FileState !=null);\r
- var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
- accountInfo.BlockSize, accountInfo.BlockHash,\r
- "Poll", isUnselectedRootFolder,token,progress);\r
- using (\r
- StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}",\r
- Path.GetFileName(localFilePath)))\r
+ var progress = new Progress<HashProgress>(d =>\r
+ StatusNotification.Notify(new StatusNotification(String.Format("Merkle Hashing for Upload {0:p} of {1}", d.Percentage, localInfo.Name))));\r
+\r
+ //Is it an unselected root folder\r
+ var isCreation = isUnselectedRootFolder ||//or a new folder under a selected parent?\r
+ (localInfo is DirectoryInfo && Selectives.IsSelected(accountInfo, localInfo) && tuple.FileState == null && tuple.ObjectInfo == null);\r
+\r
+\r
+ //Is this a result of a FILE move with no modifications? Then try to move it,\r
+ //to avoid an expensive hash\r
+ if (!await MoveForLocalMove(accountInfo, tuple))\r
{\r
- await NetworkAgent.Uploader.UploadCloudFile(action, token).ConfigureAwait(false);\r
+ await UploadLocalFile(accountInfo, tuple, token, isCreation, localInfo,processedPaths, progress).ConfigureAwait(false);\r
}\r
\r
//updateRecord( S = C )\r
//State updated by the uploader\r
-\r
- if (isUnselectedRootFolder)\r
- {\r
- ProcessChildren(accountInfo, tuple, agent, moves,token);\r
+ \r
+ if (isCreation )\r
+ { \r
+ ProcessChildren(accountInfo, tuple, agent, moves,processedPaths,token);\r
}\r
}\r
}\r
{\r
//Detect server moves\r
var targetPath = MoveForServerMove(accountInfo, tuple);\r
- StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
- \r
- AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
+ if (targetPath != null)\r
+ {\r
+ Debug.Assert(tuple.Merkle != null);\r
+ StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo, tuple.Merkle);\r
+\r
+ AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
+ }\r
}\r
else\r
{\r
{\r
//In case of error log and retry with the next poll\r
Log.ErrorFormat("[SYNC] Failed for file {0}. Will Retry.\r\n{1}",tuple.FilePath,exc);\r
+ }\r
+ }\r
+\r
+ private void DeleteLocalFile(FileAgent agent, string localFilePath)\r
+ {\r
+ StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
+ FileOverlayStatus.Deleted, "");\r
+ using (NetworkGate.Acquire(localFilePath, NetworkOperation.Deleting))\r
+ {\r
+ agent.Delete(localFilePath);\r
+ }\r
+ //updateRecord(Remove C, L)\r
+ StatusKeeper.ClearFileStatus(localFilePath);\r
+ }\r
+\r
+ private async Task DownloadCloudFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token, string targetPath)\r
+ { \r
+ //Don't create a new state for non-existent files\r
+ if (File.Exists(targetPath) || Directory.Exists(targetPath))\r
+ StatusKeeper.SetFileState(targetPath, FileStatus.Modified, FileOverlayStatus.Modified,"");\r
+\r
+ var finalHash=await\r
+ NetworkAgent.Downloader.DownloadCloudFile(accountInfo, tuple.ObjectInfo, targetPath,\r
+ token)\r
+ .ConfigureAwait(false);\r
+ //updateRecord( L = S )\r
+ StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
+ finalHash);\r
+\r
+ StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo,finalHash);\r
+ }\r
\r
+ private async Task UploadLocalFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token,\r
+ bool isUnselectedRootFolder, FileSystemInfo localInfo, HashSet<string> processedPaths, IProgress<HashProgress> progress)\r
+ {\r
+ var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
+ accountInfo.BlockSize, accountInfo.BlockHash,\r
+ "Poll", isUnselectedRootFolder, token, progress,tuple.Merkle); \r
+\r
+ using (StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}",true,\r
+ localInfo.Name))\r
+ {\r
+ await NetworkAgent.Uploader.UploadCloudFile(action, token).ConfigureAwait(false);\r
+ }\r
+\r
+ if (isUnselectedRootFolder)\r
+ {\r
+ var dirActions =(\r
+ from dir in ((DirectoryInfo) localInfo).EnumerateDirectories("*", SearchOption.AllDirectories)\r
+ let subAction = new CloudUploadAction(accountInfo, dir, null,\r
+ accountInfo.BlockSize, accountInfo.BlockHash,\r
+ "Poll", true, token, progress)\r
+ select subAction).ToList();\r
+ foreach (var dirAction in dirActions)\r
+ {\r
+ processedPaths.Add(dirAction.LocalFile.FullName);\r
+ }\r
\r
+ await TaskEx.WhenAll(dirActions.Select(a=>NetworkAgent.Uploader.UploadCloudFile(a,token)).ToArray());\r
}\r
}\r
\r
+ private async Task<bool> MoveForLocalMove(AccountInfo accountInfo, StateTuple tuple)\r
+ {\r
+ //Is the previous path missing?\r
+ if (String.IsNullOrWhiteSpace(tuple.OldFullPath))\r
+ return false;\r
+ //Has the file locally, in which case it should be uploaded rather than moved?\r
+ if (tuple.OldChecksum != tuple.Merkle.TopHash.ToHashString())\r
+ return false;\r
+\r
+ var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
+ var serverPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
+ //Has the file been renamed on the server?\r
+ if (!tuple.OldFullPath.Equals(serverPath))\r
+ {\r
+ ReportConflictForDoubleRename(tuple.FilePath);\r
+ return false;\r
+ }\r
+\r
+ try\r
+ {\r
+\r
+ var client = new CloudFilesClient(accountInfo);\r
+ var objectInfo = CloudAction.CreateObjectInfoFor(accountInfo, tuple.FileInfo);\r
+ objectInfo.X_Object_Hash = tuple.Merkle.TopHash.ToHashString();\r
+ var containerPath = Path.Combine(accountInfo.AccountPath, objectInfo.Container.ToUnescapedString());\r
+ //TODO: SImplify these multiple conversions from and to Uris\r
+ var oldName = tuple.OldFullPath.AsRelativeTo(containerPath);\r
+ //Then execute a move instead of an upload\r
+ using (StatusNotification.GetNotifier("Moving {0}", "Moved {0}", true,tuple.FileInfo.Name))\r
+ {\r
+ await client.MoveObject(objectInfo.Account, objectInfo.Container, oldName.Replace('\\','/').ToEscapedUri(),\r
+ objectInfo.Container, objectInfo.Name).ConfigureAwait(false);\r
+ StatusKeeper.MoveFileState(tuple.OldFullPath, tuple.FilePath, objectInfo, tuple.Merkle);\r
+ //StatusKeeper.StoreInfo(tuple.FilePath,objectInfo,tuple.Merkle);\r
+ //StatusKeeper.ClearFolderStatus(tuple.FilePath);\r
+ }\r
+ return true;\r
+ }\r
+ catch (Exception exc)\r
+ {\r
+ Log.ErrorFormat("[MOVE] Failed for [{0}],:\r\n{1}", tuple.FilePath, exc);\r
+ //Return false to force an upload of the file\r
+ return false;\r
+ }\r
+\r
+ }\r
+\r
private void AddOwnFolderToSelectives(AccountInfo accountInfo, StateTuple tuple, string targetPath)\r
{\r
//Not for shared folders\r
if (tuple.ObjectInfo.IsShared==true)\r
return;\r
//Also ensure that any newly created folders are added to the selectives, if the original folder was selected\r
- var containerPath = Path.Combine(accountInfo.AccountPath, tuple.ObjectInfo.Container);\r
+ var containerPath = Path.Combine(accountInfo.AccountPath, tuple.ObjectInfo.Container.ToUnescapedString());\r
\r
//If this is a root folder encountered for the first time\r
if (tuple.L == null && Directory.Exists(tuple.FileInfo.FullName) \r
if (String.Equals(tuple.FilePath ,serverPath,StringComparison.InvariantCultureIgnoreCase)) \r
return serverPath;\r
\r
- if (tuple.FileInfo.Exists)\r
+ //Has the file been renamed locally?\r
+ if (!String.IsNullOrWhiteSpace(tuple.OldFullPath) && !tuple.OldFullPath.Equals(tuple.FilePath))\r
+ {\r
+ ReportConflictForDoubleRename(tuple.FilePath);\r
+ return null;\r
+ }\r
+\r
+ tuple.FileInfo.Refresh();\r
+ //The file/folder may not exist if it was moved because its parent moved\r
+ if (!tuple.FileInfo.Exists)\r
{\r
- using (StatusNotification.GetNotifier("Moving local {0}", "Moved local {0}", Path.GetFileName(tuple.FilePath)))\r
- using(NetworkGate.Acquire(tuple.FilePath,NetworkOperation.Renaming))\r
+ var target=FileInfoExtensions.FromPath(serverPath);\r
+ if (!target.Exists)\r
{\r
- var fi = tuple.FileInfo as FileInfo;\r
- if (fi != null)\r
- fi.MoveTo(serverPath);\r
- var di = tuple.FileInfo as DirectoryInfo;\r
- if (di != null)\r
- di.MoveTo(serverPath);\r
+ Log.ErrorFormat("No source or target found while trying to move {0} to {1}", tuple.FileInfo.FullName, serverPath);\r
}\r
- StatusKeeper.StoreInfo(serverPath, tuple.ObjectInfo);\r
+ return serverPath;\r
}\r
- else\r
+\r
+ using (StatusNotification.GetNotifier("Moving local {0}", "Moved local {0}", true,Path.GetFileName(tuple.FilePath)))\r
+ using(NetworkGate.Acquire(tuple.FilePath,NetworkOperation.Renaming))\r
{\r
- Debug.Assert(false, "File does not exist");\r
+ \r
+ var fi = tuple.FileInfo as FileInfo;\r
+ if (fi != null)\r
+ {\r
+ var targetFile = new FileInfo(serverPath);\r
+ if (!targetFile.Directory.Exists)\r
+ targetFile.Directory.Create();\r
+ fi.MoveTo(serverPath);\r
+ }\r
+ var di = tuple.FileInfo as DirectoryInfo;\r
+ if (di != null)\r
+ {\r
+ var targetDir = new DirectoryInfo(serverPath);\r
+ if (!targetDir.Parent.Exists)\r
+ targetDir.Parent.Create();\r
+ di.MoveTo(serverPath);\r
+ }\r
}\r
+\r
+ StatusKeeper.StoreInfo(serverPath, tuple.ObjectInfo);\r
+\r
return serverPath;\r
}\r
\r
private void DeleteCloudFile(AccountInfo accountInfo, StateTuple tuple)\r
{\r
- using (StatusNotification.GetNotifier("Deleting server {0}", "Deleted server {0}", Path.GetFileName(tuple.FilePath)))\r
+ using (StatusNotification.GetNotifier("Deleting server {0}", "Deleted server {0}", true,Path.GetFileName(tuple.FilePath)))\r
{\r
\r
StatusKeeper.SetFileState(tuple.FilePath, FileStatus.Deleted,\r
}\r
}\r
\r
- private void ProcessChildren(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves,CancellationToken token)\r
+ private void ProcessChildren(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves,HashSet<string> processedPaths,CancellationToken token)\r
{\r
\r
var dirInfo = tuple.FileInfo as DirectoryInfo;\r
var folderTuples = from folder in dirInfo.EnumerateDirectories("*", SearchOption.AllDirectories)\r
- select new StateTuple(folder);\r
+ select new StateTuple(folder){C=Signature.MERKLE_EMPTY};\r
+ \r
var fileTuples = from file in dirInfo.EnumerateFiles("*", SearchOption.AllDirectories)\r
- select new StateTuple(file);\r
+ let state=StatusKeeper.GetStateByFilePath(file.FullName)\r
+ select new StateTuple(file){\r
+ Merkle=StatusAgent.CalculateTreeHash(file,accountInfo,state,\r
+ Settings.HashingParallelism,token,null)\r
+ };\r
\r
//Process folders first, to ensure folders appear on the sever as soon as possible\r
- folderTuples.ApplyAction(async t =>await SyncSingleItem(accountInfo, t, agent, moves, token).ConfigureAwait(false));\r
+ folderTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, moves, processedPaths,token).Wait());\r
\r
- fileTuples.ApplyAction(async t => await SyncSingleItem(accountInfo, t, agent, moves, token).ConfigureAwait(false));\r
+ fileTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, moves,processedPaths, token).Wait());\r
}\r
\r
\r
- /*\r
- * //Use the queue to retry locked file hashing\r
- var fileQueue = new ConcurrentQueue<FileSystemInfo>(localInfos);\r
- \r
\r
- var results = new List<Tuple<FileSystemInfo, string>>();\r
- var backoff = 0;\r
- while (fileQueue.Count > 0)\r
- {\r
- FileSystemInfo file;\r
- fileQueue.TryDequeue(out file);\r
- using (ThreadContext.Stacks["File"].Push(file.FullName))\r
- {\r
- try\r
- {\r
- //Replace MD5 here, do the calc while syncing individual files\r
- string hash ;\r
- if (file is DirectoryInfo)\r
- hash = MD5_EMPTY;\r
- else\r
- {\r
- //Wait in case the FileAgent has requested a Pause\r
- await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
- \r
- using (StatusNotification.GetNotifier("Hashing {0}", "", file.Name))\r
- {\r
- hash = ((FileInfo)file).ComputeShortHash(StatusNotification);\r
- backoff = 0;\r
- }\r
- } \r
- results.Add(Tuple.Create(file, hash));\r
- }\r
- catch (IOException exc)\r
- {\r
- Log.WarnFormat("[HASH] File in use, will retry [{0}]", exc);\r
- fileQueue.Enqueue(file);\r
- //If this is the only enqueued file \r
- if (fileQueue.Count != 1) continue;\r
- \r
- \r
- //Increase delay\r
- if (backoff<60000)\r
- backoff += 10000;\r
- //Pause Polling for the specified time\r
- }\r
- if (backoff>0)\r
- await PauseFor(backoff).ConfigureAwait(false);\r
- }\r
- }\r
\r
- return results;\r
-\r
- */\r
- private IEnumerable<StateTuple> MergeSources(IEnumerable<Tuple<string, ObjectInfo>> infos, IEnumerable<FileSystemInfo> files, IEnumerable<FileState> states, ConcurrentDictionary<string, MovedEventArgs> moves)\r
- {\r
- var tuplesByPath = files.ToDictionary(f => f.FullName, f => new StateTuple {FileInfo = f}); new Dictionary<string, StateTuple>();\r
-\r
- //For files that have state\r
- foreach (var state in states)\r
- {\r
- StateTuple hashTuple;\r
- if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple))\r
- {\r
- hashTuple.FileState = state;\r
- UpdateMD5(hashTuple);\r
- }\r
- else\r
- {\r
- var fsInfo = FileInfoExtensions.FromPath(state.FilePath);\r
- hashTuple = new StateTuple {FileInfo = fsInfo, FileState = state};\r
- tuplesByPath[state.FilePath] = hashTuple;\r
- }\r
- }\r
- //for files that don't have state\r
- foreach (var tuple in tuplesByPath.Values.Where(t => t.FileState == null))\r
- {\r
- UpdateMD5(tuple);\r
- }\r
-\r
- var tuplesByID = tuplesByPath.Values\r
- .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null)\r
- .ToDictionary(tuple=>tuple.FileState.ObjectID,tuple=>tuple);//new Dictionary<Guid, StateTuple>();\r
-\r
- foreach (var info in infos)\r
- {\r
- StateTuple hashTuple;\r
- var filePath = info.Item1;\r
- var objectInfo = info.Item2;\r
- var objectID = objectInfo.UUID;\r
-\r
- if (objectID != _emptyGuid && tuplesByID.TryGetValue(objectID, out hashTuple))\r
- {\r
- hashTuple.ObjectInfo = objectInfo; \r
- }\r
- else if (tuplesByPath.TryGetValue(filePath, out hashTuple))\r
- {\r
- hashTuple.ObjectInfo = objectInfo;\r
- }\r
- else\r
- {\r
-\r
- \r
- var fsInfo = FileInfoExtensions.FromPath(filePath);\r
- hashTuple= new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo};\r
- tuplesByPath[filePath] = hashTuple;\r
- \r
- if (objectInfo.UUID!=_emptyGuid)\r
- tuplesByID[objectInfo.UUID] = hashTuple;\r
- }\r
- }\r
- Debug.Assert(tuplesByPath.Values.All(t => t.HashesValid()));\r
- return tuplesByPath.Values;\r
- }\r
+ \r
\r
- private void UpdateMD5(StateTuple hashTuple)\r
- {\r
- \r
- try\r
- {\r
- var hash = Signature.MD5_EMPTY;\r
- if (hashTuple.FileInfo is FileInfo)\r
- {\r
- var file = hashTuple.FileInfo as FileInfo;\r
- var stateDate = hashTuple.NullSafe(h => h.FileState).NullSafe(s => s.LastWriteDate) ??\r
- DateTime.MinValue;\r
- if (file.LastWriteTime - stateDate < TimeSpan.FromSeconds(1) &&\r
- hashTuple.FileState.LastLength == file.Length)\r
- {\r
- hash = hashTuple.FileState.LastMD5;\r
- }\r
- else\r
- {\r
- //Modified, must calculate hash\r
- hash = file.ComputeShortHash(StatusNotification);\r
- StatusKeeper.UpdateLastMD5(file, hash);\r
- }\r
- }\r
- hashTuple.C = hash;\r
- hashTuple.MD5 = hash;\r
- }\r
- catch (IOException)\r
- {\r
- hashTuple.Locked = true;\r
- } \r
- }\r
\r
/// <summary>\r
/// Returns the latest LastModified date from the list of objects, but only if it is before\r
/// <param name="threshold"></param>\r
/// <param name="cloudObjects"></param>\r
/// <returns></returns>\r
- private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
+ private static DateTimeOffset? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
{\r
- DateTime? maxDate = null;\r
+ DateTimeOffset? maxDate = null;\r
if (cloudObjects!=null && cloudObjects.Count > 0)\r
maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
- if (maxDate == null || maxDate == DateTime.MinValue)\r
+ if (!maxDate.HasValue)\r
return threshold;\r
- if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate)\r
+ if (!threshold.HasValue|| threshold > maxDate)\r
return maxDate;\r
return threshold;\r
}\r
/// <param name="threshold"></param>\r
/// <param name="cloudObjects"></param>\r
/// <returns></returns>\r
- private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
+ private static DateTimeOffset? GetLatestDateAfter(DateTimeOffset? threshold, IList<ObjectInfo> cloudObjects)\r
{\r
- DateTime? maxDate = null;\r
+ DateTimeOffset? maxDate = null;\r
if (cloudObjects!=null && cloudObjects.Count > 0)\r
maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
- if (maxDate == null || maxDate == DateTime.MinValue)\r
+ if (!maxDate.HasValue)\r
return threshold;\r
- if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate)\r
+ if (!threshold.HasValue|| threshold < maxDate)\r
return maxDate;\r
return threshold;\r
}\r
\r
readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
private bool _pause;\r
- private readonly string _emptyGuid = Guid.Empty.ToString();\r
+ \r
\r
\r
\r
StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
}\r
\r
+ private void ReportConflictForDoubleRename(string localFilePath)\r
+ {\r
+ if (String.IsNullOrWhiteSpace(localFilePath))\r
+ throw new ArgumentNullException("localFilePath");\r
+ Contract.EndContractBlock();\r
+\r
+ StatusKeeper.SetFileState(localFilePath, FileStatus.Conflict, FileOverlayStatus.Conflict, "File renamed both locally and on the server");\r
+ UpdateStatus(PithosStatus.HasConflicts);\r
+ var message = String.Format("Double rename conflict detected for file {0}", localFilePath);\r
+ Log.Warn(message);\r
+ StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
+ }\r
+\r
\r
/// <summary>\r
/// Notify the UI to update the visual status\r
private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)\r
{\r
var containerPaths = from container in containers\r
- let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)\r
- where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)\r
+ let containerPath = Path.Combine(accountInfo.AccountPath, container.Name.ToUnescapedString())\r
+ where container.Name.ToString() != FolderConstants.TrashContainer && !Directory.Exists(containerPath)\r
select containerPath;\r
\r
foreach (var path in containerPaths)\r
\r
public void RemoveAccount(AccountInfo accountInfo)\r
{\r
+ if (accountInfo == null)\r
+ return;\r
+\r
AccountInfo account;\r
_accounts.TryRemove(accountInfo.AccountKey, out account);\r
\r