public void SynchNow(IEnumerable<string> paths=null)\r
{\r
_batchQueue.Enqueue(paths);\r
- _syncEvent.Set(); \r
+ _syncEvent.SetAsync(); \r
+\r
+ //_pollAction.Post(new PollRequest {Batch = paths});\r
+ }\r
+\r
+ /// <summary>\r
+ /// Start a manual synchronization\r
+ /// </summary>\r
+ public Task SynchNowAsync(IEnumerable<string> paths=null)\r
+ {\r
+ _batchQueue.Enqueue(paths);\r
+ return _syncEvent.SetAsync(); \r
\r
//_pollAction.Post(new PollRequest {Batch = paths});\r
}\r
//Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
//than delete a file that was created while we were executing the poll \r
\r
+ var token = _currentOperationCancellation.Token;\r
+\r
//Get the list of server objects changed since the last check\r
//The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
var listObjects = (from container in containers\r
select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
- client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
+ client.ListObjects(accountInfo.UserName, container.Name, since), container.Name,token)).ToList();\r
\r
- var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
- client.ListSharedObjects(_knownContainers,since), "shared");\r
+ var selectiveEnabled = Selectives.IsSelectiveEnabled(accountInfo.AccountKey);\r
+ var listShared = selectiveEnabled?\r
+ Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
+ client.ListSharedObjects(_knownContainers,since), "shared",token)\r
+ :Task.Factory.FromResult((IList<ObjectInfo>) new List<ObjectInfo>(),"shared");\r
+ \r
listObjects.Add(listShared);\r
var listTasks = await Task.Factory.WhenAll(listObjects.ToArray()).ConfigureAwait(false);\r
\r
using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
{\r
+\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
var dict = listTasks.ToDictionary(t => t.AsyncState);\r
\r
//Get all non-trash objects. Remember, the container name is stored in AsyncState\r
StringComparison.InvariantCultureIgnoreCase)\r
select info).ToList();\r
\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
if (_firstPoll)\r
StatusKeeper.CleanupOrphanStates();\r
\r
var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
var currentRemotes = differencer.Current.ToList();\r
+\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+ \r
StatusKeeper.CleanupStaleStates(accountInfo, currentRemotes);\r
\r
//var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
//May have to wait if the FileAgent has asked for a Pause, due to local changes\r
await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
//Get the local files here \r
var agent = AgentLocator<FileAgent>.Get(accountInfo.AccountPath); \r
var files = LoadLocalFileTuples(accountInfo, accountBatch);\r
\r
\r
- var states = StatusKeeper.GetAllStates();\r
- \r
+ \r
+ //WARNING: GetFileSystemInfo may create state entries.\r
+ //TODO: Find a different way to create the tuples and block long filenames\r
var infos = (from remote in currentRemotes\r
let path = remote.RelativeUrlToFilePath(accountInfo.UserName)\r
let info=agent.GetFileSystemInfo(path)\r
+ where info != null\r
select Tuple.Create(info.FullName,remote))\r
.ToList();\r
\r
- var token = _currentOperationCancellation.Token;\r
+ var states = StatusKeeper.GetAllStates();\r
+\r
+ var tupleBuilder = new TupleBuilder(CancellationToken,StatusKeeper,StatusNotification,Settings);\r
\r
- var tuples = MergeSources(infos, files, states,moves).ToList();\r
+ var tuples = tupleBuilder.MergeSources(infos, files, states,moves).ToList();\r
\r
var processedPaths = new HashSet<string>();\r
//Process only the changes in the batch file, if one exists\r
{\r
await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
//Set the Merkle Hash\r
//SetMerkleHash(accountInfo, tuple);\r
\r
var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
accountInfo.BlockSize, accountInfo.BlockHash,\r
"Poll", isUnselectedRootFolder, token, progress,tuple.Merkle); \r
- \r
+\r
using (StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}",true,\r
localInfo.Name))\r
{\r
\r
private async Task<bool> MoveForLocalMove(AccountInfo accountInfo, StateTuple tuple)\r
{\r
- //Is the file a directory or previous path missing?\r
- if (tuple.FileInfo is DirectoryInfo)\r
- return false;\r
//Is the previous path missing?\r
if (String.IsNullOrWhiteSpace(tuple.OldFullPath))\r
return false;\r
\r
var client = new CloudFilesClient(accountInfo);\r
var objectInfo = CloudAction.CreateObjectInfoFor(accountInfo, tuple.FileInfo);\r
+ objectInfo.X_Object_Hash = tuple.Merkle.TopHash.ToHashString();\r
var containerPath = Path.Combine(accountInfo.AccountPath, objectInfo.Container.ToUnescapedString());\r
//TODO: SImplify these multiple conversions from and to Uris\r
var oldName = tuple.OldFullPath.AsRelativeTo(containerPath);\r
//Then execute a move instead of an upload\r
using (StatusNotification.GetNotifier("Moving {0}", "Moved {0}", true,tuple.FileInfo.Name))\r
{\r
- await client.MoveObject(objectInfo.Account, objectInfo.Container, oldName.ToEscapedUri(),\r
+ await client.MoveObject(objectInfo.Account, objectInfo.Container, oldName.Replace('\\','/').ToEscapedUri(),\r
objectInfo.Container, objectInfo.Name).ConfigureAwait(false);\r
+ StatusKeeper.MoveFileState(tuple.OldFullPath, tuple.FilePath, objectInfo, tuple.Merkle);\r
+ //StatusKeeper.StoreInfo(tuple.FilePath,objectInfo,tuple.Merkle);\r
+ //StatusKeeper.ClearFolderStatus(tuple.FilePath);\r
}\r
return true;\r
}\r
\r
\r
\r
- private IEnumerable<StateTuple> MergeSources(IEnumerable<Tuple<string, ObjectInfo>> infos, IEnumerable<FileSystemInfo> files, List<FileState> states, ConcurrentDictionary<string, MovedEventArgs> moves)\r
- {\r
- var tuplesByPath = new Dictionary<string, StateTuple>();\r
- foreach (var info in files)\r
- {\r
- var tuple = new StateTuple(info);\r
- //Is this the target of a move event?\r
- var moveArg = moves.Values.FirstOrDefault(arg => info.FullName.Equals(arg.FullPath, StringComparison.InvariantCultureIgnoreCase) \r
- || info.FullName.IsAtOrBelow(arg.FullPath));\r
- if (moveArg != null)\r
- {\r
- tuple.NewFullPath = info.FullName;\r
- var relativePath = info.AsRelativeTo(moveArg.FullPath); \r
- tuple.OldFullPath = Path.Combine(moveArg.OldFullPath, relativePath);\r
- tuple.OldChecksum = states.FirstOrDefault(st => st.FilePath.Equals(tuple.OldFullPath, StringComparison.InvariantCultureIgnoreCase))\r
- .NullSafe(st => st.Checksum);\r
- }\r
-\r
- tuplesByPath[tuple.FilePath] = tuple;\r
- }\r
- \r
-\r
- \r
- \r
- //For files that have state\r
- foreach (var state in states)\r
- {\r
- StateTuple hashTuple;\r
-\r
- \r
- if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple))\r
- {\r
- hashTuple.FileState = state;\r
- UpdateHashes(hashTuple);\r
- }\r
- else if (moves.ContainsKey(state.FilePath) && tuplesByPath.TryGetValue(moves[state.FilePath].FullPath, out hashTuple))\r
- {\r
- hashTuple.FileState = state;\r
- UpdateHashes(hashTuple);\r
- }\r
- else\r
- {\r
- var fsInfo = FileInfoExtensions.FromPath(state.FilePath);\r
- hashTuple = new StateTuple {FileInfo = fsInfo, FileState = state};\r
-\r
- //Is the source of a moved item?\r
- var moveArg = moves.Values.FirstOrDefault(arg => state.FilePath.Equals(arg.OldFullPath,StringComparison.InvariantCultureIgnoreCase) \r
- || state.FilePath.IsAtOrBelow(arg.OldFullPath));\r
- if (moveArg != null)\r
- {\r
- var relativePath = state.FilePath.AsRelativeTo(moveArg.OldFullPath);\r
- hashTuple.NewFullPath = Path.Combine(moveArg.FullPath,relativePath);\r
- hashTuple.OldFullPath = state.FilePath;\r
- //Do we have the old MD5?\r
- //hashTuple.OldMD5 = state.LastMD5;\r
- }\r
-\r
-\r
- tuplesByPath[state.FilePath] = hashTuple;\r
- }\r
- }\r
- //for files that don't have state\r
- var statelessTuples = tuplesByPath.Values.Where(t => t.FileState == null).ToArray();\r
- //If there are too many stateless tuples, update them in parallel\r
- if (statelessTuples.Length > 20)\r
- Parallel.ForEach(statelessTuples, UpdateHashes);\r
- else\r
- statelessTuples.ApplyAction(UpdateHashes);\r
-\r
- var tuplesByID = tuplesByPath.Values\r
- .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null)\r
- .ToDictionary(tuple=>tuple.FileState.ObjectID,tuple=>tuple);//new Dictionary<Guid, StateTuple>();\r
-\r
- foreach (var info in infos)\r
- {\r
- StateTuple hashTuple;\r
- var filePath = info.Item1;\r
- var objectInfo = info.Item2;\r
- var objectID = objectInfo.UUID;\r
-\r
- if (objectID != _emptyGuid && tuplesByID.TryGetValue(objectID, out hashTuple))\r
- {\r
- hashTuple.ObjectInfo = objectInfo; \r
- }\r
- else if (tuplesByPath.TryGetValue(filePath, out hashTuple))\r
- {\r
- hashTuple.ObjectInfo = objectInfo;\r
- }\r
- else\r
- {\r
-\r
- \r
- var fsInfo = FileInfoExtensions.FromPath(filePath);\r
- hashTuple= new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo};\r
- tuplesByPath[filePath] = hashTuple;\r
- \r
- if (objectInfo.UUID!=_emptyGuid)\r
- tuplesByID[objectInfo.UUID] = hashTuple;\r
- }\r
- }\r
-\r
- var tuples = tuplesByPath.Values;\r
- //Sync algorithm fallout: There are multiple ways we can reach a situation where a state without a checksum exists\r
- //1: The application stopped/crashed while downloading a file. The file's entry was created when the download started. When the application restarts,\r
- // it finds no local file, a server file and a null state -> C: NULL L: NULL but exists, S: Some\r
- // It can be fixed by NOT creating a local state if the file doesn't already exist, or adding extra info to mark this as a result of an upload\r
- //2: A new file is added but the app stops/crashes after uploading finishes but before the entry gets updated and the user deletes the file. The file's entry was created. When the app restarts,\r
- // it finds no local file, a server file and a null state -> C: NULL L: NULL but exists, S: Some\r
- // \r
-\r
- var brokenTuples = from tuple in tuples\r
- where tuple.FileState != null && tuple.FileState.Checksum == null\r
- && tuple.ObjectInfo != null && (tuple.FileInfo==null || !tuple.FileInfo.Exists)\r
- select tuple;\r
-\r
-\r
- var actualTuples = tuples.Except(brokenTuples);\r
- Debug.Assert(actualTuples.All(t => t.HashesValid()));\r
-\r
- foreach (var tuple in brokenTuples)\r
- {\r
- StatusKeeper.SetFileState(tuple.FileState.FilePath, \r
- FileStatus.Conflict, FileOverlayStatus.Conflict, "FileState without checksum encountered for server object missing from disk"); \r
- }\r
- \r
- return actualTuples;\r
- }\r
\r
\r
- /// <summary>\r
- /// Update the tuple with the file's hashes, avoiding calculation if the file is unchanged\r
- /// </summary>\r
- /// <param name="hashTuple"></param>\r
- /// <remarks>\r
- /// The function first checks the file's size and last write date to see if there are any changes. If there are none,\r
- /// the file's stored hashes are used.\r
- /// Otherwise, MD5 is calculated first to ensure there are no changes. If MD5 is different, the Merkle hash is calculated\r
- /// </remarks>\r
- private void UpdateHashes(StateTuple hashTuple)\r
- {\r
- \r
- try\r
- {\r
- var state = hashTuple.NullSafe(s => s.FileState);\r
- var storedHash = state.NullSafe(s => s.Checksum);\r
- var storedHashes = state.NullSafe(s => s.Hashes);\r
- //var storedMD5 = state.NullSafe(s => s.LastMD5);\r
- var storedDate = state.NullSafe(s => s.LastWriteDate) ?? DateTime.MinValue;\r
- var storedLength = state.NullSafe(s => s.LastLength);\r
\r
- //var md5Hash = Signature.MD5_EMPTY; \r
- var merkle=TreeHash.Empty;\r
-\r
- if (hashTuple.FileInfo is FileInfo)\r
- {\r
- var file = (FileInfo)hashTuple.FileInfo.WithProperCapitalization();\r
- \r
- //Attributes unchanged?\r
- //LastWriteTime is only accurate to the second\r
- var unchangedAttributes = file.LastWriteTime - storedDate < TimeSpan.FromSeconds(1) \r
- && storedLength == file.Length;\r
- \r
- //Attributes appear unchanged but the file length doesn't match the stored hash ?\r
- var nonEmptyMismatch = unchangedAttributes && \r
- (file.Length == 0 ^ storedHash== Signature.MERKLE_EMPTY);\r
-\r
- //Missing hashes for NON-EMPTY hash ?\r
- var missingHashes = storedHash != Signature.MERKLE_EMPTY &&\r
- String.IsNullOrWhiteSpace(storedHashes);\r
-\r
- //Unchanged attributes but changed MD5 \r
- //Short-circuiting ensures MD5 is computed only if the attributes are changed\r
- \r
- //var md5Mismatch = (!unchangedAttributes && file.ComputeShortHash(StatusNotification) != storedMD5);\r
-\r
-\r
- //If the attributes are unchanged but the Merkle doesn't match the size,\r
- //or the attributes and the MD5 hash have changed, \r
- //or the hashes are missing but the tophash is NOT empty, we need to recalculate\r
- //\r
- //Otherwise we load the hashes from state\r
- if (!unchangedAttributes || nonEmptyMismatch || missingHashes)\r
- merkle = RecalculateTreehash(file);\r
- else\r
- {\r
- merkle=TreeHash.Parse(hashTuple.FileState.Hashes);\r
- //merkle.MD5 = storedMD5;\r
- }\r
-\r
-\r
- //md5Hash = merkle.MD5;\r
- }\r
- //hashTuple.MD5 = md5Hash;\r
- //Setting Merkle also updates C\r
- hashTuple.Merkle = merkle;\r
- }\r
- catch (IOException)\r
- {\r
- hashTuple.Locked = true;\r
- } \r
- }\r
-\r
- /// <summary>\r
- /// Recalculate a file's treehash and md5 and update the database\r
- /// </summary>\r
- /// <param name="file"></param>\r
- /// <returns></returns>\r
- private TreeHash RecalculateTreehash(FileInfo file)\r
- {\r
- var progress = new Progress<HashProgress>(d =>StatusNotification.Notify(\r
- new StatusNotification(String.Format("Hashing {0:p} of {1}", d.Percentage, file.Name))));\r
- var merkle = Signature.CalculateTreeHash(file, StatusKeeper.BlockSize, StatusKeeper.BlockHash, \r
- Settings.HashingParallelism, CancellationToken, progress);\r
- \r
- StatusKeeper.UpdateFileHashes(file.FullName, merkle);\r
- return merkle;\r
- }\r
\r
/// <summary>\r
/// Returns the latest LastModified date from the list of objects, but only if it is before\r
\r
readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
private bool _pause;\r
- private readonly string _emptyGuid = Guid.Empty.ToString();\r
+ \r
\r
\r
\r
\r
public void RemoveAccount(AccountInfo accountInfo)\r
{\r
+ if (accountInfo == null)\r
+ return;\r
+\r
AccountInfo account;\r
_accounts.TryRemove(accountInfo.AccountKey, out account);\r
\r
projects / pithos-ms-client / blobdiff