+ private static void SetMerkleHash(AccountInfo accountInfo, StateTuple tuple)\r
+ {\r
+ //The Merkle hash for directories is that of an empty buffer\r
+ if (tuple.FileInfo is DirectoryInfo)\r
+ tuple.C = MERKLE_EMPTY;\r
+ else if (tuple.FileState != null && tuple.MD5 == tuple.FileState.ShortHash)\r
+ {\r
+ //If there is a state whose MD5 matches, load the merkle hash from the file state\r
+ //insteaf of calculating it\r
+ tuple.C = tuple.FileState.Checksum; \r
+ }\r
+ else\r
+ {\r
+ tuple.Merkle = TaskEx.Run(()=> Signature.CalculateTreeHash(tuple.FileInfo, accountInfo.BlockSize, accountInfo.BlockHash)).Result;\r
+ //tuple.C=tuple.Merkle.TopHash.ToHashString(); \r
+ }\r
+ }\r
+\r
+ private async Task<List<Tuple<FileSystemInfo, string>>> LoadLocalFileTuples(AccountInfo accountInfo,IEnumerable<string> batch )\r
+ {\r
+ using (ThreadContext.Stacks["Account Files Hashing"].Push(accountInfo.UserName))\r
+ {\r
+ var batchPaths = (batch==null)?new List<string>():batch.ToList();\r
+ IEnumerable<FileSystemInfo> localInfos=AgentLocator<FileAgent>.Get(accountInfo.AccountPath)\r
+ .EnumerateFileSystemInfos();\r
+ if (batchPaths.Count>0)\r
+ localInfos= localInfos.Where(fi => batchPaths.Contains(fi.FullName));\r
+ \r
+ //Use the queue to retry locked file hashing\r
+ var fileQueue = new Queue<FileSystemInfo>(localInfos);\r
+ var hasher = MD5.Create();\r
+\r
+ var results = new List<Tuple<FileSystemInfo, string>>();\r
+ var backoff = 0;\r
+ while (fileQueue.Count > 0)\r
+ {\r
+ var file = fileQueue.Dequeue();\r
+ using (ThreadContext.Stacks["File"].Push(file.FullName))\r
+ {\r
+ /*\r
+ Signature.CalculateTreeHash(file, accountInfo.BlockSize,\r
+ accountInfo.BlockHash).\r
+ TopHash.ToHashString()\r
+ */\r
+ try\r
+ {\r
+ //Replace MD5 here, do the calc while syncing individual files\r
+ string hash ;\r
+ if (file is DirectoryInfo)\r
+ hash = MERKLE_EMPTY;\r
+ else\r
+ {\r
+ //Wait in case the FileAgent has requested a Pause\r
+ await _unPauseEvent.WaitAsync();\r
+ \r
+ using (StatusNotification.GetNotifier("Hashing {0}", "Finished hashing {0}", file.Name))\r
+ using (var stream = (file as FileInfo).OpenRead())\r
+ { \r
+ hash = hasher.ComputeHash(stream).ToHashString();\r
+ backoff = 0;\r
+ }\r
+ } \r
+ results.Add(Tuple.Create(file, hash));\r
+ }\r
+ catch (IOException exc)\r
+ {\r
+ Log.WarnFormat("[HASH] File in use, will retry [{0}]", exc);\r
+ fileQueue.Enqueue(file);\r
+ //If this is the only enqueued file \r
+ if (fileQueue.Count != 1) continue;\r
+ \r
+ \r
+ //Increase delay\r
+ if (backoff<60000)\r
+ backoff += 10000;\r
+ //Pause Polling for the specified time\r
+ }\r
+ if (backoff>0)\r
+ await PauseFor(backoff);\r
+ }\r
+ }\r
+\r
+ return results;\r
+ }\r
+ }\r
+\r
+ /// <summary>\r
+ /// Wait and Pause the agent while waiting\r
+ /// </summary>\r
+ /// <param name="backoff"></param>\r
+ /// <returns></returns>\r
+ private async Task PauseFor(int backoff)\r
+ {\r
+\r
+ Pause = true;\r
+ await TaskEx.Delay(backoff);\r
+ Pause = false;\r
+ }\r
+\r
+ private void SyncSingleItem(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, CancellationToken token)\r
+ {\r
+ Log.DebugFormat("Sync [{0}] C:[{1}] L:[{2}] S:[{3}]",tuple.FilePath,tuple.C,tuple.L,tuple.S);\r
+\r
+ var localFilePath = tuple.FilePath;\r
+ //Don't use the tuple info, it may have been deleted\r
+ var localInfo = FileInfoExtensions.FromPath(localFilePath);\r
+\r
+\r
+ var isUnselectedRootFolder = agent.IsUnselectedRootFolder(tuple.FilePath);\r
+\r
+ //Unselected root folders that have not yet been uploaded should be uploaded and added to the \r
+ //selective folders\r
+\r
+ if (!Selectives.IsSelected(accountInfo, localFilePath) && !(isUnselectedRootFolder && tuple.ObjectInfo==null) ) \r
+ return;\r
+\r
+ // Local file unchanged? If both C and L are null, make sure it's because \r
+ //both the file is missing and the state checksum is not missing\r
+ if (tuple.C == tuple.L /*&& (localInfo.Exists || tuple.FileState == null)*/)\r
+ {\r
+ //No local changes\r
+ //Server unchanged?\r
+ if (tuple.S == tuple.L)\r
+ {\r
+ // No server changes\r
+ //Has the file been renamed on the server?\r
+ MoveForServerMove(accountInfo, tuple);\r
+ }\r
+ else\r
+ {\r
+ //Different from server\r
+ //Does the server file exist?\r
+ if (tuple.S == null)\r
+ {\r
+ //Server file doesn't exist\r
+ //deleteObjectFromLocal()\r
+ StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
+ FileOverlayStatus.Deleted, "");\r
+ agent.Delete(localFilePath);\r
+ //updateRecord(Remove C, L)\r
+ StatusKeeper.ClearFileStatus(localFilePath);\r
+ }\r
+ else\r
+ {\r
+ //Server file exists\r
+ //downloadServerObject() // Result: L = S\r
+ //If the file has moved on the server, move it locally before downloading\r
+ var targetPath = MoveForServerMove(accountInfo, tuple);\r
+\r
+ StatusKeeper.SetFileState(targetPath, FileStatus.Modified,\r
+ FileOverlayStatus.Modified, "");\r
+ NetworkAgent.Downloader.DownloadCloudFile(accountInfo,\r
+ tuple.ObjectInfo,\r
+ targetPath, tuple.Merkle, token).Wait(token);\r
+ //updateRecord( L = S )\r
+ StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
+ tuple.ObjectInfo.X_Object_Hash);\r
+\r
+ StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
+\r
+ /*\r
+ StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
+ FileOverlayStatus.Normal, "");\r
+ */\r
+ }\r
+ }\r
+\r
+ }\r
+ else\r
+ {\r
+ //Local changes found\r
+\r
+ //Server unchanged?\r
+ if (tuple.S == tuple.L)\r
+ {\r
+ //The FileAgent selective sync checks for new root folder files\r
+ if (!agent.Ignore(localFilePath))\r
+ {\r
+ if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
+ {\r
+ //deleteObjectFromServer()\r
+ DeleteCloudFile(accountInfo, tuple);\r
+ //updateRecord( Remove L, S) \r
+ }\r
+ else\r
+ {\r
+ //uploadLocalObject() // Result: S = C, L = S \r
+\r
+ //Debug.Assert(tuple.FileState !=null);\r
+ var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
+ accountInfo.BlockSize, accountInfo.BlockHash,\r
+ "Poll", isUnselectedRootFolder);\r
+ NetworkAgent.Uploader.UploadCloudFile(action, tuple.Merkle, token).Wait(token);\r
+\r
+ //updateRecord( S = C )\r
+ //State updated by the uploader\r
+\r
+ if (isUnselectedRootFolder)\r
+ {\r
+ ProcessChildren(accountInfo, tuple, agent, token);\r
+ }\r
+ }\r
+ }\r
+ }\r
+ else\r
+ {\r
+ if (tuple.C == tuple.S)\r
+ {\r
+ // (Identical Changes) Result: L = S\r
+ //doNothing()\r
+ //Detect server moves\r
+ var targetPath = MoveForServerMove(accountInfo, tuple);\r
+ StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
+ }\r
+ else\r
+ {\r
+ if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
+ {\r
+ //deleteObjectFromServer()\r
+ DeleteCloudFile(accountInfo, tuple);\r
+ //updateRecord(Remove L, S) \r
+ }\r
+ //If both the local and server files are missing, the state is stale\r
+ else if (!localInfo.Exists && (tuple.S == null || tuple.ObjectInfo == null))\r
+ {\r
+ StatusKeeper.ClearFileStatus(localInfo.FullName);\r
+ }\r
+ else\r
+ {\r
+ ReportConflictForMismatch(localFilePath);\r
+ //identifyAsConflict() // Manual action required\r
+ }\r
+ }\r
+ }\r
+ }\r
+ }\r
+\r
+ private string MoveForServerMove(AccountInfo accountInfo, StateTuple tuple)\r
+ {\r
+ if (tuple.ObjectInfo == null)\r
+ return null;\r
+ var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
+ var serverPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
+ \r
+ //Compare Case Insensitive\r
+ if (String.Equals(tuple.FilePath ,serverPath,StringComparison.InvariantCultureIgnoreCase)) return serverPath;\r
+\r
+ if (tuple.FileInfo.Exists)\r
+ { \r
+ var fi = tuple.FileInfo as FileInfo;\r
+ if (fi != null)\r
+ fi.MoveTo(serverPath);\r
+ var di = tuple.FileInfo as DirectoryInfo;\r
+ if (di != null)\r
+ di.MoveTo(serverPath);\r
+ StatusKeeper.StoreInfo(serverPath, tuple.ObjectInfo);\r
+ }\r
+ else\r
+ {\r
+ Debug.Assert(false, "File does not exist");\r
+ }\r
+ return serverPath;\r
+ }\r
+\r
+ private void DeleteCloudFile(AccountInfo accountInfo, StateTuple tuple)\r
+ {\r
+ StatusKeeper.SetFileState(tuple.FilePath, FileStatus.Deleted,\r
+ FileOverlayStatus.Deleted, "");\r
+ NetworkAgent.DeleteAgent.DeleteCloudFile(accountInfo, tuple.ObjectInfo);\r
+ StatusKeeper.ClearFileStatus(tuple.FilePath);\r
+ }\r
+\r
+ private void ProcessChildren(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, CancellationToken token)\r
+ {\r
+\r
+ var dirInfo = tuple.FileInfo as DirectoryInfo;\r
+ var folderTuples = from folder in dirInfo.EnumerateDirectories("*", SearchOption.AllDirectories)\r
+ select new StateTuple(folder);\r
+ var fileTuples = from file in dirInfo.EnumerateFiles("*", SearchOption.AllDirectories)\r
+ select new StateTuple(file);\r
+ \r
+ //Process folders first, to ensure folders appear on the sever as soon as possible\r
+ folderTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, token));\r
+ \r
+ fileTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, token));\r
+ }\r
+\r
+ private static IEnumerable<StateTuple> MergeSources(\r
+ IEnumerable<Tuple<string, ObjectInfo>> infos, \r
+ IEnumerable<Tuple<FileSystemInfo, string>> files, \r
+ IEnumerable<FileState> states)\r
+ {\r
+ var tuplesByPath = new Dictionary<string, StateTuple>();\r
+ foreach (var file in files)\r
+ {\r
+ var fsInfo = file.Item1;\r
+ var fileHash = fsInfo is DirectoryInfo? MERKLE_EMPTY:file.Item2;\r
+\r
+ tuplesByPath[fsInfo.FullName] = new StateTuple {FileInfo = fsInfo, MD5 = fileHash};\r
+ }\r
+ foreach (var state in states)\r
+ {\r
+ StateTuple hashTuple;\r
+ if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple))\r
+ {\r
+ hashTuple.FileState = state;\r
+ }\r
+ else\r
+ {\r
+ var fsInfo = FileInfoExtensions.FromPath(state.FilePath);\r
+ tuplesByPath[state.FilePath] = new StateTuple {FileInfo = fsInfo, FileState = state};\r
+ }\r
+ }\r
+\r
+ var tuplesByID = tuplesByPath.Values\r
+ .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null)\r
+ .ToDictionary(tuple=>tuple.FileState.ObjectID,tuple=>tuple);//new Dictionary<Guid, StateTuple>();\r
+\r
+ foreach (var info in infos)\r
+ {\r
+ StateTuple hashTuple;\r
+ var filePath = info.Item1;\r
+ var objectInfo = info.Item2;\r
+ var objectID = objectInfo.UUID;\r
+\r
+ if (tuplesByID.TryGetValue(objectID, out hashTuple))\r
+ {\r
+ hashTuple.ObjectInfo = objectInfo; \r
+ }\r
+ else if (tuplesByPath.TryGetValue(filePath, out hashTuple))\r
+ {\r
+ hashTuple.ObjectInfo = objectInfo;\r
+ }\r
+ else\r
+ {\r
+ var fsInfo = FileInfoExtensions.FromPath(filePath);\r
+ var tuple = new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo};\r
+ tuplesByPath[filePath] = tuple;\r
+ tuplesByID[objectInfo.UUID] = tuple;\r
+ }\r
+ }\r
+ return tuplesByPath.Values;\r
+ }\r
+\r
projects / pithos-ms-client / blobdiff