From 1f3d113a2adc37d11c2d4529909c4daf6ae64efb Mon Sep 17 00:00:00 2001 From: pkanavos Date: Thu, 8 Nov 2012 12:20:12 +0200 Subject: [PATCH] Fix for directory renames Move detection re-enabled --- trunk/Pithos.Core/Agents/PollAgent.cs | 7 +-- trunk/Pithos.Core/Agents/StatusAgent.cs | 74 ++++++++++++++++++++++++++++++ trunk/Pithos.Core/Agents/TupleBuilder.cs | 38 +++++++++------ trunk/Pithos.Core/IStatusKeeper.cs | 18 ++++++++ 4 files changed, 119 insertions(+), 18 deletions(-) diff --git a/trunk/Pithos.Core/Agents/PollAgent.cs b/trunk/Pithos.Core/Agents/PollAgent.cs index d77b835..275fdaa 100644 --- a/trunk/Pithos.Core/Agents/PollAgent.cs +++ b/trunk/Pithos.Core/Agents/PollAgent.cs @@ -788,9 +788,6 @@ namespace Pithos.Core.Agents private async Task MoveForLocalMove(AccountInfo accountInfo, StateTuple tuple) { - //Is the file a directory or previous path missing? - if (tuple.FileInfo is DirectoryInfo) - return false; //Is the previous path missing? if (String.IsNullOrWhiteSpace(tuple.OldFullPath)) return false; @@ -812,6 +809,7 @@ namespace Pithos.Core.Agents var client = new CloudFilesClient(accountInfo); var objectInfo = CloudAction.CreateObjectInfoFor(accountInfo, tuple.FileInfo); + objectInfo.X_Object_Hash = tuple.Merkle.TopHash.ToHashString(); var containerPath = Path.Combine(accountInfo.AccountPath, objectInfo.Container.ToUnescapedString()); //TODO: SImplify these multiple conversions from and to Uris var oldName = tuple.OldFullPath.AsRelativeTo(containerPath); @@ -820,6 +818,9 @@ namespace Pithos.Core.Agents { await client.MoveObject(objectInfo.Account, objectInfo.Container, oldName.Replace('\\','/').ToEscapedUri(), objectInfo.Container, objectInfo.Name).ConfigureAwait(false); + StatusKeeper.MoveFileState(tuple.OldFullPath, tuple.FilePath, objectInfo, tuple.Merkle); + //StatusKeeper.StoreInfo(tuple.FilePath,objectInfo,tuple.Merkle); + //StatusKeeper.ClearFolderStatus(tuple.FilePath); } return true; } diff --git a/trunk/Pithos.Core/Agents/StatusAgent.cs b/trunk/Pithos.Core/Agents/StatusAgent.cs index c2f95e2..d71aaab 100644 --- a/trunk/Pithos.Core/Agents/StatusAgent.cs +++ b/trunk/Pithos.Core/Agents/StatusAgent.cs @@ -981,6 +981,80 @@ namespace Pithos.Core.Agents } } + public void MoveFileState(string oldFullPath, string newFullPath, ObjectInfo objectInfo, TreeHash treeHash) + { + if (String.IsNullOrWhiteSpace(oldFullPath)) + throw new ArgumentNullException("oldFullPath"); + if (!Path.IsPathRooted(oldFullPath)) + throw new ArgumentException("The path must be rooted", "oldFullPath"); + if (String.IsNullOrWhiteSpace(newFullPath)) + throw new ArgumentNullException("newFullPath"); + if (!Path.IsPathRooted(newFullPath)) + throw new ArgumentException("The path must be rooted", "newFullPath"); + if (treeHash == null) + throw new ArgumentNullException("treeHash"); + if (objectInfo == null) + throw new ArgumentNullException("objectInfo", "objectInfo can't be empty"); + Contract.EndContractBlock(); + + + try + { + using (var session = _factory.OpenSession()) + using (var tx=session.BeginTransaction(IsolationLevel.ReadCommitted)) + { + //An entry for the new path may exist, + IQuery deletecmd = session.CreateQuery( + "delete from FileState where FilePath=:path and ObjectID is null") + .SetString("path", newFullPath); + deletecmd.ExecuteUpdate(); + + //string md5=treeHash.NullSafe(t=>t.MD5); + string hashes = treeHash.NullSafe(t => t.ToJson()); + + var info = FileInfoExtensions.FromPath(newFullPath); + var lastWriteTime = info.LastWriteTime; + var isFolder = (info is DirectoryInfo); + var lastLength = isFolder ? 0 : ((FileInfo) info).Length; + + var state = session.Query().SingleOrDefault(s => s.ObjectID == (objectInfo.UUID??"ยง")) //Handle null UUIDs + ?? session.Query().SingleOrDefault(s => s.FilePath == newFullPath) + ?? new FileState(); + state.FilePath = newFullPath; + state.IsFolder = isFolder; + state.LastWriteDate = lastWriteTime; + state.LastLength = lastLength; + state.Checksum = objectInfo.X_Object_Hash; + state.Hashes = hashes; + state.Version = objectInfo.Version.GetValueOrDefault(); + state.VersionTimeStamp = objectInfo.VersionTimestamp; + state.ETag = objectInfo.ETag; + state.FileStatus = FileStatus.Unchanged; + state.OverlayStatus = FileOverlayStatus.Normal; + state.ObjectID = objectInfo.UUID; + state.Modified = DateTime.Now; + session.SaveOrUpdate(state); + + + //Delete the old path entry if it still exists (eg. for folders) + session.CreateQuery("delete from FileState where FilePath = :path") + .SetParameter("path", oldFullPath) + .ExecuteUpdate(); + + + session.Flush(); + tx.Commit(); + if (Log.IsDebugEnabled) + Log.DebugFormat("DebugDB [{0}]:[{1}]\r\n{2}", newFullPath, objectInfo.UUID, objectInfo.X_Object_Hash); + } + } + catch (Exception exc) + { + Log.ErrorFormat("Failed to update [{0}]:[{1}]\r\n{2}",newFullPath,objectInfo.UUID, exc); + throw; + } + } + public void UpdateFileChecksum(string path, string etag, TreeHash treeHash) { if (String.IsNullOrWhiteSpace(path)) diff --git a/trunk/Pithos.Core/Agents/TupleBuilder.cs b/trunk/Pithos.Core/Agents/TupleBuilder.cs index 40b215f..1a4a277 100644 --- a/trunk/Pithos.Core/Agents/TupleBuilder.cs +++ b/trunk/Pithos.Core/Agents/TupleBuilder.cs @@ -48,10 +48,10 @@ namespace Pithos.Core.Agents { var tuplesByPath = new Dictionary(); //Fill the tuples with the local files - CreateTuplesFromFiles(files, states, moves, tuplesByPath); + CreateTuplesFromFiles(files, tuplesByPath); //Merge the file tuples with the local states, creating new tuples for states that have no matching files - MergeLocalStates(states, moves, tuplesByPath); + MergeLocalStates(states, tuplesByPath); MergeCloudFiles(infos, tuplesByPath); @@ -99,12 +99,15 @@ namespace Pithos.Core.Agents /// private void DetectLocalMoves(Dictionary tuplesByPath) { - return; + Func isNew = t => t.C != null + && (t.L == null || t.NullSafe(tp => tp.FileState).NullSafe( s => !s.FilePath.Equals(t.FilePath))) + && t.S == null; //Newly created fiels are candidate TOs - var fileCreates = tuplesByPath.Values.Where(t=> t.FileInfo is FileInfo && t.C!=null && t.L == null && t.S == null); - var folderCreates = tuplesByPath.Values.Where(t => t.FileInfo is DirectoryInfo && t.C != null && t.L == null && t.S == null); + var fileCreates = tuplesByPath.Values.Where(t=> t.FileInfo is FileInfo && isNew(t)); + var folderCreates = tuplesByPath.Values.Where(t => t.FileInfo is DirectoryInfo && isNew(t)); //Newly deleted files are candidate FROMs var fileDeletes = tuplesByPath.Values.Where(t =>t.NullSafe(t1=>t1.FileState).NullSafe(s=>!s.IsFolder) && t.C == null && t.L != null && t.L==t.S); + var folderDeletes = tuplesByPath.Values.Where(t => t.NullSafe(t1 => t1.FileState).NullSafe(s => s.IsFolder) && t.C == null && t.L != null && t.L == t.S); var moves = (from tuple in fileCreates let froms = fileDeletes.Where(d => d.L == tuple.C) @@ -126,8 +129,12 @@ namespace Pithos.Core.Agents //Can't create ObjectInfo, FileState for a directory if we don't store directories in the database //Find a folderCreate that matches the TO and a folderDelete that matches the FROM - /* var toFolder = folderCreates.SingleOrDefault(fd => move.To.FilePath.IsAtOrDirectlyBelow(fd.FilePath)); + var toFolder = folderCreates.SingleOrDefault(fd => move.To.FilePath.IsAtOrDirectlyBelow(fd.FilePath)); + var fromFolder = folderDeletes.SingleOrDefault(fd => move.From.FilePath.IsAtOrDirectlyBelow(fd.FilePath)); + if (fromFolder !=null && toFolder!= null) + ReplaceTupleForMove(tuplesByPath,fromFolder,toFolder); //Folders may not be stored in states +/* if (toFolder != null) { var fromPath = Path.GetDirectoryName(fromTuple.FilePath); @@ -138,7 +145,8 @@ namespace Pithos.Core.Agents toFolder.FileState = fromTuple.FileState; toFolder.ObjectInfo = fromTuple.ObjectInfo; - }*/ + } +*/ } @@ -155,13 +163,13 @@ namespace Pithos.Core.Agents toTuple.ObjectInfo = fromTuple.ObjectInfo; } - private static void CreateTuplesFromFiles(IEnumerable files, List states, ConcurrentDictionary moves, Dictionary tuplesByPath) + private static void CreateTuplesFromFiles(IEnumerable files, Dictionary tuplesByPath) { foreach (var info in files) { var tuple = new StateTuple(info); //Is this the target of a move event? - var moveArg = + /*var moveArg = moves.Values.FirstOrDefault( arg => info.FullName.Equals(arg.FullPath, StringComparison.InvariantCultureIgnoreCase) || info.FullName.IsAtOrBelow(arg.FullPath)); @@ -173,13 +181,13 @@ namespace Pithos.Core.Agents tuple.OldChecksum = states.FirstOrDefault( st => st.FilePath.Equals(tuple.OldFullPath, StringComparison.InvariantCultureIgnoreCase)) .NullSafe(st => st.Checksum); - } + }*/ tuplesByPath[tuple.FilePath] = tuple; } } - private void MergeLocalStates(IEnumerable states, ConcurrentDictionary moves, Dictionary tuplesByPath) + private void MergeLocalStates(IEnumerable states, Dictionary tuplesByPath) { //For files that have state foreach (var state in states) @@ -192,19 +200,19 @@ namespace Pithos.Core.Agents hashTuple.FileState = state; UpdateHashes(hashTuple); } - else if (moves.ContainsKey(state.FilePath) && + /* else if (moves.ContainsKey(state.FilePath) && tuplesByPath.TryGetValue(moves[state.FilePath].FullPath, out hashTuple)) { hashTuple.FileState = state; UpdateHashes(hashTuple); - } + }*/ else { var fsInfo = FileInfoExtensions.FromPath(state.FilePath); hashTuple = new StateTuple { FileInfo = fsInfo, FileState = state }; //Is the source of a moved item? - var moveArg = + /* var moveArg = moves.Values.FirstOrDefault( arg => state.FilePath.Equals(arg.OldFullPath, StringComparison.InvariantCultureIgnoreCase) || state.FilePath.IsAtOrBelow(arg.OldFullPath)); @@ -216,7 +224,7 @@ namespace Pithos.Core.Agents //Do we have the old MD5? //hashTuple.OldMD5 = state.LastMD5; } - +*/ tuplesByPath[state.FilePath] = hashTuple; } diff --git a/trunk/Pithos.Core/IStatusKeeper.cs b/trunk/Pithos.Core/IStatusKeeper.cs index 0e56537..f32e7c0 100644 --- a/trunk/Pithos.Core/IStatusKeeper.cs +++ b/trunk/Pithos.Core/IStatusKeeper.cs @@ -100,6 +100,7 @@ namespace Pithos.Core List GetAllStates(); List GetAllStatePaths(); List GetConflictStates(); + void MoveFileState(string oldFullPath, string newFullPath, ObjectInfo objectInfo, TreeHash treeHash); } [ContractClassFor(typeof(IStatusKeeper))] @@ -322,6 +323,23 @@ namespace Pithos.Core return null; } + public void MoveFileState(string oldFullPath, string newFullPath, ObjectInfo objectInfo, TreeHash treeHash) + { + if (String.IsNullOrWhiteSpace(oldFullPath)) + throw new ArgumentNullException("oldFullPath"); + if (!Path.IsPathRooted(oldFullPath)) + throw new ArgumentException("The path must be rooted", "oldFullPath"); + if (String.IsNullOrWhiteSpace(newFullPath)) + throw new ArgumentNullException("newFullPath"); + if (!Path.IsPathRooted(newFullPath)) + throw new ArgumentException("The path must be rooted", "newFullPath"); + if (treeHash == null) + throw new ArgumentNullException("treeHash"); + if (objectInfo == null) + throw new ArgumentNullException("objectInfo", "objectInfo can't be empty"); + Contract.EndContractBlock(); + } + //public void UpdateLastMD5(FileInfo path, string etag) //{ //} -- 1.7.10.4