From: pkanavos Date: Tue, 5 Jun 2012 20:42:53 +0000 (+0300) Subject: Added server move detection when hashes match or downloading X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/commitdiff_plain/159098f7c366cf56938db5068d85f265d247a1cd Added server move detection when hashes match or downloading --- diff --git a/trunk/Pithos.Core/Agents/PollAgent.cs b/trunk/Pithos.Core/Agents/PollAgent.cs index ec33aac..14e5d15 100644 --- a/trunk/Pithos.Core/Agents/PollAgent.cs +++ b/trunk/Pithos.Core/Agents/PollAgent.cs @@ -503,6 +503,7 @@ namespace Pithos.Core.Agents //Don't use the tuple info, it may have been deleted var localInfo = FileInfoExtensions.FromPath(localFilePath); + // Local file unchanged? If both C and L are null, make sure it's because //both the file is missing and the state checksum is not missing if (tuple.C == tuple.L && (localInfo.Exists || tuple.FileState==null)) @@ -512,7 +513,8 @@ namespace Pithos.Core.Agents if (tuple.S == tuple.L) { // No server changes - ; + //Has the file been renamed on the server? + MoveForServerMove(accountInfo, tuple); } else { @@ -534,17 +536,24 @@ namespace Pithos.Core.Agents { //Server file exists //downloadServerObject() // Result: L = S - StatusKeeper.SetFileState(localFilePath, FileStatus.Modified, + //If the file has moved on the server, move it locally before downloading + var targetPath=MoveForServerMove(accountInfo,tuple); + + StatusKeeper.SetFileState(targetPath, FileStatus.Modified, FileOverlayStatus.Modified, ""); NetworkAgent.Downloader.DownloadCloudFile(accountInfo, tuple.ObjectInfo, - localFilePath, token).Wait(token); + targetPath, token).Wait(token); //updateRecord( L = S ) - StatusKeeper.UpdateFileChecksum(localFilePath, tuple.ObjectInfo.ETag, + StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag, tuple.ObjectInfo.X_Object_Hash); - StatusKeeper.SetFileState(localFilePath, FileStatus.Unchanged, + StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo); + +/* + StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged, FileOverlayStatus.Normal, ""); +*/ } } } @@ -576,10 +585,9 @@ namespace Pithos.Core.Agents "Poll", isUnselected); NetworkAgent.Uploader.UploadCloudFile(action, token).Wait(token); - //updateRecord( S = C ) - StatusKeeper.SetFileState(localFilePath, FileStatus.Unchanged, - FileOverlayStatus.Normal, ""); + //State updated by the uploader + if (isUnselected) { ProcessChildren(accountInfo, tuple, agent, token); @@ -619,6 +627,30 @@ namespace Pithos.Core.Agents } } + private string MoveForServerMove(AccountInfo accountInfo, StateTuple tuple) + { + var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName); + var serverPath = Path.Combine(accountInfo.AccountPath, relativePath); + + if (tuple.FilePath == serverPath) return serverPath; + + if (tuple.FileInfo.Exists) + { + var fi = tuple.FileInfo as FileInfo; + if (fi != null) + fi.MoveTo(serverPath); + var di = tuple.FileInfo as DirectoryInfo; + if (di != null) + di.MoveTo(serverPath); + StatusKeeper.StoreInfo(serverPath, tuple.ObjectInfo); + } + else + { + Debug.Assert(false, "File does not exist"); + } + return serverPath; + } + private void DeleteCloudFile(AccountInfo accountInfo, StateTuple tuple) { StatusKeeper.SetFileState(tuple.FilePath, FileStatus.Deleted, @@ -647,43 +679,56 @@ namespace Pithos.Core.Agents IEnumerable> files, IEnumerable states) { - var dct = new Dictionary(); + var tuplesByPath = new Dictionary(); foreach (var file in files) { var fsInfo = file.Item1; var fileHash = fsInfo is DirectoryInfo? MERKLE_EMPTY:file.Item2; - dct[fsInfo.FullName] = new StateTuple {FileInfo = fsInfo, MD5 = fileHash}; + tuplesByPath[fsInfo.FullName] = new StateTuple {FileInfo = fsInfo, MD5 = fileHash}; } foreach (var state in states) { StateTuple hashTuple; - if (dct.TryGetValue(state.FilePath, out hashTuple)) + if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple)) { hashTuple.FileState = state; } else { var fsInfo = FileInfoExtensions.FromPath(state.FilePath); - dct[state.FilePath] = new StateTuple {FileInfo = fsInfo, FileState = state}; + tuplesByPath[state.FilePath] = new StateTuple {FileInfo = fsInfo, FileState = state}; } } + + var tuplesByID = tuplesByPath.Values + .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null) + .ToDictionary(tuple=>tuple.FileState.ObjectID,tuple=>tuple);//new Dictionary(); + foreach (var info in infos) { StateTuple hashTuple; var filePath = info.Item1; var objectInfo = info.Item2; - if (dct.TryGetValue(filePath, out hashTuple)) + var objectID = objectInfo.UUID; + + if (tuplesByID.TryGetValue(objectID, out hashTuple)) + { + hashTuple.ObjectInfo = objectInfo; + } + else if (tuplesByPath.TryGetValue(filePath, out hashTuple)) { hashTuple.ObjectInfo = objectInfo; } else { var fsInfo = FileInfoExtensions.FromPath(filePath); - dct[filePath] = new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo}; + var tuple = new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo}; + tuplesByPath[filePath] = tuple; + tuplesByID[objectInfo.UUID] = tuple; } } - return dct.Values; + return tuplesByPath.Values; } /// diff --git a/trunk/Pithos.Core/Agents/StatusAgent.cs b/trunk/Pithos.Core/Agents/StatusAgent.cs index b95645d..4a38675 100644 --- a/trunk/Pithos.Core/Agents/StatusAgent.cs +++ b/trunk/Pithos.Core/Agents/StatusAgent.cs @@ -661,9 +661,14 @@ namespace Pithos.Core.Agents using (var connection = GetConnection()) using (var command = new SQLiteCommand(connection)) { - if (StateExists(path, connection)) + //If the ID exists, update the status + if (StateExistsByID(objectInfo.UUID,connection)) command.CommandText = - "update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE "; + "update FileState set FilePath=:path,FileStatus= :fileStatus, Checksum=:checksum, ShortHash=:shortHash,Version=:version,VersionTimeStamp=:versionTimeStamp where ObjectID = :objectID "; + else if (StateExists(path, connection)) + //If the ID doesn't exist, try to update using the path, and store the ID as well. + command.CommandText = + "update FileState set FileStatus= :fileStatus, ObjectID=:objectID, Checksum=:checksum, ShortHash=:shortHash,Version=:version,VersionTimeStamp=:versionTimeStamp where FilePath = :path COLLATE NOCASE "; else { command.CommandText = @@ -702,6 +707,17 @@ namespace Pithos.Core.Agents } + private bool StateExistsByID(string objectId,SQLiteConnection connection) + { + using (var command = new SQLiteCommand("Select count(*) from FileState where ObjectId=:id", connection)) + { + command.Parameters.AddWithValue("id", objectId); + var result = command.ExecuteScalar(); + return ((long)result >= 1); + } + + } + public void SetFileStatus(string path, FileStatus status) { if (String.IsNullOrWhiteSpace(path)) diff --git a/trunk/Pithos.Core/Agents/Uploader.cs b/trunk/Pithos.Core/Agents/Uploader.cs index 136bbbf..ea5f429 100644 --- a/trunk/Pithos.Core/Agents/Uploader.cs +++ b/trunk/Pithos.Core/Agents/Uploader.cs @@ -183,8 +183,15 @@ namespace Pithos.Core.Agents await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash,cancellationToken); } - //If everything succeeds, change the file and overlay status to normal - StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, ""); + var currentInfo=client.GetObjectInfo(cloudFile.Account, cloudFile.Container, cloudFile.Name); + //If there is no stored ObjectID in the file state, add it + if (action.FileState == null || action.FileState.ObjectID == null) + { + StatusKeeper.StoreInfo(fullFileName,currentInfo); + } + else + //If everything succeeds, change the file and overlay status to normal + StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, ""); } catch (WebException exc) { diff --git a/trunk/Pithos.Core/FileState.cs b/trunk/Pithos.Core/FileState.cs index 5e2ef27..833881c 100644 --- a/trunk/Pithos.Core/FileState.cs +++ b/trunk/Pithos.Core/FileState.cs @@ -73,6 +73,7 @@ namespace Pithos.Core //[Property(Unique = true, UniqueKey = "IX_FileState_ObjectID")] + [Property] public string ObjectID { get; set; } [Property(Unique = true, UniqueKey = "IX_FileState_FilePath")] diff --git a/trunk/Pithos.Network/CloudFilesClient.cs b/trunk/Pithos.Network/CloudFilesClient.cs index 61f1641..a1e0b99 100644 --- a/trunk/Pithos.Network/CloudFilesClient.cs +++ b/trunk/Pithos.Network/CloudFilesClient.cs @@ -789,6 +789,7 @@ namespace Pithos.Network Container = container, Name = objectName, ETag = client.GetHeaderValue("ETag"), + UUID=client.GetHeaderValue("X-Object-UUID"), X_Object_Hash = client.GetHeaderValue("X-Object-Hash"), Content_Type = client.GetHeaderValue("Content-Type"), Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),