X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/blobdiff_plain/bc9ed56662ed1f57e1d84f193b579d16a3f36eb9..88e2300a132f903fe892721360f12ec0ebb0824a:/trunk/Pithos.Core/Agents/PollAgent.cs diff --git a/trunk/Pithos.Core/Agents/PollAgent.cs b/trunk/Pithos.Core/Agents/PollAgent.cs index 0c0dad9..6d5d89b 100644 --- a/trunk/Pithos.Core/Agents/PollAgent.cs +++ b/trunk/Pithos.Core/Agents/PollAgent.cs @@ -370,8 +370,12 @@ namespace Pithos.Core.Agents select Task>.Factory.StartNew(_ => client.ListObjects(accountInfo.UserName, container.Name, since), container.Name,token)).ToList(); - var listShared = Task>.Factory.StartNew(_ => - client.ListSharedObjects(_knownContainers,since), "shared",token); + var selectiveEnabled = Selectives.IsSelectiveEnabled(accountInfo.AccountKey); + var listShared = selectiveEnabled? + Task>.Factory.StartNew(_ => + client.ListSharedObjects(_knownContainers,since), "shared",token) + :Task.Factory.FromResult((IList) new List(),"shared"); + listObjects.Add(listShared); var listTasks = await Task.Factory.WhenAll(listObjects.ToArray()).ConfigureAwait(false); @@ -446,16 +450,21 @@ namespace Pithos.Core.Agents var files = LoadLocalFileTuples(accountInfo, accountBatch); - var states = StatusKeeper.GetAllStates(); - + + //WARNING: GetFileSystemInfo may create state entries. + //TODO: Find a different way to create the tuples and block long filenames var infos = (from remote in currentRemotes let path = remote.RelativeUrlToFilePath(accountInfo.UserName) let info=agent.GetFileSystemInfo(path) + where info != null select Tuple.Create(info.FullName,remote)) .ToList(); + var states = StatusKeeper.GetAllStates(); + + var tupleBuilder = new TupleBuilder(CancellationToken,StatusKeeper,StatusNotification,Settings); - var tuples = MergeSources(infos, files, states,moves).ToList(); + var tuples = tupleBuilder.MergeSources(infos, files, states,moves).ToList(); var processedPaths = new HashSet(); //Process only the changes in the batch file, if one exists @@ -761,7 +770,7 @@ namespace Pithos.Core.Agents var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState, accountInfo.BlockSize, accountInfo.BlockHash, "Poll", isUnselectedRootFolder, token, progress,tuple.Merkle); - + using (StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}",true, localInfo.Name)) { @@ -787,9 +796,6 @@ namespace Pithos.Core.Agents private async Task MoveForLocalMove(AccountInfo accountInfo, StateTuple tuple) { - //Is the file a directory or previous path missing? - if (tuple.FileInfo is DirectoryInfo) - return false; //Is the previous path missing? if (String.IsNullOrWhiteSpace(tuple.OldFullPath)) return false; @@ -811,14 +817,18 @@ namespace Pithos.Core.Agents var client = new CloudFilesClient(accountInfo); var objectInfo = CloudAction.CreateObjectInfoFor(accountInfo, tuple.FileInfo); + objectInfo.X_Object_Hash = tuple.Merkle.TopHash.ToHashString(); var containerPath = Path.Combine(accountInfo.AccountPath, objectInfo.Container.ToUnescapedString()); //TODO: SImplify these multiple conversions from and to Uris var oldName = tuple.OldFullPath.AsRelativeTo(containerPath); //Then execute a move instead of an upload using (StatusNotification.GetNotifier("Moving {0}", "Moved {0}", true,tuple.FileInfo.Name)) { - await client.MoveObject(objectInfo.Account, objectInfo.Container, oldName.ToEscapedUri(), + await client.MoveObject(objectInfo.Account, objectInfo.Container, oldName.Replace('\\','/').ToEscapedUri(), objectInfo.Container, objectInfo.Name).ConfigureAwait(false); + StatusKeeper.MoveFileState(tuple.OldFullPath, tuple.FilePath, objectInfo, tuple.Merkle); + //StatusKeeper.StoreInfo(tuple.FilePath,objectInfo,tuple.Merkle); + //StatusKeeper.ClearFolderStatus(tuple.FilePath); } return true; } @@ -948,223 +958,9 @@ namespace Pithos.Core.Agents - private IEnumerable MergeSources(IEnumerable> infos, IEnumerable files, List states, ConcurrentDictionary moves) - { - var tuplesByPath = new Dictionary(); - foreach (var info in files) - { - var tuple = new StateTuple(info); - //Is this the target of a move event? - var moveArg = moves.Values.FirstOrDefault(arg => info.FullName.Equals(arg.FullPath, StringComparison.InvariantCultureIgnoreCase) - || info.FullName.IsAtOrBelow(arg.FullPath)); - if (moveArg != null) - { - tuple.NewFullPath = info.FullName; - var relativePath = info.AsRelativeTo(moveArg.FullPath); - tuple.OldFullPath = Path.Combine(moveArg.OldFullPath, relativePath); - tuple.OldChecksum = states.FirstOrDefault(st => st.FilePath.Equals(tuple.OldFullPath, StringComparison.InvariantCultureIgnoreCase)) - .NullSafe(st => st.Checksum); - } - - tuplesByPath[tuple.FilePath] = tuple; - } - - - - - //For files that have state - foreach (var state in states) - { - StateTuple hashTuple; - - - if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple)) - { - hashTuple.FileState = state; - UpdateHashes(hashTuple); - } - else if (moves.ContainsKey(state.FilePath) && tuplesByPath.TryGetValue(moves[state.FilePath].FullPath, out hashTuple)) - { - hashTuple.FileState = state; - UpdateHashes(hashTuple); - } - else - { - var fsInfo = FileInfoExtensions.FromPath(state.FilePath); - hashTuple = new StateTuple {FileInfo = fsInfo, FileState = state}; - - //Is the source of a moved item? - var moveArg = moves.Values.FirstOrDefault(arg => state.FilePath.Equals(arg.OldFullPath,StringComparison.InvariantCultureIgnoreCase) - || state.FilePath.IsAtOrBelow(arg.OldFullPath)); - if (moveArg != null) - { - var relativePath = state.FilePath.AsRelativeTo(moveArg.OldFullPath); - hashTuple.NewFullPath = Path.Combine(moveArg.FullPath,relativePath); - hashTuple.OldFullPath = state.FilePath; - //Do we have the old MD5? - //hashTuple.OldMD5 = state.LastMD5; - } - - - tuplesByPath[state.FilePath] = hashTuple; - } - } - //for files that don't have state - var statelessTuples = tuplesByPath.Values.Where(t => t.FileState == null).ToArray(); - //If there are too many stateless tuples, update them in parallel - if (statelessTuples.Length > 20) - Parallel.ForEach(statelessTuples, UpdateHashes); - else - statelessTuples.ApplyAction(UpdateHashes); - - var tuplesByID = tuplesByPath.Values - .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null) - .ToDictionary(tuple=>tuple.FileState.ObjectID,tuple=>tuple);//new Dictionary(); - - foreach (var info in infos) - { - StateTuple hashTuple; - var filePath = info.Item1; - var objectInfo = info.Item2; - var objectID = objectInfo.UUID; - - if (objectID != _emptyGuid && tuplesByID.TryGetValue(objectID, out hashTuple)) - { - hashTuple.ObjectInfo = objectInfo; - } - else if (tuplesByPath.TryGetValue(filePath, out hashTuple)) - { - hashTuple.ObjectInfo = objectInfo; - } - else - { - - - var fsInfo = FileInfoExtensions.FromPath(filePath); - hashTuple= new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo}; - tuplesByPath[filePath] = hashTuple; - - if (objectInfo.UUID!=_emptyGuid) - tuplesByID[objectInfo.UUID] = hashTuple; - } - } - - var tuples = tuplesByPath.Values; - //Sync algorithm fallout: There are multiple ways we can reach a situation where a state without a checksum exists - //1: The application stopped/crashed while downloading a file. The file's entry was created when the download started. When the application restarts, - // it finds no local file, a server file and a null state -> C: NULL L: NULL but exists, S: Some - // It can be fixed by NOT creating a local state if the file doesn't already exist, or adding extra info to mark this as a result of an upload - //2: A new file is added but the app stops/crashes after uploading finishes but before the entry gets updated and the user deletes the file. The file's entry was created. When the app restarts, - // it finds no local file, a server file and a null state -> C: NULL L: NULL but exists, S: Some - // - - var brokenTuples = from tuple in tuples - where tuple.FileState != null && tuple.FileState.Checksum == null - && tuple.ObjectInfo != null && (tuple.FileInfo==null || !tuple.FileInfo.Exists) - select tuple; - - - var actualTuples = tuples.Except(brokenTuples); - Debug.Assert(actualTuples.All(t => t.HashesValid())); - - foreach (var tuple in brokenTuples) - { - StatusKeeper.SetFileState(tuple.FileState.FilePath, - FileStatus.Conflict, FileOverlayStatus.Conflict, "FileState without checksum encountered for server object missing from disk"); - } - - return actualTuples; - } - /// - /// Update the tuple with the file's hashes, avoiding calculation if the file is unchanged - /// - /// - /// - /// The function first checks the file's size and last write date to see if there are any changes. If there are none, - /// the file's stored hashes are used. - /// Otherwise, MD5 is calculated first to ensure there are no changes. If MD5 is different, the Merkle hash is calculated - /// - private void UpdateHashes(StateTuple hashTuple) - { - - try - { - var state = hashTuple.NullSafe(s => s.FileState); - var storedHash = state.NullSafe(s => s.Checksum); - var storedHashes = state.NullSafe(s => s.Hashes); - //var storedMD5 = state.NullSafe(s => s.LastMD5); - var storedDate = state.NullSafe(s => s.LastWriteDate) ?? DateTime.MinValue; - var storedLength = state.NullSafe(s => s.LastLength); - - //var md5Hash = Signature.MD5_EMPTY; - var merkle=TreeHash.Empty; - if (hashTuple.FileInfo is FileInfo) - { - var file = (FileInfo)hashTuple.FileInfo.WithProperCapitalization(); - - //Attributes unchanged? - //LastWriteTime is only accurate to the second - var unchangedAttributes = file.LastWriteTime - storedDate < TimeSpan.FromSeconds(1) - && storedLength == file.Length; - - //Attributes appear unchanged but the file length doesn't match the stored hash ? - var nonEmptyMismatch = unchangedAttributes && - (file.Length == 0 ^ storedHash== Signature.MERKLE_EMPTY); - - //Missing hashes for NON-EMPTY hash ? - var missingHashes = storedHash != Signature.MERKLE_EMPTY && - String.IsNullOrWhiteSpace(storedHashes); - - //Unchanged attributes but changed MD5 - //Short-circuiting ensures MD5 is computed only if the attributes are changed - - //var md5Mismatch = (!unchangedAttributes && file.ComputeShortHash(StatusNotification) != storedMD5); - - - //If the attributes are unchanged but the Merkle doesn't match the size, - //or the attributes and the MD5 hash have changed, - //or the hashes are missing but the tophash is NOT empty, we need to recalculate - // - //Otherwise we load the hashes from state - if (!unchangedAttributes || nonEmptyMismatch || missingHashes) - merkle = RecalculateTreehash(file); - else - { - merkle=TreeHash.Parse(hashTuple.FileState.Hashes); - //merkle.MD5 = storedMD5; - } - - - //md5Hash = merkle.MD5; - } - //hashTuple.MD5 = md5Hash; - //Setting Merkle also updates C - hashTuple.Merkle = merkle; - } - catch (IOException) - { - hashTuple.Locked = true; - } - } - - /// - /// Recalculate a file's treehash and md5 and update the database - /// - /// - /// - private TreeHash RecalculateTreehash(FileInfo file) - { - var progress = new Progress(d =>StatusNotification.Notify( - new StatusNotification(String.Format("Hashing {0:p} of {1}", d.Percentage, file.Name)))); - var merkle = Signature.CalculateTreeHash(file, StatusKeeper.BlockSize, StatusKeeper.BlockHash, - Settings.HashingParallelism, CancellationToken, progress); - - StatusKeeper.UpdateFileHashes(file, merkle); - return merkle; - } /// /// Returns the latest LastModified date from the list of objects, but only if it is before @@ -1206,7 +1002,7 @@ namespace Pithos.Core.Agents readonly AccountsDifferencer _differencer = new AccountsDifferencer(); private bool _pause; - private readonly string _emptyGuid = Guid.Empty.ToString(); +