Selective filtering modifications to allow uploading of new root folders
[pithos-ms-client] / trunk / Pithos.Core / Agents / PollAgent.cs
index a3ff53e..875606e 100644 (file)
@@ -45,7 +45,9 @@ using System.ComponentModel.Composition;
 using System.Diagnostics;\r
 using System.Diagnostics.Contracts;\r
 using System.IO;\r
+using System.Linq.Expressions;\r
 using System.Reflection;\r
+using System.Security.Cryptography;\r
 using System.Threading;\r
 using System.Threading.Tasks;\r
 using Castle.ActiveRecord;\r
@@ -59,6 +61,71 @@ namespace Pithos.Core.Agents
     using System.Collections.Generic;\r
     using System.Linq;\r
 \r
+    [DebuggerDisplay("{FilePath} C:{C} L:{L} S:{S}")]\r
+    public class StateTuple\r
+    {\r
+        public string FilePath { get; private set; }\r
+\r
+        public string MD5 { get; set; }\r
+\r
+        public string L\r
+        {\r
+            get { return FileState==null?null:FileState.Checksum; }\r
+        }\r
+\r
+        private string _c;\r
+        public string C\r
+        {\r
+            get { return _c; }\r
+            set {\r
+                _c = String.IsNullOrWhiteSpace(value) ? null : value;\r
+            }\r
+        }\r
+\r
+        public string S\r
+        {\r
+            get { return ObjectInfo == null ? null : ObjectInfo.X_Object_Hash; }\r
+        }\r
+\r
+        private FileSystemInfo _fileInfo;\r
+        private TreeHash _merkle;\r
+\r
+        public FileSystemInfo FileInfo\r
+        {\r
+            get { return _fileInfo; }\r
+            set\r
+            {\r
+                _fileInfo = value;\r
+                FilePath = value.FullName;\r
+            }\r
+        }\r
+\r
+        public FileState FileState { get; set; }\r
+        public ObjectInfo ObjectInfo{ get; set; }\r
+\r
+\r
+        public TreeHash Merkle\r
+        {\r
+            get {\r
+                return _merkle;\r
+            }\r
+            set {\r
+                _merkle = value;\r
+                C = _merkle.TopHash.ToHashString();\r
+            }\r
+        }\r
+\r
+        public StateTuple() { }\r
+\r
+        public StateTuple(FileSystemInfo info)\r
+        {\r
+            FileInfo = info;\r
+        }\r
+\r
+\r
+    }\r
+\r
+\r
     /// <summary>\r
     /// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all\r
     /// objects and compares it with a previously cached version to detect differences. \r
@@ -84,6 +151,29 @@ namespace Pithos.Core.Agents
 \r
         public IStatusNotification StatusNotification { get; set; }\r
 \r
+        private CancellationTokenSource _currentOperationCancellation = new CancellationTokenSource();\r
+\r
+        public void CancelCurrentOperation()\r
+        {\r
+            //What does it mean to cancel the current upload/download?\r
+            //Obviously, the current operation will be cancelled by throwing\r
+            //a cancellation exception.\r
+            //\r
+            //The default behavior is to retry any operations that throw.\r
+            //Obviously this is not what we want in this situation.\r
+            //The cancelled operation should NOT bea retried. \r
+            //\r
+            //This can be done by catching the cancellation exception\r
+            //and avoiding the retry.\r
+            //\r
+\r
+            //Have to reset the cancellation source - it is not possible to reset the source\r
+            //Have to prevent a case where an operation requests a token from the old source\r
+            var oldSource = Interlocked.Exchange(ref _currentOperationCancellation, new CancellationTokenSource());\r
+            oldSource.Cancel();\r
+\r
+        }\r
+\r
         public bool Pause\r
         {\r
             get {\r
@@ -114,11 +204,13 @@ namespace Pithos.Core.Agents
         /// <summary>\r
         /// Start a manual synchronization\r
         /// </summary>\r
-        public void SynchNow()\r
+        public void SynchNow(IEnumerable<string> paths=null)\r
         {            \r
+            _batchQueue.Enqueue(paths);\r
             _syncEvent.Set();\r
         }\r
 \r
+        readonly ConcurrentQueue<IEnumerable<string>> _batchQueue=new ConcurrentQueue<IEnumerable<string>>();\r
 \r
         /// <summary>\r
         /// Remote files are polled periodically. Any changes are processed\r
@@ -132,7 +224,7 @@ namespace Pithos.Core.Agents
 \r
             Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
 \r
-            GC.Collect();\r
+            //GC.Collect();\r
 \r
             using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
             {\r
@@ -143,8 +235,23 @@ namespace Pithos.Core.Agents
                     await _unPauseEvent.WaitAsync();\r
                     UpdateStatus(PithosStatus.PollSyncing);\r
 \r
-                    var tasks = from accountInfo in _accounts.Values\r
-                                select ProcessAccountFiles(accountInfo, since);\r
+                    var accountBatches=new Dictionary<Uri, IEnumerable<string>>();\r
+                    IEnumerable<string> batch = null;\r
+                    if (_batchQueue.TryDequeue(out batch) && batch != null)\r
+                        foreach (var account in _accounts.Values)\r
+                        {\r
+                            var accountBatch = batch.Where(path => path.IsAtOrBelow(account.AccountPath));\r
+                            accountBatches[account.AccountKey] = accountBatch;\r
+                        }\r
+\r
+\r
+                    IEnumerable<Task<DateTime?>> tasks = new List<Task<DateTime?>>();\r
+                    foreach(var accountInfo in _accounts.Values)\r
+                    {\r
+                        IEnumerable<string> accountBatch ;\r
+                        accountBatches.TryGetValue(accountInfo.AccountKey,out accountBatch);\r
+                        ProcessAccountFiles (accountInfo, accountBatch, since);\r
+                    }\r
 \r
                     var nextTimes=await TaskEx.WhenAll(tasks.ToList());\r
 \r
@@ -187,7 +294,7 @@ namespace Pithos.Core.Agents
         private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
         {\r
             var sync = _syncEvent.WaitAsync();\r
-            var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval), NetworkAgent.CancellationToken);\r
+            var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval));\r
             \r
             var signaledTask = await TaskEx.WhenAny(sync, wait);\r
             \r
@@ -208,7 +315,7 @@ namespace Pithos.Core.Agents
             return since;\r
         }\r
 \r
-        public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, DateTime? since = null)\r
+        public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, IEnumerable<string> accountBatch, DateTime? since = null)\r
         {\r
             if (accountInfo == null)\r
                 throw new ArgumentNullException("accountInfo");\r
@@ -264,6 +371,7 @@ namespace Pithos.Core.Agents
                         var remoteObjects = (from objectList in listTasks\r
                                             where (string)objectList.AsyncState != "trash"\r
                                             from obj in objectList.Result\r
+                                            orderby obj.Bytes ascending \r
                                             select obj).ToList();\r
                         \r
                         //Get the latest remote object modification date, only if it is after\r
@@ -285,7 +393,7 @@ namespace Pithos.Core.Agents
                                         where\r
                                             !remoteObjects.Any(\r
                                                 info => info.Name == trash.Name && info.Hash == trash.Hash)\r
-                                        select trash;\r
+                                   8     select trash;\r
                         ProcessTrashedFiles(accountInfo, realTrash);\r
 */\r
 \r
@@ -300,42 +408,52 @@ namespace Pithos.Core.Agents
                             StatusKeeper.CleanupOrphanStates();\r
                         StatusKeeper.CleanupStaleStates(accountInfo, cleanRemotes);\r
                         \r
-                        var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
+                        //var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
 \r
-                        var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
+                        //var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
 \r
 \r
-                        //On the first run\r
-                        if (_firstPoll)\r
-                        {\r
-                            MarkSuspectedDeletes(accountInfo, cleanRemotes);\r
-                        }\r
-                        ProcessDeletedFiles(accountInfo, differencer.Deleted.FilterDirectlyBelow(filterUris));\r
+                        //Get the local files here                        \r
+                        var agent = AgentLocator<FileAgent>.Get(accountInfo.AccountPath);\r
 \r
-                        // @@@ NEED To add previous state here as well, To compare with previous hash\r
+                        var files = LoadLocalFileTuples(accountInfo);\r
 \r
+                        var states = FileState.Queryable.ToList();                        \r
                         \r
 \r
-                        //Create a list of actions from the remote files\r
-                        \r
-                        var allActions = MovesToActions(accountInfo,differencer.Moved.FilterDirectlyBelow(filterUris))\r
-                                        .Union(\r
-                                        ChangesToActions(accountInfo, differencer.Changed.FilterDirectlyBelow(filterUris)))\r
-                                        .Union(\r
-                                        CreatesToActions(accountInfo, differencer.Created.FilterDirectlyBelow(filterUris)));\r
-\r
-                        //And remove those that are already being processed by the agent\r
-                        var distinctActions = allActions\r
-                            .Except(NetworkAgent.GetEnumerable(), new LocalFileComparer())\r
-                            .ToList();\r
-\r
-                        await _unPauseEvent.WaitAsync();\r
-                        //Queue all the actions\r
-                        foreach (var message in distinctActions)\r
+                        var infos = (from remote in cleanRemotes\r
+                                    let path = remote.RelativeUrlToFilePath(accountInfo.UserName)\r
+                                    let info=agent.GetFileSystemInfo(path)\r
+                                    select Tuple.Create(info.FullName,remote))\r
+                                    .ToList();\r
+\r
+                        var token = _currentOperationCancellation.Token;\r
+\r
+                        var tuples = MergeSources(infos, files, states).ToList();\r
+\r
+\r
+                        var stateTuples = accountBatch==null?tuples:tuples.Where(t => accountBatch.Contains(t.FilePath));\r
+                        foreach (var tuple in stateTuples)\r
                         {\r
-                            NetworkAgent.Post(message);\r
+                            await _unPauseEvent.WaitAsync();\r
+\r
+                            //Set the Merkle Hash\r
+                            SetMerkleHash(accountInfo, tuple);\r
+\r
+                            SyncSingleItem(accountInfo, tuple, agent, token);\r
+\r
                         }\r
 \r
+\r
+                        //On the first run\r
+/*\r
+                        if (_firstPoll)\r
+                        {\r
+                            MarkSuspectedDeletes(accountInfo, cleanRemotes);\r
+                        }\r
+*/\r
+\r
+\r
                         Log.Info("[LISTENER] End Processing");\r
                     }\r
                 }\r
@@ -350,6 +468,318 @@ namespace Pithos.Core.Agents
             }\r
         }\r
 \r
+        private static void SetMerkleHash(AccountInfo accountInfo, StateTuple tuple)\r
+        {\r
+            //The Merkle hash for directories is that of an empty buffer\r
+            if (tuple.FileInfo is DirectoryInfo)\r
+                tuple.C = MERKLE_EMPTY;\r
+            else if (tuple.FileState != null && tuple.MD5 == tuple.FileState.ShortHash)\r
+            {\r
+                //If there is a state whose MD5 matches, load the merkle hash from the file state\r
+                //insteaf of calculating it\r
+                tuple.C = tuple.FileState.Checksum;                              \r
+            }\r
+            else\r
+            {\r
+                tuple.Merkle = Signature.CalculateTreeHash(tuple.FileInfo, accountInfo.BlockSize, accountInfo.BlockHash);\r
+                //tuple.C=tuple.Merkle.TopHash.ToHashString();                \r
+            }\r
+        }\r
+\r
+        private static List<Tuple<FileSystemInfo, string>> LoadLocalFileTuples(AccountInfo accountInfo)\r
+        {\r
+            using (ThreadContext.Stacks["Account Files Hashing"].Push(accountInfo.UserName))\r
+            {\r
+\r
+                var localInfos = AgentLocator<FileAgent>.Get(accountInfo.AccountPath).EnumerateFileSystemInfos();\r
+                //Use the queue to retry locked file hashing\r
+                var fileQueue = new Queue<FileSystemInfo>(localInfos);\r
+                var hasher = MD5.Create();\r
+\r
+                var results = new List<Tuple<FileSystemInfo, string>>();\r
+\r
+                while (fileQueue.Count > 0)\r
+                {\r
+                    var file = fileQueue.Dequeue();\r
+                    using (ThreadContext.Stacks["File"].Push(file.FullName))\r
+                    {\r
+                        /*\r
+                                                Signature.CalculateTreeHash(file, accountInfo.BlockSize,\r
+                                                                                                 accountInfo.BlockHash).\r
+                                                                         TopHash.ToHashString()\r
+                        */\r
+                        try\r
+                        {\r
+                            //Replace MD5 here, do the calc while syncing individual files\r
+                            string hash ;\r
+                            if (file is DirectoryInfo)\r
+                                hash = MERKLE_EMPTY;\r
+                            else\r
+                            {\r
+                                using (var stream = (file as FileInfo).OpenRead())\r
+                                {\r
+                                    hash = hasher.ComputeHash(stream).ToHashString();\r
+                                }\r
+                            }                            \r
+                            results.Add(Tuple.Create(file, hash));\r
+                        }\r
+                        catch (IOException exc)\r
+                        {\r
+                            Log.WarnFormat("[HASH] File in use, will retry [{0}]", exc);\r
+                            fileQueue.Enqueue(file);\r
+                        }\r
+                    }\r
+                }\r
+\r
+                return results;\r
+            }\r
+        }\r
+\r
+        private void SyncSingleItem(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, CancellationToken token)\r
+        {\r
+            Log.DebugFormat("Sync [{0}] C:[{1}] L:[{2}] S:[{3}]",tuple.FilePath,tuple.C,tuple.L,tuple.S);\r
+\r
+            var localFilePath = tuple.FilePath;\r
+            //Don't use the tuple info, it may have been deleted\r
+            var localInfo = FileInfoExtensions.FromPath(localFilePath);\r
+\r
+\r
+            var isUnselectedRootFolder = agent.IsUnselectedRootFolder(tuple.FilePath);\r
+\r
+            //Unselected root folders that have not yet been uploaded should be uploaded and added to the \r
+            //selective folders\r
+\r
+            if (!Selectives.IsSelected(accountInfo, localFilePath) && !(isUnselectedRootFolder && tuple.ObjectInfo==null) )                \r
+                return;\r
+\r
+            // Local file unchanged? If both C and L are null, make sure it's because \r
+            //both the file is missing and the state checksum is not missing\r
+            if (tuple.C == tuple.L /*&& (localInfo.Exists || tuple.FileState == null)*/)\r
+            {\r
+                //No local changes\r
+                //Server unchanged?\r
+                if (tuple.S == tuple.L)\r
+                {\r
+                    // No server changes\r
+                    //Has the file been renamed on the server?\r
+                    MoveForServerMove(accountInfo, tuple);\r
+                }\r
+                else\r
+                {\r
+                    //Different from server\r
+                    //Does the server file exist?\r
+                    if (tuple.S == null)\r
+                    {\r
+                        //Server file doesn't exist\r
+                        //deleteObjectFromLocal()\r
+                        StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
+                                                  FileOverlayStatus.Deleted, "");\r
+                        agent.Delete(localFilePath);\r
+                        //updateRecord(Remove C, L)\r
+                        StatusKeeper.ClearFileStatus(localFilePath);\r
+                    }\r
+                    else\r
+                    {\r
+                        //Server file exists\r
+                        //downloadServerObject() // Result: L = S\r
+                        //If the file has moved on the server, move it locally before downloading\r
+                        var targetPath = MoveForServerMove(accountInfo, tuple);\r
+\r
+                        StatusKeeper.SetFileState(targetPath, FileStatus.Modified,\r
+                                                  FileOverlayStatus.Modified, "");\r
+                        NetworkAgent.Downloader.DownloadCloudFile(accountInfo,\r
+                                                                  tuple.ObjectInfo,\r
+                                                                  targetPath, tuple.Merkle, token).Wait(token);\r
+                        //updateRecord( L = S )\r
+                        StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
+                                                        tuple.ObjectInfo.X_Object_Hash);\r
+\r
+                        StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
+\r
+                        /*\r
+                                                        StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
+                                                                                  FileOverlayStatus.Normal, "");\r
+                            */\r
+                    }\r
+                }\r
+\r
+            }\r
+            else\r
+            {\r
+                //Local changes found\r
+\r
+                //Server unchanged?\r
+                if (tuple.S == tuple.L)\r
+                {\r
+                    //The FileAgent selective sync checks for new root folder files\r
+                    if (!agent.Ignore(localFilePath))\r
+                    {\r
+                        if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
+                        {\r
+                            //deleteObjectFromServer()\r
+                            DeleteCloudFile(accountInfo, tuple);\r
+                            //updateRecord( Remove L, S)                  \r
+                        }\r
+                        else\r
+                        {\r
+                            //uploadLocalObject() // Result: S = C, L = S                        \r
+\r
+                            //Debug.Assert(tuple.FileState !=null);\r
+                            var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
+                                                               accountInfo.BlockSize, accountInfo.BlockHash,\r
+                                                               "Poll", isUnselectedRootFolder);\r
+                            NetworkAgent.Uploader.UploadCloudFile(action, tuple.Merkle, token).Wait(token);\r
+\r
+                            //updateRecord( S = C )\r
+                            //State updated by the uploader\r
+\r
+                            if (isUnselectedRootFolder)\r
+                            {\r
+                                ProcessChildren(accountInfo, tuple, agent, token);\r
+                            }\r
+                        }\r
+                    }\r
+                }\r
+                else\r
+                {\r
+                    if (tuple.C == tuple.S)\r
+                    {\r
+                        // (Identical Changes) Result: L = S\r
+                        //doNothing()\r
+                        //Detect server moves\r
+                        var targetPath = MoveForServerMove(accountInfo, tuple);\r
+                        StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
+                    }\r
+                    else\r
+                    {\r
+                        if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
+                        {\r
+                            //deleteObjectFromServer()\r
+                            DeleteCloudFile(accountInfo, tuple);\r
+                            //updateRecord(Remove L, S)                  \r
+                        }\r
+                            //If both the local and server files are missing, the state is stale\r
+                        else if (!localInfo.Exists && (tuple.S == null || tuple.ObjectInfo == null))\r
+                        {\r
+                            StatusKeeper.ClearFileStatus(localInfo.FullName);\r
+                        }\r
+                        else\r
+                        {\r
+                            ReportConflictForMismatch(localFilePath);\r
+                            //identifyAsConflict() // Manual action required\r
+                        }\r
+                    }\r
+                }\r
+            }\r
+        }\r
+\r
+        private string MoveForServerMove(AccountInfo accountInfo, StateTuple tuple)\r
+        {\r
+            if (tuple.ObjectInfo == null)\r
+                return null;\r
+            var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
+            var serverPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
+            \r
+            //Compare Case Insensitive\r
+            if (String.Equals(tuple.FilePath ,serverPath,StringComparison.InvariantCultureIgnoreCase)) return serverPath;\r
+\r
+            if (tuple.FileInfo.Exists)\r
+            {                    \r
+                var fi = tuple.FileInfo as FileInfo;\r
+                if (fi != null)\r
+                    fi.MoveTo(serverPath);\r
+                var di = tuple.FileInfo as DirectoryInfo;\r
+                if (di != null)\r
+                    di.MoveTo(serverPath);\r
+                StatusKeeper.StoreInfo(serverPath, tuple.ObjectInfo);\r
+            }\r
+            else\r
+            {\r
+                Debug.Assert(false, "File does not exist");\r
+            }\r
+            return serverPath;\r
+        }\r
+\r
+        private void DeleteCloudFile(AccountInfo accountInfo, StateTuple tuple)\r
+        {\r
+            StatusKeeper.SetFileState(tuple.FilePath, FileStatus.Deleted,\r
+                                      FileOverlayStatus.Deleted, "");\r
+            NetworkAgent.DeleteAgent.DeleteCloudFile(accountInfo, tuple.ObjectInfo);\r
+            StatusKeeper.ClearFileStatus(tuple.FilePath);\r
+        }\r
+\r
+        private void ProcessChildren(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, CancellationToken token)\r
+        {\r
+\r
+            var dirInfo = tuple.FileInfo as DirectoryInfo;\r
+            var folderTuples = from folder in dirInfo.EnumerateDirectories("*", SearchOption.AllDirectories)\r
+                               select new StateTuple(folder);\r
+            var fileTuples = from file in dirInfo.EnumerateFiles("*", SearchOption.AllDirectories)\r
+                             select new StateTuple(file);\r
+            \r
+            //Process folders first, to ensure folders appear on the sever as soon as possible\r
+            folderTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, token));\r
+            \r
+            fileTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, token));\r
+        }\r
+\r
+        private static IEnumerable<StateTuple> MergeSources(\r
+            IEnumerable<Tuple<string, ObjectInfo>> infos, \r
+            IEnumerable<Tuple<FileSystemInfo, string>> files, \r
+            IEnumerable<FileState> states)\r
+        {\r
+            var tuplesByPath = new Dictionary<string, StateTuple>();\r
+            foreach (var file in files)\r
+            {\r
+                var fsInfo = file.Item1;\r
+                var fileHash = fsInfo is DirectoryInfo? MERKLE_EMPTY:file.Item2;\r
+\r
+                tuplesByPath[fsInfo.FullName] = new StateTuple {FileInfo = fsInfo, MD5 = fileHash};\r
+            }\r
+            foreach (var state in states)\r
+            {\r
+                StateTuple hashTuple;\r
+                if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple))\r
+                {\r
+                    hashTuple.FileState = state;\r
+                }\r
+                else\r
+                {\r
+                    var fsInfo = FileInfoExtensions.FromPath(state.FilePath);\r
+                    tuplesByPath[state.FilePath] = new StateTuple {FileInfo = fsInfo, FileState = state};\r
+                }\r
+            }\r
+\r
+            var tuplesByID = tuplesByPath.Values\r
+                .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null)\r
+                .ToDictionary(tuple=>tuple.FileState.ObjectID,tuple=>tuple);//new Dictionary<Guid, StateTuple>();\r
+\r
+            foreach (var info in infos)\r
+            {\r
+                StateTuple hashTuple;\r
+                var filePath = info.Item1;\r
+                var objectInfo = info.Item2;\r
+                var objectID = objectInfo.UUID;\r
+\r
+                if (tuplesByID.TryGetValue(objectID, out hashTuple))\r
+                {\r
+                    hashTuple.ObjectInfo = objectInfo;                    \r
+                }\r
+                else if (tuplesByPath.TryGetValue(filePath, out hashTuple))\r
+                {\r
+                    hashTuple.ObjectInfo = objectInfo;\r
+                }\r
+                else\r
+                {\r
+                    var fsInfo = FileInfoExtensions.FromPath(filePath);\r
+                    var tuple = new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo};\r
+                    tuplesByPath[filePath] = tuple;\r
+                    tuplesByID[objectInfo.UUID] = tuple;\r
+                }\r
+            }\r
+            return tuplesByPath.Values;\r
+        }\r
+\r
         /// <summary>\r
         /// Returns the latest LastModified date from the list of objects, but only if it is before\r
         /// than the threshold value\r
@@ -391,6 +821,7 @@ namespace Pithos.Core.Agents
         readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
         private Dictionary<Uri, List<Uri>> _selectiveUris = new Dictionary<Uri, List<Uri>>();\r
         private bool _pause;\r
+        private static string MERKLE_EMPTY = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";\r
 \r
         /// <summary>\r
         /// Deletes local files that are not found in the list of cloud files\r
@@ -484,6 +915,21 @@ namespace Pithos.Core.Agents
                                               TraceLevel.Info);\r
         }\r
 \r
+        private void ReportConflictForMismatch(string localFilePath)\r
+        {\r
+            if (String.IsNullOrWhiteSpace(localFilePath))\r
+                throw new ArgumentNullException("localFilePath");\r
+            Contract.EndContractBlock();\r
+\r
+            StatusKeeper.SetFileState(localFilePath, FileStatus.Conflict, FileOverlayStatus.Conflict, "File changed at the server");\r
+            UpdateStatus(PithosStatus.HasConflicts);\r
+            var message = String.Format("Conflict detected for file {0}", localFilePath);\r
+            Log.Warn(message);\r
+            StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
+        }\r
+\r
+\r
+\r
         /// <summary>\r
         /// Creates a Sync action for each changed server file\r
         /// </summary>\r
@@ -560,7 +1006,7 @@ namespace Pithos.Core.Agents
                                                      previousFile, objectInfo, state, accountInfo.BlockSize,\r
                                                      accountInfo.BlockHash,"Poll Moves");\r
                         //For modified files, we need to download the changes as well\r
-                        if (objectInfo.Hash!=objectInfo.PreviousHash)\r
+                        if (objectInfo.X_Object_Hash != objectInfo.PreviousHash)\r
                             yield return new CloudDownloadAction(accountInfo,objectInfo, "Poll Moves");\r
                     }\r
                 }\r
@@ -655,6 +1101,7 @@ namespace Pithos.Core.Agents
         {\r
             AccountInfo account;\r
             _accounts.TryRemove(accountInfo.AccountKey, out account);\r
+\r
             SnapshotDifferencer differencer;\r
             _differencer.Differencers.TryRemove(accountInfo.AccountKey, out differencer);\r
         }\r
@@ -662,9 +1109,10 @@ namespace Pithos.Core.Agents
         public void SetSelectivePaths(AccountInfo accountInfo,Uri[] added, Uri[] removed)\r
         {\r
             AbortRemovedPaths(accountInfo,removed);\r
-            DownloadNewPaths(accountInfo,added);\r
+            //DownloadNewPaths(accountInfo,added);\r
         }\r
 \r
+/*\r
         private void DownloadNewPaths(AccountInfo accountInfo, Uri[] added)\r
         {\r
             var client = new CloudFilesClient(accountInfo);\r
@@ -717,8 +1165,9 @@ namespace Pithos.Core.Agents
             //Need to get a listing of each of the URLs, then post them to the NetworkAgent\r
             //CreatesToActions(accountInfo,)\r
 \r
-/*            NetworkAgent.Post();*/\r
+/*            NetworkAgent.Post();#1#\r
         }\r
+*/\r
 \r
         private void AbortRemovedPaths(AccountInfo accountInfo, Uri[] removed)\r
         {\r