code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/PollAgent.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="PollAgent.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 \r
  43 using System.Collections.Concurrent;\r
  44 using System.ComponentModel.Composition;\r
  45 using System.Diagnostics;\r
  46 using System.Diagnostics.Contracts;\r
  47 using System.IO;\r
  48 using System.Linq.Expressions;\r
  49 using System.Reflection;\r
  50 using System.Security.Cryptography;\r
  51 using System.Threading;\r
  52 using System.Threading.Tasks;\r
  53 using System.Threading.Tasks.Dataflow;\r
  54 using Castle.ActiveRecord;\r
  55 using Pithos.Interfaces;\r
  56 using Pithos.Network;\r
  57 using log4net;\r
  58 \r
  59 namespace Pithos.Core.Agents\r
  60 {\r
  61     using System;\r
  62     using System.Collections.Generic;\r
  63     using System.Linq;\r
  64 \r
  65     /*public class PollRequest\r
  66     {\r
  67         public DateTime? Since { get; set; }\r
  68         public IEnumerable<string> Batch { get; set; }\r
  69     }*/\r
  70 \r
  71 \r
  72     /// <summary>\r
  73     /// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all\r
  74     /// objects and compares it with a previously cached version to detect differences. \r
  75     /// New files are downloaded, missing files are deleted from the local file system and common files are compared\r
  76     /// to determine the appropriate action\r
  77     /// </summary>\r
  78     [Export]\r
  79     public class PollAgent\r
  80     {\r
  81         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  82 \r
  83         [System.ComponentModel.Composition.Import]\r
  84         public IStatusKeeper StatusKeeper { get; set; }\r
  85 \r
  86         [System.ComponentModel.Composition.Import]\r
  87         public IPithosSettings Settings { get; set; }\r
  88 \r
  89         [System.ComponentModel.Composition.Import]\r
  90         public NetworkAgent NetworkAgent { get; set; }\r
  91 \r
  92         [System.ComponentModel.Composition.Import]\r
  93         public Selectives Selectives { get; set; }\r
  94 \r
  95         public IStatusNotification StatusNotification { get; set; }\r
  96 \r
  97         private CancellationTokenSource _currentOperationCancellation = new CancellationTokenSource();\r
  98 \r
  99         public void CancelCurrentOperation()\r
 100         {\r
 101             //What does it mean to cancel the current upload/download?\r
 102             //Obviously, the current operation will be cancelled by throwing\r
 103             //a cancellation exception.\r
 104             //\r
 105             //The default behavior is to retry any operations that throw.\r
 106             //Obviously this is not what we want in this situation.\r
 107             //The cancelled operation should NOT bea retried. \r
 108             //\r
 109             //This can be done by catching the cancellation exception\r
 110             //and avoiding the retry.\r
 111             //\r
 112 \r
 113             //Have to reset the cancellation source - it is not possible to reset the source\r
 114             //Have to prevent a case where an operation requests a token from the old source\r
 115             var oldSource = Interlocked.Exchange(ref _currentOperationCancellation, new CancellationTokenSource());\r
 116             oldSource.Cancel();\r
 117 \r
 118         }\r
 119 \r
 120         public bool Pause\r
 121         {\r
 122             get {\r
 123                 return _pause;\r
 124             }\r
 125             set {\r
 126                 _pause = value;                \r
 127                 if (!_pause)\r
 128                     _unPauseEvent.Set();\r
 129                 else\r
 130                 {\r
 131                     _unPauseEvent.Reset();\r
 132                 }\r
 133             }\r
 134         }\r
 135 \r
 136         public CancellationToken CancellationToken\r
 137         {\r
 138             get { return _currentOperationCancellation.Token; }\r
 139         }\r
 140 \r
 141         private bool _firstPoll = true;\r
 142 \r
 143         //The Sync Event signals a manual synchronisation\r
 144         private readonly AsyncManualResetEvent _syncEvent = new AsyncManualResetEvent();\r
 145 \r
 146         private readonly AsyncManualResetEvent _unPauseEvent = new AsyncManualResetEvent(true);\r
 147 \r
 148         private readonly ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>();\r
 149         private readonly ConcurrentDictionary<Uri, AccountInfo> _accounts = new ConcurrentDictionary<Uri,AccountInfo>();\r
 150 \r
 151         //private readonly ActionBlock<PollRequest>  _pollAction;\r
 152 \r
 153         readonly HashSet<string> _knownContainers = new HashSet<string>();\r
 154 \r
 155         \r
 156         /// <summary>\r
 157         /// Start a manual synchronization\r
 158         /// </summary>\r
 159         public void SynchNow(IEnumerable<string> paths=null)\r
 160         {\r
 161             _batchQueue.Enqueue(paths);\r
 162             _syncEvent.Set();                \r
 163 \r
 164             //_pollAction.Post(new PollRequest {Batch = paths});\r
 165         }\r
 166 \r
 167         readonly ConcurrentQueue<IEnumerable<string>> _batchQueue=new ConcurrentQueue<IEnumerable<string>>();\r
 168 \r
 169         ConcurrentDictionary<string,MovedEventArgs> _moves=new ConcurrentDictionary<string, MovedEventArgs>(); \r
 170 \r
 171         public void PostMove(MovedEventArgs args)\r
 172         {\r
 173             TaskEx.Run(() => _moves.AddOrUpdate(args.OldFullPath, args,(s,e)=>e));            \r
 174         }\r
 175 \r
 176         /// <summary>\r
 177         /// Remote files are polled periodically. Any changes are processed\r
 178         /// </summary>\r
 179         /// <param name="since"></param>\r
 180         /// <returns></returns>\r
 181         public  void PollRemoteFiles(DateTime? since = null)\r
 182         {\r
 183             if (Log.IsDebugEnabled)\r
 184                 Log.DebugFormat("Polling changes after [{0}]",since);\r
 185 \r
 186             Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
 187 \r
 188             //GC.Collect();\r
 189 \r
 190             using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
 191             {\r
 192                 //If this poll fails, we will retry with the same since value\r
 193                 var nextSince = since;\r
 194                 try\r
 195                 {\r
 196                     _unPauseEvent.Wait();\r
 197                     UpdateStatus(PithosStatus.PollSyncing);\r
 198 \r
 199                     var accountBatches=new Dictionary<Uri, IEnumerable<string>>();\r
 200                     IEnumerable<string> batch = null;\r
 201                     if (_batchQueue.TryDequeue(out batch) && batch != null)\r
 202                         foreach (var account in _accounts.Values)\r
 203                         {\r
 204                             var accountBatch = batch.Where(path => path.IsAtOrBelow(account.AccountPath));\r
 205                             accountBatches[account.AccountKey] = accountBatch;\r
 206                         }\r
 207                     \r
 208                     var moves=Interlocked.Exchange(ref _moves, new ConcurrentDictionary<string, MovedEventArgs>());\r
 209 \r
 210                     var tasks = new List<Task<DateTime?>>();\r
 211                     foreach(var accountInfo in _accounts.Values)\r
 212                     {\r
 213                         IEnumerable<string> accountBatch ;\r
 214                         accountBatches.TryGetValue(accountInfo.AccountKey,out accountBatch);\r
 215                         var t=ProcessAccountFiles (accountInfo, accountBatch, moves,since);\r
 216                         tasks.Add(t);\r
 217                     }\r
 218 \r
 219                     var nextTimes=TaskEx.WhenAll(tasks.ToList()).Result;\r
 220 \r
 221                     _firstPoll = false;\r
 222                     //Reschedule the poll with the current timestamp as a "since" value\r
 223 \r
 224                     if (nextTimes.Length>0)\r
 225                         nextSince = nextTimes.Min();\r
 226                     if (Log.IsDebugEnabled)\r
 227                         Log.DebugFormat("Next Poll at [{0}]",nextSince);\r
 228                 }\r
 229                 catch (Exception ex)\r
 230                 {\r
 231                     Log.ErrorFormat("Error while processing accounts\r\n{0}", ex);\r
 232                     //In case of failure retry with the same "since" value\r
 233                 }\r
 234 \r
 235                 UpdateStatus(PithosStatus.PollComplete);\r
 236                 //The multiple try blocks are required because we can't have an await call\r
 237                 //inside a finally block\r
 238                 //TODO: Find a more elegant solution for reschedulling in the event of an exception\r
 239                 try\r
 240                 {\r
 241                     //Wait for the polling interval to pass or the Sync event to be signalled\r
 242                     nextSince = WaitForScheduledOrManualPoll(nextSince).Result;\r
 243                 }\r
 244                 finally\r
 245                 {\r
 246                     //Ensure polling is scheduled even in case of error\r
 247                     TaskEx.Run(()=>PollRemoteFiles(nextSince));\r
 248                     //_pollAction.Post(new PollRequest {Since = nextSince});\r
 249                 }\r
 250             }\r
 251         }\r
 252 \r
 253         /// <summary>\r
 254         /// Wait for the polling period to expire or a manual sync request\r
 255         /// </summary>\r
 256         /// <param name="since"></param>\r
 257         /// <returns></returns>\r
 258         private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
 259         {\r
 260             var sync = _syncEvent.WaitAsync();\r
 261             var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval));\r
 262 \r
 263             var signaledTask = await TaskEx.WhenAny(sync, wait).ConfigureAwait(false);\r
 264             \r
 265             //Pausing takes precedence over manual sync or awaiting\r
 266             _unPauseEvent.Wait();\r
 267             \r
 268             //Wait for network processing to finish before polling\r
 269             var pauseTask=NetworkAgent.ProceedEvent.WaitAsync();\r
 270             await TaskEx.WhenAll(signaledTask, pauseTask).ConfigureAwait(false);\r
 271 \r
 272             //If polling is signalled by SynchNow, ignore the since tag\r
 273             if (sync.IsCompleted)\r
 274             {                \r
 275                 _syncEvent.Reset();\r
 276                 return null;\r
 277             }\r
 278             return since;\r
 279         }\r
 280 \r
 281         \r
 282 \r
 283         public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, IEnumerable<string> accountBatch, ConcurrentDictionary<string, MovedEventArgs> moves, DateTime? since = null)\r
 284         {\r
 285             if (accountInfo == null)\r
 286                 throw new ArgumentNullException("accountInfo");\r
 287             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
 288                 throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
 289             Contract.EndContractBlock();\r
 290 \r
 291 \r
 292             using (ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))\r
 293             {\r
 294 \r
 295                 await NetworkAgent.GetDeleteAwaiter().ConfigureAwait(false);\r
 296 \r
 297                 Log.Info("Scheduled");\r
 298                 var client = new CloudFilesClient(accountInfo);\r
 299 \r
 300                 //We don't need to check the trash container\r
 301                 var containers = client.ListContainers(accountInfo.UserName)\r
 302                     .Where(c=>c.Name.ToString()!="trash")\r
 303                     .ToList();\r
 304 \r
 305 \r
 306                 CreateContainerFolders(accountInfo, containers);\r
 307 \r
 308                 //The nextSince time fallback time is the same as the current.\r
 309                 //If polling succeeds, the next Since time will be the smallest of the maximum modification times\r
 310                 //of the shared and account objects\r
 311                 var nextSince = since;\r
 312 \r
 313                 try\r
 314                 {\r
 315                     //Wait for any deletions to finish\r
 316                     await NetworkAgent.GetDeleteAwaiter().ConfigureAwait(false);\r
 317                     //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
 318                     //than delete a file that was created while we were executing the poll                    \r
 319 \r
 320                     //Get the list of server objects changed since the last check\r
 321                     //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
 322                     var listObjects = (from container in containers\r
 323                                        select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
 324                                              client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
 325 \r
 326                     var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
 327                         client.ListSharedObjects(_knownContainers,since), "shared");\r
 328                     listObjects.Add(listShared);\r
 329                     var listTasks = await Task.Factory.WhenAll(listObjects.ToArray()).ConfigureAwait(false);\r
 330 \r
 331                     using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
 332                     {\r
 333                         var dict = listTasks.ToDictionary(t => t.AsyncState);\r
 334 \r
 335                         //Get all non-trash objects. Remember, the container name is stored in AsyncState\r
 336                         var remoteObjects = (from objectList in listTasks\r
 337                                             where (string)objectList.AsyncState.ToString() != "trash"\r
 338                                             from obj in objectList.Result\r
 339                                             orderby obj.Bytes ascending \r
 340                                             select obj).ToList();\r
 341                         \r
 342                         //Get the latest remote object modification date, only if it is after\r
 343                         //the original since date                        \r
 344                         nextSince = GetLatestDateAfter(nextSince, remoteObjects);\r
 345 \r
 346                         var sharedObjects = dict["shared"].Result;\r
 347 \r
 348                         //DON'T process trashed files\r
 349                         //If some files are deleted and added again to a folder, they will be deleted\r
 350                         //even though they are new.\r
 351                         //We would have to check file dates and hashes to ensure that a trashed file\r
 352                         //can be deleted safely from the local hard drive.\r
 353                         /*\r
 354                         //Items with the same name, hash may be both in the container and the trash\r
 355                         //Don't delete items that exist in the container\r
 356                         var realTrash = from trash in trashObjects\r
 357                                         where\r
 358                                             !remoteObjects.Any(\r
 359                                                 info => info.Name == trash.Name && info.Hash == trash.Hash)\r
 360                                    8     select trash;\r
 361                         ProcessTrashedFiles(accountInfo, realTrash);\r
 362 */\r
 363 \r
 364                         var cleanRemotes = (from info in remoteObjects.Union(sharedObjects)\r
 365                                             let name = info.Name.ToUnescapedString()??""\r
 366                                             where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&\r
 367                                                   !name.StartsWith(FolderConstants.CacheFolder + "/",\r
 368                                                                    StringComparison.InvariantCultureIgnoreCase)\r
 369                                             select info).ToList();\r
 370 \r
 371                         if (_firstPoll)\r
 372                             StatusKeeper.CleanupOrphanStates();\r
 373                         \r
 374                         var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
 375                         var currentRemotes = differencer.Current.ToList();\r
 376                         StatusKeeper.CleanupStaleStates(accountInfo, currentRemotes);\r
 377 \r
 378                         //var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
 379 \r
 380                         //May have to wait if the FileAgent has asked for a Pause, due to local changes\r
 381                         await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
 382 \r
 383                         //Get the local files here                        \r
 384                         var agent = AgentLocator<FileAgent>.Get(accountInfo.AccountPath);                                                \r
 385                         var files = LoadLocalFileTuples(accountInfo, accountBatch);\r
 386 \r
 387                         var states = FileState.Queryable.ToList();                        \r
 388                         \r
 389                         var infos = (from remote in currentRemotes\r
 390                                     let path = remote.RelativeUrlToFilePath(accountInfo.UserName)\r
 391                                     let info=agent.GetFileSystemInfo(path)\r
 392                                     select Tuple.Create(info.FullName,remote))\r
 393                                     .ToList();\r
 394 \r
 395                         var token = _currentOperationCancellation.Token;\r
 396 \r
 397                         var tuples = MergeSources(infos, files, states,moves).ToList();\r
 398 \r
 399                         var processedPaths = new HashSet<string>();\r
 400                         //Process only the changes in the batch file, if one exists\r
 401                         var stateTuples = accountBatch==null?tuples:tuples.Where(t => accountBatch.Contains(t.FilePath));\r
 402                         foreach (var tuple in stateTuples.Where(s=>!s.Locked))\r
 403                         {\r
 404                             await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
 405 \r
 406                             //Set the Merkle Hash\r
 407                             //SetMerkleHash(accountInfo, tuple);\r
 408 \r
 409                             await SyncSingleItem(accountInfo, tuple, agent, moves,processedPaths,token).ConfigureAwait(false);\r
 410 \r
 411                         }\r
 412 \r
 413 \r
 414                         //On the first run\r
 415 /*\r
 416                         if (_firstPoll)\r
 417                         {\r
 418                             MarkSuspectedDeletes(accountInfo, cleanRemotes);\r
 419                         }\r
 420 */\r
 421 \r
 422 \r
 423                         Log.Info("[LISTENER] End Processing");\r
 424                     }\r
 425                 }\r
 426                 catch (Exception ex)\r
 427                 {\r
 428                     Log.ErrorFormat("[FAIL] ListObjects for {0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);\r
 429                     return nextSince;\r
 430                 }\r
 431 \r
 432                 Log.Info("[LISTENER] Finished");\r
 433                 return nextSince;\r
 434             }\r
 435         }\r
 436 /*\r
 437 \r
 438         private static void SetMerkleHash(AccountInfo accountInfo, StateTuple tuple)\r
 439         {\r
 440             //The Merkle hash for directories is that of an empty buffer\r
 441             if (tuple.FileInfo is DirectoryInfo)\r
 442                 tuple.C = MERKLE_EMPTY;\r
 443             else if (tuple.FileState != null && tuple.MD5 == tuple.FileState.ETag)\r
 444             {\r
 445                 //If there is a state whose MD5 matches, load the merkle hash from the file state\r
 446                 //insteaf of calculating it\r
 447                 tuple.C = tuple.FileState.Checksum;                              \r
 448             }\r
 449             else\r
 450             {\r
 451                 tuple.Merkle = Signature.CalculateTreeHashAsync((FileInfo)tuple.FileInfo, accountInfo.BlockSize, accountInfo.BlockHash,1,progress);\r
 452                 //tuple.C=tuple.Merkle.TopHash.ToHashString();                \r
 453             }\r
 454         }\r
 455 */\r
 456 \r
 457         private IEnumerable<FileSystemInfo> LoadLocalFileTuples(AccountInfo accountInfo,IEnumerable<string> batch )\r
 458         {\r
 459             using (ThreadContext.Stacks["Account Files Hashing"].Push(accountInfo.UserName))\r
 460             {\r
 461                 var batchPaths = (batch==null)?new List<string>():batch.ToList();\r
 462                 IEnumerable<FileSystemInfo> localInfos=AgentLocator<FileAgent>.Get(accountInfo.AccountPath)\r
 463                                                         .EnumerateFileSystemInfos();\r
 464                 if (batchPaths.Count>0)\r
 465                     localInfos= localInfos.Where(fi => batchPaths.Contains(fi.FullName));\r
 466 \r
 467                 return localInfos;\r
 468             }\r
 469         }\r
 470 \r
 471         /// <summary>\r
 472         /// Wait and Pause the agent while waiting\r
 473         /// </summary>\r
 474         /// <param name="backoff"></param>\r
 475         /// <returns></returns>\r
 476         private async Task PauseFor(int backoff)\r
 477         {\r
 478 \r
 479             Pause = true;\r
 480             await TaskEx.Delay(backoff).ConfigureAwait(false);\r
 481             Pause = false;\r
 482         }\r
 483 \r
 484         private async Task SyncSingleItem(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves,HashSet<string> processedPaths, CancellationToken token)\r
 485         {\r
 486             Log.DebugFormat("Sync [{0}] C:[{1}] L:[{2}] S:[{3}]", tuple.FilePath, tuple.C, tuple.L, tuple.S);\r
 487 \r
 488             //If the processed paths already contain the current path, exit\r
 489             if (!processedPaths.Add(tuple.FilePath))\r
 490                 return;\r
 491 \r
 492             try\r
 493             {\r
 494                 bool isInferredParent = tuple.ObjectInfo != null && tuple.ObjectInfo.UUID.StartsWith("00000000-0000-0000");\r
 495 \r
 496                 var localFilePath = tuple.FilePath;\r
 497                 //Don't use the tuple info, it may have been deleted\r
 498                 var localInfo = FileInfoExtensions.FromPath(localFilePath);\r
 499 \r
 500 \r
 501                 var isUnselectedRootFolder = agent.IsUnselectedRootFolder(tuple.FilePath);\r
 502 \r
 503                 //Unselected root folders that have not yet been uploaded should be uploaded and added to the \r
 504                 //selective folders\r
 505 \r
 506                 if (!Selectives.IsSelected(accountInfo, localFilePath) &&\r
 507                     !(isUnselectedRootFolder && tuple.ObjectInfo == null))\r
 508                     return;\r
 509 \r
 510                 // Local file unchanged? If both C and L are null, make sure it's because \r
 511                 //both the file is missing and the state checksum is not missing\r
 512                 if (tuple.C == tuple.L /*&& (localInfo.Exists || tuple.FileState == null)*/)\r
 513                 {\r
 514                     //No local changes\r
 515                     //Server unchanged?\r
 516                     if (tuple.S == tuple.L)\r
 517                     {\r
 518                         // No server changes\r
 519                         //Has the file been renamed locally?\r
 520                         if (!MoveForLocalMove(accountInfo,tuple))\r
 521                             //Has the file been renamed on the server?\r
 522                             MoveForServerMove(accountInfo, tuple);\r
 523                     }\r
 524                     else\r
 525                     {\r
 526                         //Different from server\r
 527                         //Does the server file exist?\r
 528                         if (tuple.S == null)\r
 529                         {\r
 530                             //Server file doesn't exist\r
 531                             //deleteObjectFromLocal()\r
 532                             using (\r
 533                                 StatusNotification.GetNotifier("Deleting local {0}", "Deleted local {0}",\r
 534                                                                localInfo.Name))\r
 535                             {\r
 536                                 DeleteLocalFile(agent, localFilePath);\r
 537                             }\r
 538                         }\r
 539                         else\r
 540                         {\r
 541                             //Server file exists\r
 542                             //downloadServerObject() // Result: L = S\r
 543                             //If the file has moved on the server, move it locally before downloading\r
 544                             using (\r
 545                                 StatusNotification.GetNotifier("Downloading {0}", "Downloaded {0}",\r
 546                                                                localInfo.Name))\r
 547                             {\r
 548                                 var targetPath = MoveForServerMove(accountInfo, tuple);\r
 549                                 if (targetPath != null)\r
 550                                 {\r
 551 \r
 552                                     await DownloadCloudFile(accountInfo, tuple, token, targetPath).ConfigureAwait(false);\r
 553 \r
 554                                     AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
 555                                 }\r
 556                             }\r
 557 \r
 558                             /*\r
 559                                                             StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
 560                                                                                       FileOverlayStatus.Normal, "");\r
 561                                 */\r
 562                         }\r
 563                     }\r
 564                 }\r
 565                 else\r
 566                 {                   \r
 567 \r
 568                     //Local changes found\r
 569 \r
 570                     //Server unchanged?\r
 571                     if (tuple.S == tuple.L)\r
 572                     {\r
 573                         //The FileAgent selective sync checks for new root folder files\r
 574                         if (!agent.Ignore(localFilePath))\r
 575                         {\r
 576                             if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
 577                             {\r
 578                                 //deleteObjectFromServer()\r
 579                                 DeleteCloudFile(accountInfo, tuple);\r
 580                                 //updateRecord( Remove L, S)                  \r
 581                             }\r
 582                             else\r
 583                             {\r
 584                                 //uploadLocalObject() // Result: S = C, L = S                        \r
 585                                 var progress = new Progress<double>(d =>\r
 586                                     StatusNotification.Notify(new StatusNotification(String.Format("Merkle Hashing for Upload {0:p} of {1}", d, localInfo.Name))));\r
 587 \r
 588                                 //Is it an unselected root folder\r
 589                                 var isCreation = isUnselectedRootFolder ||//or a new folder under a selected parent?\r
 590                                         (localInfo is DirectoryInfo && Selectives.IsSelected(accountInfo, localInfo) && tuple.FileState == null && tuple.ObjectInfo == null);\r
 591 \r
 592 \r
 593                                 //Is this a result of a FILE move with no modifications? Then try to move it,\r
 594                                 //to avoid an expensive hash\r
 595                                 if (!MoveForLocalMove(accountInfo, tuple))\r
 596                                 {\r
 597                                     await UploadLocalFile(accountInfo, tuple, token, isCreation, localInfo,processedPaths, progress).ConfigureAwait(false);\r
 598                                 }\r
 599 \r
 600                                 //updateRecord( S = C )\r
 601                                 //State updated by the uploader\r
 602                                 \r
 603                                 if (isCreation )\r
 604                                 {                                    \r
 605                                     ProcessChildren(accountInfo, tuple, agent, moves,processedPaths,token);\r
 606                                 }\r
 607                             }\r
 608                         }\r
 609                     }\r
 610                     else\r
 611                     {\r
 612                         if (tuple.C == tuple.S)\r
 613                         {\r
 614                             // (Identical Changes) Result: L = S\r
 615                             //doNothing()\r
 616                             \r
 617                             //Don't update anything for nonexistend server files\r
 618                             if (tuple.S != null)\r
 619                             {\r
 620                                 //Detect server moves\r
 621                                 var targetPath = MoveForServerMove(accountInfo, tuple);\r
 622                                 if (targetPath != null)\r
 623                                 {\r
 624                                     Debug.Assert(tuple.Merkle != null);\r
 625                                     StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo, tuple.Merkle);\r
 626 \r
 627                                     AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
 628                                 }\r
 629                             }\r
 630                             else\r
 631                             {\r
 632                                 //At this point, C==S==NULL and we have a stale state (L)\r
 633                                 //Log the stale tuple for investigation\r
 634                                 Log.WarnFormat("Stale tuple detected FilePathPath:[{0}], State:[{1}], LocalFile:[{2}]", tuple.FilePath, tuple.FileState, tuple.FileInfo);\r
 635 \r
 636                                 //And remove it\r
 637                                 if (!String.IsNullOrWhiteSpace(tuple.FilePath))\r
 638                                     StatusKeeper.ClearFileStatus(tuple.FilePath);\r
 639                             }\r
 640                         }\r
 641                         else\r
 642                         {\r
 643                             if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
 644                             {\r
 645                                 //deleteObjectFromServer()\r
 646                                 DeleteCloudFile(accountInfo, tuple);\r
 647                                 //updateRecord(Remove L, S)                  \r
 648                             }\r
 649                                 //If both the local and server files are missing, the state is stale\r
 650                             else if (!localInfo.Exists && (tuple.S == null || tuple.ObjectInfo == null))\r
 651                             {\r
 652                                 StatusKeeper.ClearFileStatus(localInfo.FullName);\r
 653                             }\r
 654                             else\r
 655                             {\r
 656                                 ReportConflictForMismatch(localFilePath);\r
 657                                 //identifyAsConflict() // Manual action required\r
 658                             }\r
 659                         }\r
 660                     }\r
 661                 }\r
 662             }\r
 663             catch (Exception exc)\r
 664             {\r
 665                 //In case of error log and retry with the next poll\r
 666                 Log.ErrorFormat("[SYNC] Failed for file {0}. Will Retry.\r\n{1}",tuple.FilePath,exc);\r
 667             }\r
 668         }\r
 669 \r
 670         private void DeleteLocalFile(FileAgent agent, string localFilePath)\r
 671         {\r
 672             StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
 673                                       FileOverlayStatus.Deleted, "");\r
 674             using (NetworkGate.Acquire(localFilePath, NetworkOperation.Deleting))\r
 675             {\r
 676                 agent.Delete(localFilePath);\r
 677             }\r
 678             //updateRecord(Remove C, L)\r
 679             StatusKeeper.ClearFileStatus(localFilePath);\r
 680         }\r
 681 \r
 682         private async Task DownloadCloudFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token, string targetPath)\r
 683         {\r
 684             StatusKeeper.SetFileState(targetPath, FileStatus.Modified, FileOverlayStatus.Modified,\r
 685                                       "");\r
 686 \r
 687             var finalHash=await\r
 688                 NetworkAgent.Downloader.DownloadCloudFile(accountInfo, tuple.ObjectInfo, targetPath,\r
 689                                                           token)\r
 690                     .ConfigureAwait(false);\r
 691             //updateRecord( L = S )\r
 692             StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
 693                                             finalHash);\r
 694 \r
 695             StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo,finalHash);\r
 696         }\r
 697 \r
 698         private async Task UploadLocalFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token,\r
 699                                      bool isUnselectedRootFolder, FileSystemInfo localInfo, HashSet<string> processedPaths, Progress<double> progress)\r
 700         {\r
 701             var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
 702                                                accountInfo.BlockSize, accountInfo.BlockHash,\r
 703                                                "Poll", isUnselectedRootFolder, token, progress,tuple.Merkle);            \r
 704             \r
 705             using (StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}",\r
 706                                                   localInfo.Name))\r
 707             {\r
 708                 await NetworkAgent.Uploader.UploadCloudFile(action, token).ConfigureAwait(false);\r
 709             }\r
 710 \r
 711             if (isUnselectedRootFolder)\r
 712             {\r
 713                 var dirActions =(\r
 714                     from dir in ((DirectoryInfo) localInfo).EnumerateDirectories("*", SearchOption.AllDirectories)\r
 715                     let subAction = new CloudUploadAction(accountInfo, dir, null,\r
 716                                                           accountInfo.BlockSize, accountInfo.BlockHash,\r
 717                                                           "Poll", true, token, progress)\r
 718                     select subAction).ToList();\r
 719                 foreach (var dirAction in dirActions)\r
 720                 {\r
 721                     processedPaths.Add(dirAction.LocalFile.FullName);\r
 722                 }\r
 723                 \r
 724                 await TaskEx.WhenAll(dirActions.Select(a=>NetworkAgent.Uploader.UploadCloudFile(a,token)).ToArray());\r
 725             }\r
 726         }\r
 727 \r
 728         private bool MoveForLocalMove(AccountInfo accountInfo, StateTuple tuple)\r
 729         {\r
 730             //Is the file a directory or previous path missing?\r
 731             if (tuple.FileInfo is DirectoryInfo)\r
 732                 return false;\r
 733             //Is the previous path missing?\r
 734             if (String.IsNullOrWhiteSpace(tuple.OldFullPath))\r
 735                 return false;\r
 736             //Has the file locally, in which case it should be uploaded rather than moved?\r
 737             if (tuple.OldMD5 != tuple.MD5)\r
 738                 return false;\r
 739 \r
 740             var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 741             var serverPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
 742             //Has the file been renamed on the server?\r
 743             if (!tuple.OldFullPath.Equals(serverPath))\r
 744             {\r
 745                 ReportConflictForDoubleRename(tuple.FilePath);\r
 746                 return false;\r
 747             }\r
 748 \r
 749             try\r
 750             {\r
 751 \r
 752                 var client = new CloudFilesClient(accountInfo);\r
 753                 var objectInfo = CloudAction.CreateObjectInfoFor(accountInfo, tuple.FileInfo);\r
 754                 var containerPath = Path.Combine(accountInfo.AccountPath, objectInfo.Container.ToUnescapedString());\r
 755                 //TODO: SImplify these multiple conversions from and to Uris\r
 756                 var oldName = tuple.OldFullPath.AsRelativeTo(containerPath);\r
 757                 //Then execute a move instead of an upload\r
 758                 using (StatusNotification.GetNotifier("Moving {0}", "Moved {0}", tuple.FileInfo.Name))\r
 759                 {\r
 760                     client.MoveObject(objectInfo.Account, objectInfo.Container, oldName.ToEscapedUri(),\r
 761                                                           objectInfo.Container, objectInfo.Name);\r
 762                 }\r
 763                 return true;\r
 764             }\r
 765             catch (Exception exc)\r
 766             {\r
 767                 Log.ErrorFormat("[MOVE] Failed for [{0}],:\r\n{1}", tuple.FilePath, exc);\r
 768                 //Return false to force an upload of the file\r
 769                 return false;\r
 770             }\r
 771 \r
 772         }\r
 773 \r
 774         private void AddOwnFolderToSelectives(AccountInfo accountInfo, StateTuple tuple, string targetPath)\r
 775         {\r
 776             //Not for shared folders\r
 777             if (tuple.ObjectInfo.IsShared==true)\r
 778                 return;\r
 779             //Also ensure that any newly created folders are added to the selectives, if the original folder was selected\r
 780             var containerPath = Path.Combine(accountInfo.AccountPath, tuple.ObjectInfo.Container.ToUnescapedString());\r
 781 \r
 782             //If this is a root folder encountered for the first time\r
 783             if (tuple.L == null && Directory.Exists(tuple.FileInfo.FullName) \r
 784                 && (tuple.FileInfo.FullName.IsAtOrBelow(containerPath)))\r
 785             {\r
 786                 \r
 787                 var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 788                 var initialPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
 789 \r
 790                 //var hasMoved = true;// !initialPath.Equals(targetPath);\r
 791                 //If the new path is under a selected folder, add it to the selectives as well\r
 792                 if (Selectives.IsSelected(accountInfo, initialPath))\r
 793                 {\r
 794                     Selectives.AddUri(accountInfo, tuple.ObjectInfo.Uri);\r
 795                     Selectives.Save(accountInfo);\r
 796                 }\r
 797             }\r
 798         }\r
 799 \r
 800         private string MoveForServerMove(AccountInfo accountInfo, StateTuple tuple)\r
 801         {\r
 802             if (tuple.ObjectInfo == null)\r
 803                 return null;\r
 804             var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 805             var serverPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
 806             \r
 807             //Compare Case Insensitive\r
 808             if (String.Equals(tuple.FilePath ,serverPath,StringComparison.InvariantCultureIgnoreCase)) \r
 809                 return serverPath;\r
 810 \r
 811             //Has the file been renamed locally?\r
 812             if (!String.IsNullOrWhiteSpace(tuple.OldFullPath) &&  !tuple.OldFullPath.Equals(tuple.FilePath))\r
 813             {\r
 814                 ReportConflictForDoubleRename(tuple.FilePath);\r
 815                 return null;\r
 816             }\r
 817 \r
 818             tuple.FileInfo.Refresh();\r
 819             //The file/folder may not exist if it was moved because its parent moved\r
 820             if (!tuple.FileInfo.Exists)\r
 821             {\r
 822                 var target=FileInfoExtensions.FromPath(serverPath);\r
 823                 if (!target.Exists)\r
 824                 {\r
 825                     Log.ErrorFormat("No source or target found while trying to move {0} to {1}", tuple.FileInfo.FullName, serverPath);\r
 826                 }\r
 827                 return serverPath;\r
 828             }\r
 829 \r
 830             using (StatusNotification.GetNotifier("Moving local {0}", "Moved local {0}", Path.GetFileName(tuple.FilePath)))\r
 831             using(NetworkGate.Acquire(tuple.FilePath,NetworkOperation.Renaming))\r
 832             {\r
 833                     \r
 834                 var fi = tuple.FileInfo as FileInfo;\r
 835                 if (fi != null)\r
 836                 {\r
 837                     var targetFile = new FileInfo(serverPath);\r
 838                     if (!targetFile.Directory.Exists)\r
 839                         targetFile.Directory.Create();\r
 840                     fi.MoveTo(serverPath);\r
 841                 }\r
 842                 var di = tuple.FileInfo as DirectoryInfo;\r
 843                 if (di != null)\r
 844                 {\r
 845                     var targetDir = new DirectoryInfo(serverPath);\r
 846                     if (!targetDir.Parent.Exists)\r
 847                         targetDir.Parent.Create();\r
 848                     di.MoveTo(serverPath);\r
 849                 }\r
 850             }\r
 851 \r
 852             StatusKeeper.StoreInfo(serverPath, tuple.ObjectInfo);\r
 853 \r
 854             return serverPath;\r
 855         }\r
 856 \r
 857         private void DeleteCloudFile(AccountInfo accountInfo, StateTuple tuple)\r
 858         {\r
 859             using (StatusNotification.GetNotifier("Deleting server {0}", "Deleted server {0}", Path.GetFileName(tuple.FilePath)))\r
 860             {\r
 861 \r
 862                 StatusKeeper.SetFileState(tuple.FilePath, FileStatus.Deleted,\r
 863                                           FileOverlayStatus.Deleted, "");\r
 864                 NetworkAgent.DeleteAgent.DeleteCloudFile(accountInfo, tuple.ObjectInfo);\r
 865                 StatusKeeper.ClearFileStatus(tuple.FilePath);\r
 866             }\r
 867         }\r
 868 \r
 869         private void ProcessChildren(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves,HashSet<string> processedPaths,CancellationToken token)\r
 870         {\r
 871 \r
 872             var dirInfo = tuple.FileInfo as DirectoryInfo;\r
 873             var folderTuples = from folder in dirInfo.EnumerateDirectories("*", SearchOption.AllDirectories)\r
 874                                select new StateTuple(folder){C=Signature.MERKLE_EMPTY};\r
 875             \r
 876             var fileTuples = from file in dirInfo.EnumerateFiles("*", SearchOption.AllDirectories)\r
 877                              let state=StatusKeeper.GetStateByFilePath(file.FullName)\r
 878                              select new StateTuple(file){\r
 879                                             Merkle=StatusAgent.CalculateTreeHash(file,accountInfo,state,\r
 880                                             Settings.HashingParallelism,token,null)\r
 881                                         };\r
 882             \r
 883             //Process folders first, to ensure folders appear on the sever as soon as possible\r
 884             folderTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, moves, processedPaths,token).Wait());\r
 885             \r
 886             fileTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, moves,processedPaths, token).Wait());\r
 887         }\r
 888 \r
 889 \r
 890         /*\r
 891          *                 //Use the queue to retry locked file hashing\r
 892                 var fileQueue = new ConcurrentQueue<FileSystemInfo>(localInfos);\r
 893                 \r
 894 \r
 895                 var results = new List<Tuple<FileSystemInfo, string>>();\r
 896                 var backoff = 0;\r
 897                 while (fileQueue.Count > 0)\r
 898                 {\r
 899                     FileSystemInfo file;\r
 900                     fileQueue.TryDequeue(out file);\r
 901                     using (ThreadContext.Stacks["File"].Push(file.FullName))\r
 902                     {\r
 903                         try\r
 904                         {\r
 905                             //Replace MD5 here, do the calc while syncing individual files\r
 906                             string hash ;\r
 907                             if (file is DirectoryInfo)\r
 908                                 hash = MD5_EMPTY;\r
 909                             else\r
 910                             {\r
 911                                 //Wait in case the FileAgent has requested a Pause\r
 912                                 await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
 913                                 \r
 914                                 using (StatusNotification.GetNotifier("Hashing {0}", "", file.Name))\r
 915                                 {\r
 916                                     hash = ((FileInfo)file).ComputeShortHash(StatusNotification);\r
 917                                     backoff = 0;\r
 918                                 }\r
 919                             }                            \r
 920                             results.Add(Tuple.Create(file, hash));\r
 921                         }\r
 922                         catch (IOException exc)\r
 923                         {\r
 924                             Log.WarnFormat("[HASH] File in use, will retry [{0}]", exc);\r
 925                             fileQueue.Enqueue(file);\r
 926                             //If this is the only enqueued file                            \r
 927                             if (fileQueue.Count != 1) continue;\r
 928                             \r
 929                             \r
 930                             //Increase delay\r
 931                             if (backoff<60000)\r
 932                                 backoff += 10000;\r
 933                             //Pause Polling for the specified time\r
 934                         }\r
 935                         if (backoff>0)\r
 936                             await PauseFor(backoff).ConfigureAwait(false);\r
 937                     }\r
 938                 }\r
 939 \r
 940                 return results;\r
 941 \r
 942          */\r
 943         private IEnumerable<StateTuple> MergeSources(IEnumerable<Tuple<string, ObjectInfo>> infos, IEnumerable<FileSystemInfo> files, List<FileState> states, ConcurrentDictionary<string, MovedEventArgs> moves)\r
 944         {\r
 945             var tuplesByPath = new Dictionary<string, StateTuple>();\r
 946             foreach (var info in files)\r
 947             {\r
 948                 var tuple = new StateTuple(info);\r
 949                 //Is this the target of a move event?\r
 950                 var moveArg = moves.Values.FirstOrDefault(arg => info.FullName.Equals(arg.FullPath, StringComparison.InvariantCultureIgnoreCase) \r
 951                     || info.FullName.IsAtOrBelow(arg.FullPath));\r
 952                 if (moveArg != null)\r
 953                 {\r
 954                     tuple.NewFullPath = info.FullName;\r
 955                     var relativePath = info.AsRelativeTo(moveArg.FullPath);                    \r
 956                     tuple.OldFullPath = Path.Combine(moveArg.OldFullPath, relativePath);\r
 957                     tuple.OldMD5 = states.FirstOrDefault(st => st.FilePath.Equals(tuple.OldFullPath, StringComparison.InvariantCultureIgnoreCase))\r
 958                                 .NullSafe(st => st.LastMD5);\r
 959                 }\r
 960 \r
 961                 tuplesByPath[tuple.FilePath] = tuple;\r
 962             }\r
 963             \r
 964 \r
 965             \r
 966             \r
 967             //For files that have state\r
 968             foreach (var state in states)\r
 969             {\r
 970                 StateTuple hashTuple;\r
 971 \r
 972                 \r
 973                 if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple))\r
 974                 {\r
 975                     hashTuple.FileState = state;\r
 976                     UpdateHashes(hashTuple);\r
 977                 }\r
 978                 else if (moves.ContainsKey(state.FilePath) && tuplesByPath.TryGetValue(moves[state.FilePath].FullPath, out hashTuple))\r
 979                 {\r
 980                     hashTuple.FileState = state;\r
 981                     UpdateHashes(hashTuple);\r
 982                 }\r
 983                 else\r
 984                 {\r
 985                     var fsInfo = FileInfoExtensions.FromPath(state.FilePath);\r
 986                     hashTuple = new StateTuple {FileInfo = fsInfo, FileState = state};\r
 987 \r
 988                     //Is the source of a moved item?\r
 989                     var moveArg = moves.Values.FirstOrDefault(arg => state.FilePath.Equals(arg.OldFullPath,StringComparison.InvariantCultureIgnoreCase) \r
 990                         || state.FilePath.IsAtOrBelow(arg.OldFullPath));\r
 991                     if (moveArg != null)\r
 992                     {\r
 993                         var relativePath = state.FilePath.AsRelativeTo(moveArg.OldFullPath);\r
 994                         hashTuple.NewFullPath = Path.Combine(moveArg.FullPath,relativePath);\r
 995                         hashTuple.OldFullPath = state.FilePath;\r
 996                         //Do we have the old MD5?\r
 997                         hashTuple.OldMD5 = state.LastMD5;\r
 998                     }\r
 999 \r
1000 \r
1001                     tuplesByPath[state.FilePath] = hashTuple;\r
1002                 }\r
1003             }\r
1004             //for files that don't have state\r
1005             foreach (var tuple in tuplesByPath.Values.Where(t => t.FileState == null))\r
1006             {\r
1007                 UpdateHashes(tuple);\r
1008             }\r
1009 \r
1010             var tuplesByID = tuplesByPath.Values\r
1011                 .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null)\r
1012                 .ToDictionary(tuple=>tuple.FileState.ObjectID,tuple=>tuple);//new Dictionary<Guid, StateTuple>();\r
1013 \r
1014             foreach (var info in infos)\r
1015             {\r
1016                 StateTuple hashTuple;\r
1017                 var filePath = info.Item1;\r
1018                 var objectInfo = info.Item2;\r
1019                 var objectID = objectInfo.UUID;\r
1020 \r
1021                 if (objectID != _emptyGuid &&  tuplesByID.TryGetValue(objectID, out hashTuple))\r
1022                 {\r
1023                     hashTuple.ObjectInfo = objectInfo;                    \r
1024                 }\r
1025                 else if (tuplesByPath.TryGetValue(filePath, out hashTuple))\r
1026                 {\r
1027                     hashTuple.ObjectInfo = objectInfo;\r
1028                 }\r
1029                 else\r
1030                 {\r
1031 \r
1032                     \r
1033                     var fsInfo = FileInfoExtensions.FromPath(filePath);\r
1034                     hashTuple= new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo};\r
1035                     tuplesByPath[filePath] = hashTuple;\r
1036                     \r
1037                     if (objectInfo.UUID!=_emptyGuid)\r
1038                         tuplesByID[objectInfo.UUID] = hashTuple;\r
1039                 }\r
1040             }\r
1041 \r
1042             var tuples = tuplesByPath.Values;\r
1043             var brokenTuples = from tuple in tuples\r
1044                                where tuple.FileState != null && tuple.FileState.Checksum == null\r
1045                                     && tuple.ObjectInfo != null && (tuple.FileInfo==null || !tuple.FileInfo.Exists)\r
1046                                select tuple;\r
1047             var actualTuples = tuples.Except(brokenTuples);\r
1048             Debug.Assert(actualTuples.All(t => t.HashesValid()));\r
1049 \r
1050             foreach (var tuple in brokenTuples)\r
1051             {\r
1052                 StatusKeeper.SetFileState(tuple.FileState.FilePath, \r
1053                     FileStatus.Conflict, FileOverlayStatus.Conflict, "FileState without checksum encountered for server object missing from disk");    \r
1054             }\r
1055                                    \r
1056             return actualTuples;\r
1057         }\r
1058 \r
1059         \r
1060         /// <summary>\r
1061         /// Update the tuple with the file's hashes, avoiding calculation if the file is unchanged\r
1062         /// </summary>\r
1063         /// <param name="hashTuple"></param>\r
1064         /// <remarks>\r
1065         /// The function first checks the file's size and last write date to see if there are any changes. If there are none,\r
1066         /// the file's stored hashes are used.\r
1067         /// Otherwise, MD5 is calculated first to ensure there are no changes. If MD5 is different, the Merkle hash is calculated\r
1068         /// </remarks>\r
1069         private void  UpdateHashes(StateTuple hashTuple)\r
1070         {\r
1071             \r
1072             try\r
1073             {\r
1074                 var state = hashTuple.NullSafe(s => s.FileState);\r
1075                 var storedHash = state.NullSafe(s => s.Checksum);\r
1076                 var storedHashes = state.NullSafe(s => s.Hashes);\r
1077                 var storedMD5 = state.NullSafe(s => s.LastMD5);\r
1078                 var storedDate = state.NullSafe(s => s.LastWriteDate) ?? DateTime.MinValue;\r
1079                 var storedLength = state.NullSafe(s => s.LastLength);\r
1080 \r
1081                 var md5Hash = Signature.MD5_EMPTY;                \r
1082                 var merkle=TreeHash.Empty;\r
1083 \r
1084                 if (hashTuple.FileInfo is FileInfo)\r
1085                 {\r
1086                     var file = (FileInfo)hashTuple.FileInfo.WithProperCapitalization();\r
1087                     \r
1088                     //Attributes unchanged?\r
1089                     //LastWriteTime is only accurate to the second\r
1090                     var unchangedAttributes = file.LastWriteTime - storedDate < TimeSpan.FromSeconds(1) \r
1091                         && storedLength == file.Length;\r
1092                     \r
1093                     //Attributes appear unchanged but the file length doesn't match the stored hash ?\r
1094                     var nonEmptyMismatch = unchangedAttributes && \r
1095                         (file.Length == 0 ^ storedHash== Signature.MERKLE_EMPTY);\r
1096 \r
1097                     //Missing hashes for NON-EMPTY hash ?\r
1098                     var missingHashes = storedHash != Signature.MERKLE_EMPTY &&\r
1099                         String.IsNullOrWhiteSpace(storedHashes);\r
1100 \r
1101                     //Unchanged attributes but changed MD5 \r
1102                     //Short-circuiting ensures MD5 is computed only if the attributes are changed\r
1103                     var md5Mismatch = (!unchangedAttributes &&\r
1104                                        file.ComputeShortHash(StatusNotification) != storedMD5);\r
1105 \r
1106 \r
1107                     //If the attributes are unchanged but the Merkle doesn't match the size,\r
1108                     //or the attributes and the MD5 hash have changed, \r
1109                     //or the hashes are missing but the tophash is NOT empty, we need to recalculate\r
1110                     //\r
1111                     //Otherwise we load the hashes from state\r
1112                     if (nonEmptyMismatch || md5Mismatch || missingHashes)\r
1113                         merkle = RecalculateTreehash(file);\r
1114                     else\r
1115                     {\r
1116                         merkle=TreeHash.Parse(hashTuple.FileState.Hashes);\r
1117                         merkle.MD5 = storedMD5;\r
1118                     }\r
1119 \r
1120 \r
1121                     md5Hash = merkle.MD5;\r
1122                 }\r
1123                 hashTuple.MD5 = md5Hash;\r
1124                 //Setting Merkle also updates C\r
1125                 hashTuple.Merkle = merkle;\r
1126             }\r
1127             catch (IOException)\r
1128             {\r
1129                 hashTuple.Locked = true;\r
1130             }            \r
1131         }\r
1132 \r
1133         /// <summary>\r
1134         /// Recalculate a file's treehash and md5 and update the database\r
1135         /// </summary>\r
1136         /// <param name="file"></param>\r
1137         /// <returns></returns>\r
1138         private TreeHash RecalculateTreehash(FileInfo file)\r
1139         {\r
1140             var progress = new Progress<double>(d =>StatusNotification.Notify(\r
1141                                                     new StatusNotification(String.Format("Hashing {0} of {1}", d, file.Name))));\r
1142             var merkle = Signature.CalculateTreeHash(file, StatusKeeper.BlockSize, \r
1143                 StatusKeeper.BlockHash,CancellationToken, progress);\r
1144             StatusKeeper.UpdateFileTreeHash(file.FullName, merkle);\r
1145             return merkle;\r
1146         }\r
1147 \r
1148         /// <summary>\r
1149         /// Returns the latest LastModified date from the list of objects, but only if it is before\r
1150         /// than the threshold value\r
1151         /// </summary>\r
1152         /// <param name="threshold"></param>\r
1153         /// <param name="cloudObjects"></param>\r
1154         /// <returns></returns>\r
1155         private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
1156         {\r
1157             DateTime? maxDate = null;\r
1158             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
1159                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
1160             if (maxDate == null || maxDate == DateTime.MinValue)\r
1161                 return threshold;\r
1162             if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate)\r
1163                 return maxDate;\r
1164             return threshold;\r
1165         }\r
1166 \r
1167         /// <summary>\r
1168         /// Returns the latest LastModified date from the list of objects, but only if it is after\r
1169         /// the threshold value\r
1170         /// </summary>\r
1171         /// <param name="threshold"></param>\r
1172         /// <param name="cloudObjects"></param>\r
1173         /// <returns></returns>\r
1174         private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
1175         {\r
1176             DateTime? maxDate = null;\r
1177             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
1178                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
1179             if (maxDate == null || maxDate == DateTime.MinValue)\r
1180                 return threshold;\r
1181             if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate)\r
1182                 return maxDate;\r
1183             return threshold;\r
1184         }\r
1185 \r
1186         readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
1187         private bool _pause;\r
1188         private readonly string _emptyGuid = Guid.Empty.ToString();\r
1189 \r
1190 \r
1191 \r
1192         private void ReportConflictForMismatch(string localFilePath)\r
1193         {\r
1194             if (String.IsNullOrWhiteSpace(localFilePath))\r
1195                 throw new ArgumentNullException("localFilePath");\r
1196             Contract.EndContractBlock();\r
1197 \r
1198             StatusKeeper.SetFileState(localFilePath, FileStatus.Conflict, FileOverlayStatus.Conflict, "File changed at the server");\r
1199             UpdateStatus(PithosStatus.HasConflicts);\r
1200             var message = String.Format("Conflict detected for file {0}", localFilePath);\r
1201             Log.Warn(message);\r
1202             StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
1203         }\r
1204 \r
1205         private void ReportConflictForDoubleRename(string localFilePath)\r
1206         {\r
1207             if (String.IsNullOrWhiteSpace(localFilePath))\r
1208                 throw new ArgumentNullException("localFilePath");\r
1209             Contract.EndContractBlock();\r
1210 \r
1211             StatusKeeper.SetFileState(localFilePath, FileStatus.Conflict, FileOverlayStatus.Conflict, "File renamed both locally and on the server");\r
1212             UpdateStatus(PithosStatus.HasConflicts);\r
1213             var message = String.Format("Double rename conflict detected for file {0}", localFilePath);\r
1214             Log.Warn(message);\r
1215             StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
1216         }\r
1217 \r
1218 \r
1219         /// <summary>\r
1220         /// Notify the UI to update the visual status\r
1221         /// </summary>\r
1222         /// <param name="status"></param>\r
1223         private void UpdateStatus(PithosStatus status)\r
1224         {\r
1225             try\r
1226             {\r
1227                 StatusNotification.SetPithosStatus(status);\r
1228                 //StatusNotification.Notify(new Notification());\r
1229             }\r
1230             catch (Exception exc)\r
1231             {\r
1232                 //Failure is not critical, just log it\r
1233                 Log.Warn("Error while updating status", exc);\r
1234             }\r
1235         }\r
1236 \r
1237         private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)\r
1238         {\r
1239             var containerPaths = from container in containers\r
1240                                  let containerPath = Path.Combine(accountInfo.AccountPath, container.Name.ToUnescapedString())\r
1241                                  where container.Name.ToString() != FolderConstants.TrashContainer && !Directory.Exists(containerPath)\r
1242                                  select containerPath;\r
1243 \r
1244             foreach (var path in containerPaths)\r
1245             {\r
1246                 Directory.CreateDirectory(path);\r
1247             }\r
1248         }\r
1249 \r
1250         public void AddAccount(AccountInfo accountInfo)\r
1251         {\r
1252             //Avoid adding a duplicate accountInfo\r
1253             _accounts.TryAdd(accountInfo.AccountKey, accountInfo);\r
1254         }\r
1255 \r
1256         public void RemoveAccount(AccountInfo accountInfo)\r
1257         {\r
1258             AccountInfo account;\r
1259             _accounts.TryRemove(accountInfo.AccountKey, out account);\r
1260 \r
1261             SnapshotDifferencer differencer;\r
1262             _differencer.Differencers.TryRemove(accountInfo.AccountKey, out differencer);\r
1263         }\r
1264        \r
1265     }\r
1266 }\r