code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/PollAgent.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="PollAgent.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 \r
  43 using System.Collections.Concurrent;\r
  44 using System.ComponentModel.Composition;\r
  45 using System.Diagnostics;\r
  46 using System.Diagnostics.Contracts;\r
  47 using System.IO;\r
  48 using System.Linq.Expressions;\r
  49 using System.Reflection;\r
  50 using System.Security.Cryptography;\r
  51 using System.Threading;\r
  52 using System.Threading.Tasks;\r
  53 using System.Threading.Tasks.Dataflow;\r
  54 using Castle.ActiveRecord;\r
  55 using Pithos.Interfaces;\r
  56 using Pithos.Network;\r
  57 using log4net;\r
  58 \r
  59 namespace Pithos.Core.Agents\r
  60 {\r
  61     using System;\r
  62     using System.Collections.Generic;\r
  63     using System.Linq;\r
  64 \r
  65     /*public class PollRequest\r
  66     {\r
  67         public DateTime? Since { get; set; }\r
  68         public IEnumerable<string> Batch { get; set; }\r
  69     }*/\r
  70 \r
  71 \r
  72     /// <summary>\r
  73     /// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all\r
  74     /// objects and compares it with a previously cached version to detect differences. \r
  75     /// New files are downloaded, missing files are deleted from the local file system and common files are compared\r
  76     /// to determine the appropriate action\r
  77     /// </summary>\r
  78     [Export]\r
  79     public class PollAgent\r
  80     {\r
  81         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  82 \r
  83         [System.ComponentModel.Composition.Import]\r
  84         public IStatusKeeper StatusKeeper { get; set; }\r
  85 \r
  86         [System.ComponentModel.Composition.Import]\r
  87         public IPithosSettings Settings { get; set; }\r
  88 \r
  89         [System.ComponentModel.Composition.Import]\r
  90         public NetworkAgent NetworkAgent { get; set; }\r
  91 \r
  92         [System.ComponentModel.Composition.Import]\r
  93         public Selectives Selectives { get; set; }\r
  94 \r
  95         public IStatusNotification StatusNotification { get; set; }\r
  96 \r
  97         private CancellationTokenSource _currentOperationCancellation = new CancellationTokenSource();\r
  98 \r
  99         public void CancelCurrentOperation()\r
 100         {\r
 101             //What does it mean to cancel the current upload/download?\r
 102             //Obviously, the current operation will be cancelled by throwing\r
 103             //a cancellation exception.\r
 104             //\r
 105             //The default behavior is to retry any operations that throw.\r
 106             //Obviously this is not what we want in this situation.\r
 107             //The cancelled operation should NOT bea retried. \r
 108             //\r
 109             //This can be done by catching the cancellation exception\r
 110             //and avoiding the retry.\r
 111             //\r
 112 \r
 113             //Have to reset the cancellation source - it is not possible to reset the source\r
 114             //Have to prevent a case where an operation requests a token from the old source\r
 115             var oldSource = Interlocked.Exchange(ref _currentOperationCancellation, new CancellationTokenSource());\r
 116             oldSource.Cancel();\r
 117 \r
 118         }\r
 119 \r
 120         public bool Pause\r
 121         {\r
 122             get {\r
 123                 return _pause;\r
 124             }\r
 125             set {\r
 126                 _pause = value;                \r
 127                 if (!_pause)\r
 128                     _unPauseEvent.Set();\r
 129                 else\r
 130                 {\r
 131                     _unPauseEvent.Reset();\r
 132                 }\r
 133             }\r
 134         }\r
 135 \r
 136         public CancellationToken CancellationToken\r
 137         {\r
 138             get { return _currentOperationCancellation.Token; }\r
 139         }\r
 140 \r
 141         private bool _firstPoll = true;\r
 142 \r
 143         //The Sync Event signals a manual synchronisation\r
 144         private readonly AsyncManualResetEvent _syncEvent = new AsyncManualResetEvent();\r
 145 \r
 146         private readonly AsyncManualResetEvent _unPauseEvent = new AsyncManualResetEvent(true);\r
 147 \r
 148         private readonly ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>();\r
 149         private readonly ConcurrentDictionary<Uri, AccountInfo> _accounts = new ConcurrentDictionary<Uri,AccountInfo>();\r
 150 \r
 151         //private readonly ActionBlock<PollRequest>  _pollAction;\r
 152 \r
 153         readonly HashSet<string> _knownContainers = new HashSet<string>();\r
 154 \r
 155         \r
 156         /// <summary>\r
 157         /// Start a manual synchronization\r
 158         /// </summary>\r
 159         public void SynchNow(IEnumerable<string> paths=null)\r
 160         {\r
 161             _batchQueue.Enqueue(paths);\r
 162             _syncEvent.Set();                \r
 163 \r
 164             //_pollAction.Post(new PollRequest {Batch = paths});\r
 165         }\r
 166 \r
 167         readonly ConcurrentQueue<IEnumerable<string>> _batchQueue=new ConcurrentQueue<IEnumerable<string>>();\r
 168 \r
 169         ConcurrentDictionary<string,MovedEventArgs> _moves=new ConcurrentDictionary<string, MovedEventArgs>(); \r
 170 \r
 171         public void PostMove(MovedEventArgs args)\r
 172         {\r
 173             TaskEx.Run(() => _moves.AddOrUpdate(args.OldFullPath, args,(s,e)=>e));            \r
 174         }\r
 175 \r
 176         /// <summary>\r
 177         /// Remote files are polled periodically. Any changes are processed\r
 178         /// </summary>\r
 179         /// <param name="since"></param>\r
 180         /// <returns></returns>\r
 181         public  void PollRemoteFiles(DateTime? since = null)\r
 182         {\r
 183             if (Log.IsDebugEnabled)\r
 184                 Log.DebugFormat("Polling changes after [{0}]",since);\r
 185 \r
 186             Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
 187 \r
 188             //GC.Collect();\r
 189 \r
 190             using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
 191             {\r
 192                 //If this poll fails, we will retry with the same since value\r
 193                 var nextSince = since;\r
 194                 try\r
 195                 {\r
 196                     _unPauseEvent.Wait();\r
 197                     UpdateStatus(PithosStatus.PollSyncing);\r
 198 \r
 199                     var accountBatches=new Dictionary<Uri, IEnumerable<string>>();\r
 200                     IEnumerable<string> batch = null;\r
 201                     if (_batchQueue.TryDequeue(out batch) && batch != null)\r
 202                         foreach (var account in _accounts.Values)\r
 203                         {\r
 204                             var accountBatch = batch.Where(path => path.IsAtOrBelow(account.AccountPath));\r
 205                             accountBatches[account.AccountKey] = accountBatch;\r
 206                         }\r
 207                     \r
 208                     var moves=Interlocked.Exchange(ref _moves, new ConcurrentDictionary<string, MovedEventArgs>());\r
 209 \r
 210                     var tasks = new List<Task<DateTime?>>();\r
 211                     foreach(var accountInfo in _accounts.Values)\r
 212                     {\r
 213                         IEnumerable<string> accountBatch ;\r
 214                         accountBatches.TryGetValue(accountInfo.AccountKey,out accountBatch);\r
 215                         var t=ProcessAccountFiles (accountInfo, accountBatch, moves,since);\r
 216                         tasks.Add(t);\r
 217                     }\r
 218 \r
 219                     var nextTimes=TaskEx.WhenAll(tasks.ToList()).Result;\r
 220 \r
 221                     _firstPoll = false;\r
 222                     //Reschedule the poll with the current timestamp as a "since" value\r
 223 \r
 224                     if (nextTimes.Length>0)\r
 225                         nextSince = nextTimes.Min();\r
 226                     if (Log.IsDebugEnabled)\r
 227                         Log.DebugFormat("Next Poll at [{0}]",nextSince);\r
 228                 }\r
 229                 catch (Exception ex)\r
 230                 {\r
 231                     Log.ErrorFormat("Error while processing accounts\r\n{0}", ex);\r
 232                     //In case of failure retry with the same "since" value\r
 233                 }\r
 234 \r
 235                 UpdateStatus(PithosStatus.PollComplete);\r
 236                 //The multiple try blocks are required because we can't have an await call\r
 237                 //inside a finally block\r
 238                 //TODO: Find a more elegant solution for reschedulling in the event of an exception\r
 239                 try\r
 240                 {\r
 241                     //Wait for the polling interval to pass or the Sync event to be signalled\r
 242                     nextSince = WaitForScheduledOrManualPoll(nextSince).Result;\r
 243                 }\r
 244                 finally\r
 245                 {\r
 246                     //Ensure polling is scheduled even in case of error\r
 247                     TaskEx.Run(()=>PollRemoteFiles(nextSince));\r
 248                     //_pollAction.Post(new PollRequest {Since = nextSince});\r
 249                 }\r
 250             }\r
 251         }\r
 252 \r
 253         /// <summary>\r
 254         /// Wait for the polling period to expire or a manual sync request\r
 255         /// </summary>\r
 256         /// <param name="since"></param>\r
 257         /// <returns></returns>\r
 258         private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
 259         {\r
 260             var sync = _syncEvent.WaitAsync();\r
 261             var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval));\r
 262 \r
 263             var signaledTask = await TaskEx.WhenAny(sync, wait).ConfigureAwait(false);\r
 264             \r
 265             //Pausing takes precedence over manual sync or awaiting\r
 266             _unPauseEvent.Wait();\r
 267             \r
 268             //Wait for network processing to finish before polling\r
 269             var pauseTask=NetworkAgent.ProceedEvent.WaitAsync();\r
 270             await TaskEx.WhenAll(signaledTask, pauseTask).ConfigureAwait(false);\r
 271 \r
 272             //If polling is signalled by SynchNow, ignore the since tag\r
 273             if (sync.IsCompleted)\r
 274             {                \r
 275                 _syncEvent.Reset();\r
 276                 return null;\r
 277             }\r
 278             return since;\r
 279         }\r
 280 \r
 281         \r
 282 \r
 283         public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, IEnumerable<string> accountBatch, ConcurrentDictionary<string, MovedEventArgs> moves, DateTime? since = null)\r
 284         {\r
 285             if (accountInfo == null)\r
 286                 throw new ArgumentNullException("accountInfo");\r
 287             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
 288                 throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
 289             Contract.EndContractBlock();\r
 290 \r
 291 \r
 292             using (ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))\r
 293             {\r
 294 \r
 295                 await NetworkAgent.GetDeleteAwaiter().ConfigureAwait(false);\r
 296 \r
 297                 Log.Info("Scheduled");\r
 298                 var client = new CloudFilesClient(accountInfo);\r
 299 \r
 300                 //We don't need to check the trash container\r
 301                 var containers = client.ListContainers(accountInfo.UserName)\r
 302                     .Where(c=>c.Name!="trash")\r
 303                     .ToList();\r
 304 \r
 305 \r
 306                 CreateContainerFolders(accountInfo, containers);\r
 307 \r
 308                 //The nextSince time fallback time is the same as the current.\r
 309                 //If polling succeeds, the next Since time will be the smallest of the maximum modification times\r
 310                 //of the shared and account objects\r
 311                 var nextSince = since;\r
 312 \r
 313                 try\r
 314                 {\r
 315                     //Wait for any deletions to finish\r
 316                     await NetworkAgent.GetDeleteAwaiter().ConfigureAwait(false);\r
 317                     //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
 318                     //than delete a file that was created while we were executing the poll                    \r
 319 \r
 320                     //Get the list of server objects changed since the last check\r
 321                     //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
 322                     var listObjects = (from container in containers\r
 323                                        select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
 324                                              client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
 325 \r
 326                     var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
 327                         client.ListSharedObjects(_knownContainers,since), "shared");\r
 328                     listObjects.Add(listShared);\r
 329                     var listTasks = await Task.Factory.WhenAll(listObjects.ToArray()).ConfigureAwait(false);\r
 330 \r
 331                     using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
 332                     {\r
 333                         var dict = listTasks.ToDictionary(t => t.AsyncState);\r
 334 \r
 335                         //Get all non-trash objects. Remember, the container name is stored in AsyncState\r
 336                         var remoteObjects = (from objectList in listTasks\r
 337                                             where (string)objectList.AsyncState != "trash"\r
 338                                             from obj in objectList.Result\r
 339                                             orderby obj.Bytes ascending \r
 340                                             select obj).ToList();\r
 341                         \r
 342                         //Get the latest remote object modification date, only if it is after\r
 343                         //the original since date                        \r
 344                         nextSince = GetLatestDateAfter(nextSince, remoteObjects);\r
 345 \r
 346                         var sharedObjects = dict["shared"].Result;\r
 347 \r
 348                         //DON'T process trashed files\r
 349                         //If some files are deleted and added again to a folder, they will be deleted\r
 350                         //even though they are new.\r
 351                         //We would have to check file dates and hashes to ensure that a trashed file\r
 352                         //can be deleted safely from the local hard drive.\r
 353                         /*\r
 354                         //Items with the same name, hash may be both in the container and the trash\r
 355                         //Don't delete items that exist in the container\r
 356                         var realTrash = from trash in trashObjects\r
 357                                         where\r
 358                                             !remoteObjects.Any(\r
 359                                                 info => info.Name == trash.Name && info.Hash == trash.Hash)\r
 360                                    8     select trash;\r
 361                         ProcessTrashedFiles(accountInfo, realTrash);\r
 362 */\r
 363 \r
 364                         var cleanRemotes = (from info in remoteObjects.Union(sharedObjects)\r
 365                                             let name = info.Name??""\r
 366                                             where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&\r
 367                                                   !name.StartsWith(FolderConstants.CacheFolder + "/",\r
 368                                                                    StringComparison.InvariantCultureIgnoreCase)\r
 369                                             select info).ToList();\r
 370 \r
 371                         if (_firstPoll)\r
 372                             StatusKeeper.CleanupOrphanStates();\r
 373                         \r
 374                         var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
 375                         var currentRemotes = differencer.Current.ToList();\r
 376                         StatusKeeper.CleanupStaleStates(accountInfo, currentRemotes);\r
 377 \r
 378                         //var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
 379 \r
 380                         //May have to wait if the FileAgent has asked for a Pause, due to local changes\r
 381                         await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
 382 \r
 383                         //Get the local files here                        \r
 384                         var agent = AgentLocator<FileAgent>.Get(accountInfo.AccountPath);                                                \r
 385                         var files = LoadLocalFileTuples(accountInfo, accountBatch);\r
 386 \r
 387                         var states = FileState.Queryable.ToList();                        \r
 388                         \r
 389                         var infos = (from remote in currentRemotes\r
 390                                     let path = remote.RelativeUrlToFilePath(accountInfo.UserName)\r
 391                                     let info=agent.GetFileSystemInfo(path)\r
 392                                     select Tuple.Create(info.FullName,remote))\r
 393                                     .ToList();\r
 394 \r
 395                         var token = _currentOperationCancellation.Token;\r
 396 \r
 397                         var tuples = MergeSources(infos, files, states,moves).ToList();\r
 398 \r
 399                         var processedPaths = new HashSet<string>();\r
 400                         //Process only the changes in the batch file, if one exists\r
 401                         var stateTuples = accountBatch==null?tuples:tuples.Where(t => accountBatch.Contains(t.FilePath));\r
 402                         foreach (var tuple in stateTuples.Where(s=>!s.Locked))\r
 403                         {\r
 404                             await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
 405 \r
 406                             //Set the Merkle Hash\r
 407                             //SetMerkleHash(accountInfo, tuple);\r
 408 \r
 409                             await SyncSingleItem(accountInfo, tuple, agent, moves,processedPaths,token).ConfigureAwait(false);\r
 410 \r
 411                         }\r
 412 \r
 413 \r
 414                         //On the first run\r
 415 /*\r
 416                         if (_firstPoll)\r
 417                         {\r
 418                             MarkSuspectedDeletes(accountInfo, cleanRemotes);\r
 419                         }\r
 420 */\r
 421 \r
 422 \r
 423                         Log.Info("[LISTENER] End Processing");\r
 424                     }\r
 425                 }\r
 426                 catch (Exception ex)\r
 427                 {\r
 428                     Log.ErrorFormat("[FAIL] ListObjects for {0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);\r
 429                     return nextSince;\r
 430                 }\r
 431 \r
 432                 Log.Info("[LISTENER] Finished");\r
 433                 return nextSince;\r
 434             }\r
 435         }\r
 436 /*\r
 437 \r
 438         private static void SetMerkleHash(AccountInfo accountInfo, StateTuple tuple)\r
 439         {\r
 440             //The Merkle hash for directories is that of an empty buffer\r
 441             if (tuple.FileInfo is DirectoryInfo)\r
 442                 tuple.C = MERKLE_EMPTY;\r
 443             else if (tuple.FileState != null && tuple.MD5 == tuple.FileState.ETag)\r
 444             {\r
 445                 //If there is a state whose MD5 matches, load the merkle hash from the file state\r
 446                 //insteaf of calculating it\r
 447                 tuple.C = tuple.FileState.Checksum;                              \r
 448             }\r
 449             else\r
 450             {\r
 451                 tuple.Merkle = Signature.CalculateTreeHashAsync((FileInfo)tuple.FileInfo, accountInfo.BlockSize, accountInfo.BlockHash,1,progress);\r
 452                 //tuple.C=tuple.Merkle.TopHash.ToHashString();                \r
 453             }\r
 454         }\r
 455 */\r
 456 \r
 457         private IEnumerable<FileSystemInfo> LoadLocalFileTuples(AccountInfo accountInfo,IEnumerable<string> batch )\r
 458         {\r
 459             using (ThreadContext.Stacks["Account Files Hashing"].Push(accountInfo.UserName))\r
 460             {\r
 461                 var batchPaths = (batch==null)?new List<string>():batch.ToList();\r
 462                 IEnumerable<FileSystemInfo> localInfos=AgentLocator<FileAgent>.Get(accountInfo.AccountPath)\r
 463                                                         .EnumerateFileSystemInfos();\r
 464                 if (batchPaths.Count>0)\r
 465                     localInfos= localInfos.Where(fi => batchPaths.Contains(fi.FullName));\r
 466 \r
 467                 return localInfos;\r
 468             }\r
 469         }\r
 470 \r
 471         /// <summary>\r
 472         /// Wait and Pause the agent while waiting\r
 473         /// </summary>\r
 474         /// <param name="backoff"></param>\r
 475         /// <returns></returns>\r
 476         private async Task PauseFor(int backoff)\r
 477         {\r
 478 \r
 479             Pause = true;\r
 480             await TaskEx.Delay(backoff).ConfigureAwait(false);\r
 481             Pause = false;\r
 482         }\r
 483 \r
 484         private async Task SyncSingleItem(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves,HashSet<string> processedPaths, CancellationToken token)\r
 485         {\r
 486             Log.DebugFormat("Sync [{0}] C:[{1}] L:[{2}] S:[{3}]", tuple.FilePath, tuple.C, tuple.L, tuple.S);\r
 487 \r
 488             //If the processed paths already contain the current path, exit\r
 489             if (!processedPaths.Add(tuple.FilePath))\r
 490                 return;\r
 491 \r
 492             try\r
 493             {\r
 494                 bool isInferredParent = tuple.ObjectInfo != null && tuple.ObjectInfo.UUID.StartsWith("00000000-0000-0000");\r
 495 \r
 496                 var localFilePath = tuple.FilePath;\r
 497                 //Don't use the tuple info, it may have been deleted\r
 498                 var localInfo = FileInfoExtensions.FromPath(localFilePath);\r
 499 \r
 500 \r
 501                 var isUnselectedRootFolder = agent.IsUnselectedRootFolder(tuple.FilePath);\r
 502 \r
 503                 //Unselected root folders that have not yet been uploaded should be uploaded and added to the \r
 504                 //selective folders\r
 505 \r
 506                 if (!Selectives.IsSelected(accountInfo, localFilePath) &&\r
 507                     !(isUnselectedRootFolder && tuple.ObjectInfo == null))\r
 508                     return;\r
 509 \r
 510                 // Local file unchanged? If both C and L are null, make sure it's because \r
 511                 //both the file is missing and the state checksum is not missing\r
 512                 if (tuple.C == tuple.L /*&& (localInfo.Exists || tuple.FileState == null)*/)\r
 513                 {\r
 514                     //No local changes\r
 515                     //Server unchanged?\r
 516                     if (tuple.S == tuple.L)\r
 517                     {\r
 518                         // No server changes\r
 519                         //Has the file been renamed on the server?\r
 520                         MoveForServerMove(accountInfo, tuple);\r
 521                     }\r
 522                     else\r
 523                     {\r
 524                         //Different from server\r
 525                         //Does the server file exist?\r
 526                         if (tuple.S == null)\r
 527                         {\r
 528                             //Server file doesn't exist\r
 529                             //deleteObjectFromLocal()\r
 530                             using (\r
 531                                 StatusNotification.GetNotifier("Deleting local {0}", "Deleted local {0}",\r
 532                                                                localInfo.Name))\r
 533                             {\r
 534                                 DeleteLocalFile(agent, localFilePath);\r
 535                             }\r
 536                         }\r
 537                         else\r
 538                         {\r
 539                             //Server file exists\r
 540                             //downloadServerObject() // Result: L = S\r
 541                             //If the file has moved on the server, move it locally before downloading\r
 542                             using (\r
 543                                 StatusNotification.GetNotifier("Downloading {0}", "Downloaded {0}",\r
 544                                                                localInfo.Name))\r
 545                             {\r
 546                                 var targetPath = MoveForServerMove(accountInfo, tuple);\r
 547 \r
 548                                 await DownloadCloudFile(accountInfo, tuple, token, targetPath).ConfigureAwait(false);\r
 549 \r
 550                                 AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
 551                             }\r
 552 \r
 553                             /*\r
 554                                                             StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
 555                                                                                       FileOverlayStatus.Normal, "");\r
 556                                 */\r
 557                         }\r
 558                     }\r
 559                 }\r
 560                 else\r
 561                 {                   \r
 562 \r
 563                     //Local changes found\r
 564 \r
 565                     //Server unchanged?\r
 566                     if (tuple.S == tuple.L)\r
 567                     {\r
 568                         //The FileAgent selective sync checks for new root folder files\r
 569                         if (!agent.Ignore(localFilePath))\r
 570                         {\r
 571                             if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
 572                             {\r
 573                                 //deleteObjectFromServer()\r
 574                                 DeleteCloudFile(accountInfo, tuple);\r
 575                                 //updateRecord( Remove L, S)                  \r
 576                             }\r
 577                             else\r
 578                             {\r
 579                                 //uploadLocalObject() // Result: S = C, L = S                        \r
 580                                 var progress = new Progress<double>(d =>\r
 581                                     StatusNotification.Notify(new StatusNotification(String.Format("Merkle Hashing for Upload {0:p} of {1}", d, localInfo.Name))));\r
 582 \r
 583                                 //Is it an unselected root folder\r
 584                                 var isCreation = isUnselectedRootFolder ||//or a new folder under a selected parent?\r
 585                                         (localInfo is DirectoryInfo && Selectives.IsSelected(accountInfo, localInfo) && tuple.FileState == null && tuple.ObjectInfo == null);\r
 586 \r
 587 \r
 588                                 //Is this a result of a FILE move with no modifications? Then try to move it,\r
 589                                 //to avoid an expensive hash\r
 590                                 if (!MoveForLocalMove(accountInfo, tuple))\r
 591                                 {\r
 592                                     await UploadLocalFile(accountInfo, tuple, token, isCreation, localInfo,processedPaths, progress).ConfigureAwait(false);\r
 593                                 }\r
 594 \r
 595                                 //updateRecord( S = C )\r
 596                                 //State updated by the uploader\r
 597                                 \r
 598                                 if (isCreation )\r
 599                                 {                                    \r
 600                                     ProcessChildren(accountInfo, tuple, agent, moves,processedPaths,token);\r
 601                                 }\r
 602                             }\r
 603                         }\r
 604                     }\r
 605                     else\r
 606                     {\r
 607                         if (tuple.C == tuple.S)\r
 608                         {\r
 609                             // (Identical Changes) Result: L = S\r
 610                             //doNothing()\r
 611                             \r
 612                             //Don't update anything for nonexistend server files\r
 613                             if (tuple.S != null)\r
 614                             {\r
 615                                 //Detect server moves\r
 616                                 var targetPath = MoveForServerMove(accountInfo, tuple);\r
 617                                 Debug.Assert(tuple.Merkle!=null);\r
 618                                 StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo,tuple.Merkle);\r
 619                                 \r
 620                                 AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
 621                             }\r
 622                             else\r
 623                             {\r
 624                                 //At this point, C==S==NULL and we have a stale state (L)\r
 625                                 //Log the stale tuple for investigation\r
 626                                 Log.WarnFormat("Stale tuple detected FilePathPath:[{0}], State:[{1}], LocalFile:[{2}]", tuple.FilePath, tuple.FileState, tuple.FileInfo);\r
 627 \r
 628                                 //And remove it\r
 629                                 if (!String.IsNullOrWhiteSpace(tuple.FilePath))\r
 630                                     StatusKeeper.ClearFileStatus(tuple.FilePath);\r
 631                             }\r
 632                         }\r
 633                         else\r
 634                         {\r
 635                             if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
 636                             {\r
 637                                 //deleteObjectFromServer()\r
 638                                 DeleteCloudFile(accountInfo, tuple);\r
 639                                 //updateRecord(Remove L, S)                  \r
 640                             }\r
 641                                 //If both the local and server files are missing, the state is stale\r
 642                             else if (!localInfo.Exists && (tuple.S == null || tuple.ObjectInfo == null))\r
 643                             {\r
 644                                 StatusKeeper.ClearFileStatus(localInfo.FullName);\r
 645                             }\r
 646                             else\r
 647                             {\r
 648                                 ReportConflictForMismatch(localFilePath);\r
 649                                 //identifyAsConflict() // Manual action required\r
 650                             }\r
 651                         }\r
 652                     }\r
 653                 }\r
 654             }\r
 655             catch (Exception exc)\r
 656             {\r
 657                 //In case of error log and retry with the next poll\r
 658                 Log.ErrorFormat("[SYNC] Failed for file {0}. Will Retry.\r\n{1}",tuple.FilePath,exc);\r
 659             }\r
 660         }\r
 661 \r
 662         private void DeleteLocalFile(FileAgent agent, string localFilePath)\r
 663         {\r
 664             StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
 665                                       FileOverlayStatus.Deleted, "");\r
 666             using (NetworkGate.Acquire(localFilePath, NetworkOperation.Deleting))\r
 667             {\r
 668                 agent.Delete(localFilePath);\r
 669             }\r
 670             //updateRecord(Remove C, L)\r
 671             StatusKeeper.ClearFileStatus(localFilePath);\r
 672         }\r
 673 \r
 674         private async Task DownloadCloudFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token, string targetPath)\r
 675         {\r
 676             StatusKeeper.SetFileState(targetPath, FileStatus.Modified, FileOverlayStatus.Modified,\r
 677                                       "");\r
 678 \r
 679             var finalHash=await\r
 680                 NetworkAgent.Downloader.DownloadCloudFile(accountInfo, tuple.ObjectInfo, targetPath,\r
 681                                                           token)\r
 682                     .ConfigureAwait(false);\r
 683             //updateRecord( L = S )\r
 684             StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
 685                                             finalHash);\r
 686 \r
 687             StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo,finalHash);\r
 688         }\r
 689 \r
 690         private async Task UploadLocalFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token,\r
 691                                      bool isUnselectedRootFolder, FileSystemInfo localInfo, HashSet<string> processedPaths, Progress<double> progress)\r
 692         {\r
 693             var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
 694                                                accountInfo.BlockSize, accountInfo.BlockHash,\r
 695                                                "Poll", isUnselectedRootFolder, token, progress,tuple.Merkle);            \r
 696             \r
 697             using (StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}",\r
 698                                                   localInfo.Name))\r
 699             {\r
 700                 await NetworkAgent.Uploader.UploadCloudFile(action, token).ConfigureAwait(false);\r
 701             }\r
 702 \r
 703             if (isUnselectedRootFolder)\r
 704             {\r
 705                 var dirActions =(\r
 706                     from dir in ((DirectoryInfo) localInfo).EnumerateDirectories("*", SearchOption.AllDirectories)\r
 707                     let subAction = new CloudUploadAction(accountInfo, dir, null,\r
 708                                                           accountInfo.BlockSize, accountInfo.BlockHash,\r
 709                                                           "Poll", true, token, progress)\r
 710                     select subAction).ToList();\r
 711                 foreach (var dirAction in dirActions)\r
 712                 {\r
 713                     processedPaths.Add(dirAction.LocalFile.FullName);\r
 714                 }\r
 715                 \r
 716                 await TaskEx.WhenAll(dirActions.Select(a=>NetworkAgent.Uploader.UploadCloudFile(a,token)).ToArray());\r
 717             }\r
 718         }\r
 719 \r
 720         private bool MoveForLocalMove(AccountInfo accountInfo, StateTuple tuple)\r
 721         {\r
 722             if (!(tuple.FileInfo is FileInfo) || String.IsNullOrWhiteSpace(tuple.OldFullPath) || tuple.OldMD5 != tuple.C)\r
 723                 return false;\r
 724 \r
 725             try\r
 726             {\r
 727 \r
 728                 var client = new CloudFilesClient(accountInfo);\r
 729                 var objectInfo = CloudAction.CreateObjectInfoFor(accountInfo, tuple.FileInfo);\r
 730                 var containerPath = Path.Combine(accountInfo.AccountPath, objectInfo.Container);\r
 731                 var oldName = tuple.OldFullPath.AsRelativeTo(containerPath);\r
 732                 //Then execute a move instead of an upload\r
 733                 using (StatusNotification.GetNotifier("Moving {0}", "Moved {0}", tuple.FileInfo.Name))\r
 734                 {\r
 735                     client.MoveObject(objectInfo.Account, objectInfo.Container, oldName,\r
 736                                                           objectInfo.Container, objectInfo.Name);\r
 737                 }\r
 738                 return true;\r
 739             }\r
 740             catch (Exception exc)\r
 741             {\r
 742                 Log.ErrorFormat("[MOVE] Failed for [{0}],:\r\n{1}", tuple.FilePath, exc);\r
 743                 //Return false to force an upload of the file\r
 744                 return false;\r
 745             }\r
 746 \r
 747         }\r
 748 \r
 749         private void AddOwnFolderToSelectives(AccountInfo accountInfo, StateTuple tuple, string targetPath)\r
 750         {\r
 751             //Not for shared folders\r
 752             if (tuple.ObjectInfo.IsShared==true)\r
 753                 return;\r
 754             //Also ensure that any newly created folders are added to the selectives, if the original folder was selected\r
 755             var containerPath = Path.Combine(accountInfo.AccountPath, tuple.ObjectInfo.Container);\r
 756 \r
 757             //If this is a root folder encountered for the first time\r
 758             if (tuple.L == null && Directory.Exists(tuple.FileInfo.FullName) \r
 759                 && (tuple.FileInfo.FullName.IsAtOrBelow(containerPath)))\r
 760             {\r
 761                 \r
 762                 var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 763                 var initialPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
 764 \r
 765                 //var hasMoved = true;// !initialPath.Equals(targetPath);\r
 766                 //If the new path is under a selected folder, add it to the selectives as well\r
 767                 if (Selectives.IsSelected(accountInfo, initialPath))\r
 768                 {\r
 769                     Selectives.AddUri(accountInfo, tuple.ObjectInfo.Uri);\r
 770                     Selectives.Save(accountInfo);\r
 771                 }\r
 772             }\r
 773         }\r
 774 \r
 775         private string MoveForServerMove(AccountInfo accountInfo, StateTuple tuple)\r
 776         {\r
 777             if (tuple.ObjectInfo == null)\r
 778                 return null;\r
 779             var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 780             var serverPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
 781             \r
 782             //Compare Case Insensitive\r
 783             if (String.Equals(tuple.FilePath ,serverPath,StringComparison.InvariantCultureIgnoreCase)) \r
 784                 return serverPath;\r
 785 \r
 786             tuple.FileInfo.Refresh();\r
 787             //The file/folder may not exist if it was moved because its parent moved\r
 788             if (!tuple.FileInfo.Exists)\r
 789             {\r
 790                 var target=FileInfoExtensions.FromPath(serverPath);\r
 791                 if (!target.Exists)\r
 792                 {\r
 793                     Log.ErrorFormat("No source or target found while trying to move {0} to {1}", tuple.FileInfo.FullName, serverPath);\r
 794                 }\r
 795                 return serverPath;\r
 796             }\r
 797 \r
 798             using (StatusNotification.GetNotifier("Moving local {0}", "Moved local {0}", Path.GetFileName(tuple.FilePath)))\r
 799             using(NetworkGate.Acquire(tuple.FilePath,NetworkOperation.Renaming))\r
 800             {\r
 801                     \r
 802                 var fi = tuple.FileInfo as FileInfo;\r
 803                 if (fi != null)\r
 804                 {\r
 805                     var targetFile = new FileInfo(serverPath);\r
 806                     if (!targetFile.Directory.Exists)\r
 807                         targetFile.Directory.Create();\r
 808                     fi.MoveTo(serverPath);\r
 809                 }\r
 810                 var di = tuple.FileInfo as DirectoryInfo;\r
 811                 if (di != null)\r
 812                 {\r
 813                     var targetDir = new DirectoryInfo(serverPath);\r
 814                     if (!targetDir.Parent.Exists)\r
 815                         targetDir.Parent.Create();\r
 816                     di.MoveTo(serverPath);\r
 817                 }\r
 818             }\r
 819 \r
 820             StatusKeeper.StoreInfo(serverPath, tuple.ObjectInfo,null);\r
 821 \r
 822             return serverPath;\r
 823         }\r
 824 \r
 825         private void DeleteCloudFile(AccountInfo accountInfo, StateTuple tuple)\r
 826         {\r
 827             using (StatusNotification.GetNotifier("Deleting server {0}", "Deleted server {0}", Path.GetFileName(tuple.FilePath)))\r
 828             {\r
 829 \r
 830                 StatusKeeper.SetFileState(tuple.FilePath, FileStatus.Deleted,\r
 831                                           FileOverlayStatus.Deleted, "");\r
 832                 NetworkAgent.DeleteAgent.DeleteCloudFile(accountInfo, tuple.ObjectInfo);\r
 833                 StatusKeeper.ClearFileStatus(tuple.FilePath);\r
 834             }\r
 835         }\r
 836 \r
 837         private void ProcessChildren(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves,HashSet<string> processedPaths,CancellationToken token)\r
 838         {\r
 839 \r
 840             var dirInfo = tuple.FileInfo as DirectoryInfo;\r
 841             var folderTuples = from folder in dirInfo.EnumerateDirectories("*", SearchOption.AllDirectories)\r
 842                                select new StateTuple(folder){C=Signature.MERKLE_EMPTY};\r
 843             \r
 844             var fileTuples = from file in dirInfo.EnumerateFiles("*", SearchOption.AllDirectories)\r
 845                              let state=StatusKeeper.GetStateByFilePath(file.FullName)\r
 846                              select new StateTuple(file){\r
 847                                             Merkle=StatusAgent.CalculateTreeHash(file,accountInfo,state,\r
 848                                             Settings.HashingParallelism,token,null)\r
 849                                         };\r
 850             \r
 851             //Process folders first, to ensure folders appear on the sever as soon as possible\r
 852             folderTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, moves, processedPaths,token).Wait());\r
 853             \r
 854             fileTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, moves,processedPaths, token).Wait());\r
 855         }\r
 856 \r
 857 \r
 858         /*\r
 859          *                 //Use the queue to retry locked file hashing\r
 860                 var fileQueue = new ConcurrentQueue<FileSystemInfo>(localInfos);\r
 861                 \r
 862 \r
 863                 var results = new List<Tuple<FileSystemInfo, string>>();\r
 864                 var backoff = 0;\r
 865                 while (fileQueue.Count > 0)\r
 866                 {\r
 867                     FileSystemInfo file;\r
 868                     fileQueue.TryDequeue(out file);\r
 869                     using (ThreadContext.Stacks["File"].Push(file.FullName))\r
 870                     {\r
 871                         try\r
 872                         {\r
 873                             //Replace MD5 here, do the calc while syncing individual files\r
 874                             string hash ;\r
 875                             if (file is DirectoryInfo)\r
 876                                 hash = MD5_EMPTY;\r
 877                             else\r
 878                             {\r
 879                                 //Wait in case the FileAgent has requested a Pause\r
 880                                 await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
 881                                 \r
 882                                 using (StatusNotification.GetNotifier("Hashing {0}", "", file.Name))\r
 883                                 {\r
 884                                     hash = ((FileInfo)file).ComputeShortHash(StatusNotification);\r
 885                                     backoff = 0;\r
 886                                 }\r
 887                             }                            \r
 888                             results.Add(Tuple.Create(file, hash));\r
 889                         }\r
 890                         catch (IOException exc)\r
 891                         {\r
 892                             Log.WarnFormat("[HASH] File in use, will retry [{0}]", exc);\r
 893                             fileQueue.Enqueue(file);\r
 894                             //If this is the only enqueued file                            \r
 895                             if (fileQueue.Count != 1) continue;\r
 896                             \r
 897                             \r
 898                             //Increase delay\r
 899                             if (backoff<60000)\r
 900                                 backoff += 10000;\r
 901                             //Pause Polling for the specified time\r
 902                         }\r
 903                         if (backoff>0)\r
 904                             await PauseFor(backoff).ConfigureAwait(false);\r
 905                     }\r
 906                 }\r
 907 \r
 908                 return results;\r
 909 \r
 910          */\r
 911         private IEnumerable<StateTuple> MergeSources(IEnumerable<Tuple<string, ObjectInfo>> infos, IEnumerable<FileSystemInfo> files, List<FileState> states, ConcurrentDictionary<string, MovedEventArgs> moves)\r
 912         {\r
 913             var tuplesByPath = new Dictionary<string, StateTuple>();\r
 914             foreach (var info in files)\r
 915             {\r
 916                 var tuple = new StateTuple(info);\r
 917                 //Is this the target of a move event?\r
 918                 var moveArg = moves.Values.FirstOrDefault(arg => info.FullName.Equals(arg.FullPath, StringComparison.InvariantCultureIgnoreCase) \r
 919                     || info.FullName.IsAtOrBelow(arg.FullPath));\r
 920                 if (moveArg != null)\r
 921                 {\r
 922                     tuple.NewFullPath = info.FullName;\r
 923                     var relativePath = info.AsRelativeTo(moveArg.FullPath);                    \r
 924                     tuple.OldFullPath = Path.Combine(moveArg.OldFullPath, relativePath);\r
 925                     tuple.OldMD5 = states.FirstOrDefault(st => st.FilePath.Equals(tuple.OldFullPath, StringComparison.InvariantCultureIgnoreCase))\r
 926                                 .NullSafe(st => st.LastMD5);\r
 927                 }\r
 928 \r
 929                 tuplesByPath[tuple.FilePath] = tuple;\r
 930             }\r
 931             \r
 932 \r
 933             \r
 934             \r
 935             //For files that have state\r
 936             foreach (var state in states)\r
 937             {\r
 938                 StateTuple hashTuple;\r
 939 \r
 940                 \r
 941                 if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple))\r
 942                 {\r
 943                     hashTuple.FileState = state;\r
 944                     UpdateHashes(hashTuple);\r
 945                 }\r
 946                 else\r
 947                 {\r
 948                     var fsInfo = FileInfoExtensions.FromPath(state.FilePath);\r
 949                     hashTuple = new StateTuple {FileInfo = fsInfo, FileState = state};\r
 950 \r
 951                     //Is the source of a moved item?\r
 952                     var moveArg = moves.Values.FirstOrDefault(arg => state.FilePath.Equals(arg.OldFullPath,StringComparison.InvariantCultureIgnoreCase) \r
 953                         || state.FilePath.IsAtOrBelow(arg.OldFullPath));\r
 954                     if (moveArg != null)\r
 955                     {\r
 956                         var relativePath = state.FilePath.AsRelativeTo(moveArg.OldFullPath);\r
 957                         hashTuple.NewFullPath = Path.Combine(moveArg.FullPath,relativePath);\r
 958                         hashTuple.OldFullPath = state.FilePath;\r
 959                         //Do we have the old MD5?\r
 960                         hashTuple.OldMD5 = state.LastMD5;\r
 961                     }\r
 962 \r
 963 \r
 964                     tuplesByPath[state.FilePath] = hashTuple;\r
 965                 }\r
 966             }\r
 967             //for files that don't have state\r
 968             foreach (var tuple in tuplesByPath.Values.Where(t => t.FileState == null))\r
 969             {\r
 970                 UpdateHashes(tuple);\r
 971             }\r
 972 \r
 973             var tuplesByID = tuplesByPath.Values\r
 974                 .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null)\r
 975                 .ToDictionary(tuple=>tuple.FileState.ObjectID,tuple=>tuple);//new Dictionary<Guid, StateTuple>();\r
 976 \r
 977             foreach (var info in infos)\r
 978             {\r
 979                 StateTuple hashTuple;\r
 980                 var filePath = info.Item1;\r
 981                 var objectInfo = info.Item2;\r
 982                 var objectID = objectInfo.UUID;\r
 983 \r
 984                 if (objectID != _emptyGuid &&  tuplesByID.TryGetValue(objectID, out hashTuple))\r
 985                 {\r
 986                     hashTuple.ObjectInfo = objectInfo;                    \r
 987                 }\r
 988                 else if (tuplesByPath.TryGetValue(filePath, out hashTuple))\r
 989                 {\r
 990                     hashTuple.ObjectInfo = objectInfo;\r
 991                 }\r
 992                 else\r
 993                 {\r
 994 \r
 995                     \r
 996                     var fsInfo = FileInfoExtensions.FromPath(filePath);\r
 997                     hashTuple= new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo};\r
 998                     tuplesByPath[filePath] = hashTuple;\r
 999                     \r
1000                     if (objectInfo.UUID!=_emptyGuid)\r
1001                         tuplesByID[objectInfo.UUID] = hashTuple;\r
1002                 }\r
1003             }\r
1004             Debug.Assert(tuplesByPath.Values.All(t => t.HashesValid()));\r
1005             return tuplesByPath.Values;\r
1006         }\r
1007 \r
1008         \r
1009         /// <summary>\r
1010         /// Update the tuple with the file's hashes, avoiding calculation if the file is unchanged\r
1011         /// </summary>\r
1012         /// <param name="hashTuple"></param>\r
1013         /// <remarks>\r
1014         /// The function first checks the file's size and last write date to see if there are any changes. If there are none,\r
1015         /// the file's stored hashes are used.\r
1016         /// Otherwise, MD5 is calculated first to ensure there are no changes. If MD5 is different, the Merkle hash is calculated\r
1017         /// </remarks>\r
1018         private void  UpdateHashes(StateTuple hashTuple)\r
1019         {\r
1020             \r
1021             try\r
1022             {\r
1023                 var state = hashTuple.NullSafe(s => s.FileState);\r
1024                 var storedHash = state.NullSafe(s => s.Checksum);\r
1025                 var storedHashes = state.NullSafe(s => s.Hashes);\r
1026                 var storedMD5 = state.NullSafe(s => s.LastMD5);\r
1027                 var storedDate = state.NullSafe(s => s.LastWriteDate) ?? DateTime.MinValue;\r
1028                 var storedLength = state.NullSafe(s => s.LastLength);\r
1029 \r
1030                 var md5Hash = Signature.MD5_EMPTY;                \r
1031                 var merkle=TreeHash.Empty;\r
1032 \r
1033                 if (hashTuple.FileInfo is FileInfo)\r
1034                 {\r
1035                     var file = (FileInfo)hashTuple.FileInfo.WithProperCapitalization();\r
1036                     \r
1037                     //Attributes unchanged?\r
1038                     //LastWriteTime is only accurate to the second\r
1039                     var unchangedAttributes = file.LastWriteTime - storedDate < TimeSpan.FromSeconds(1) \r
1040                         && storedLength == file.Length;\r
1041                     \r
1042                     //Attributes appear unchanged but the file length doesn't match the stored hash ?\r
1043                     var nonEmptyMismatch = unchangedAttributes && \r
1044                         (file.Length == 0 ^ storedHash== Signature.MERKLE_EMPTY);\r
1045 \r
1046                     //Missing hashes for NON-EMPTY hash ?\r
1047                     var missingHashes = storedHash != Signature.MERKLE_EMPTY &&\r
1048                         String.IsNullOrWhiteSpace(storedHashes);\r
1049 \r
1050                     //Unchanged attributes but changed MD5 \r
1051                     //Short-circuiting ensures MD5 is computed only if the attributes are changed\r
1052                     var md5Mismatch = (!unchangedAttributes &&\r
1053                                        file.ComputeShortHash(StatusNotification) != storedMD5);\r
1054 \r
1055 \r
1056                     //If the attributes are unchanged but the Merkle doesn't match the size,\r
1057                     //or the attributes and the MD5 hash have changed, \r
1058                     //or the hashes are missing but the tophash is NOT empty, we need to recalculate\r
1059                     //\r
1060                     //Otherwise we load the hashes from state\r
1061                     merkle = (nonEmptyMismatch || md5Mismatch || missingHashes)\r
1062                                  ? RecalculateTreehash(file)\r
1063                                  : TreeHash.Parse(hashTuple.FileState.Hashes);\r
1064 \r
1065 \r
1066                     md5Hash = merkle.MD5;\r
1067                 }\r
1068                 hashTuple.MD5 = md5Hash;\r
1069                 //Setting Merkle also updates C\r
1070                 hashTuple.Merkle = merkle;\r
1071             }\r
1072             catch (IOException)\r
1073             {\r
1074                 hashTuple.Locked = true;\r
1075             }            \r
1076         }\r
1077 \r
1078         /// <summary>\r
1079         /// Recalculate a file's treehash and md5 and update the database\r
1080         /// </summary>\r
1081         /// <param name="file"></param>\r
1082         /// <returns></returns>\r
1083         private TreeHash RecalculateTreehash(FileInfo file)\r
1084         {\r
1085             var progress = new Progress<double>(d =>StatusNotification.Notify(\r
1086                                                     new StatusNotification(String.Format("Hashing {0} of {1}", d, file.Name))));\r
1087             var merkle = Signature.CalculateTreeHash(file, StatusKeeper.BlockSize, \r
1088                 StatusKeeper.BlockHash,CancellationToken, progress);\r
1089             StatusKeeper.UpdateFileTreeHash(file.FullName, merkle);\r
1090             return merkle;\r
1091         }\r
1092 \r
1093         /// <summary>\r
1094         /// Returns the latest LastModified date from the list of objects, but only if it is before\r
1095         /// than the threshold value\r
1096         /// </summary>\r
1097         /// <param name="threshold"></param>\r
1098         /// <param name="cloudObjects"></param>\r
1099         /// <returns></returns>\r
1100         private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
1101         {\r
1102             DateTime? maxDate = null;\r
1103             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
1104                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
1105             if (maxDate == null || maxDate == DateTime.MinValue)\r
1106                 return threshold;\r
1107             if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate)\r
1108                 return maxDate;\r
1109             return threshold;\r
1110         }\r
1111 \r
1112         /// <summary>\r
1113         /// Returns the latest LastModified date from the list of objects, but only if it is after\r
1114         /// the threshold value\r
1115         /// </summary>\r
1116         /// <param name="threshold"></param>\r
1117         /// <param name="cloudObjects"></param>\r
1118         /// <returns></returns>\r
1119         private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
1120         {\r
1121             DateTime? maxDate = null;\r
1122             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
1123                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
1124             if (maxDate == null || maxDate == DateTime.MinValue)\r
1125                 return threshold;\r
1126             if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate)\r
1127                 return maxDate;\r
1128             return threshold;\r
1129         }\r
1130 \r
1131         readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
1132         private bool _pause;\r
1133         private readonly string _emptyGuid = Guid.Empty.ToString();\r
1134 \r
1135 \r
1136 \r
1137         private void ReportConflictForMismatch(string localFilePath)\r
1138         {\r
1139             if (String.IsNullOrWhiteSpace(localFilePath))\r
1140                 throw new ArgumentNullException("localFilePath");\r
1141             Contract.EndContractBlock();\r
1142 \r
1143             StatusKeeper.SetFileState(localFilePath, FileStatus.Conflict, FileOverlayStatus.Conflict, "File changed at the server");\r
1144             UpdateStatus(PithosStatus.HasConflicts);\r
1145             var message = String.Format("Conflict detected for file {0}", localFilePath);\r
1146             Log.Warn(message);\r
1147             StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
1148         }\r
1149 \r
1150 \r
1151         /// <summary>\r
1152         /// Notify the UI to update the visual status\r
1153         /// </summary>\r
1154         /// <param name="status"></param>\r
1155         private void UpdateStatus(PithosStatus status)\r
1156         {\r
1157             try\r
1158             {\r
1159                 StatusNotification.SetPithosStatus(status);\r
1160                 //StatusNotification.Notify(new Notification());\r
1161             }\r
1162             catch (Exception exc)\r
1163             {\r
1164                 //Failure is not critical, just log it\r
1165                 Log.Warn("Error while updating status", exc);\r
1166             }\r
1167         }\r
1168 \r
1169         private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)\r
1170         {\r
1171             var containerPaths = from container in containers\r
1172                                  let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)\r
1173                                  where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)\r
1174                                  select containerPath;\r
1175 \r
1176             foreach (var path in containerPaths)\r
1177             {\r
1178                 Directory.CreateDirectory(path);\r
1179             }\r
1180         }\r
1181 \r
1182         public void AddAccount(AccountInfo accountInfo)\r
1183         {\r
1184             //Avoid adding a duplicate accountInfo\r
1185             _accounts.TryAdd(accountInfo.AccountKey, accountInfo);\r
1186         }\r
1187 \r
1188         public void RemoveAccount(AccountInfo accountInfo)\r
1189         {\r
1190             AccountInfo account;\r
1191             _accounts.TryRemove(accountInfo.AccountKey, out account);\r
1192 \r
1193             SnapshotDifferencer differencer;\r
1194             _differencer.Differencers.TryRemove(accountInfo.AccountKey, out differencer);\r
1195         }\r
1196        \r
1197     }\r
1198 }\r