code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/PollAgent.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="PollAgent.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 \r
  43 using System.Collections.Concurrent;\r
  44 using System.ComponentModel.Composition;\r
  45 using System.Diagnostics;\r
  46 using System.Diagnostics.Contracts;\r
  47 using System.IO;\r
  48 using System.Linq.Expressions;\r
  49 using System.Reflection;\r
  50 using System.Security.Cryptography;\r
  51 using System.Threading;\r
  52 using System.Threading.Tasks;\r
  53 using System.Threading.Tasks.Dataflow;\r
  54 using Castle.ActiveRecord;\r
  55 using Pithos.Interfaces;\r
  56 using Pithos.Network;\r
  57 using log4net;\r
  58 \r
  59 namespace Pithos.Core.Agents\r
  60 {\r
  61     using System;\r
  62     using System.Collections.Generic;\r
  63     using System.Linq;\r
  64 \r
  65     /*public class PollRequest\r
  66     {\r
  67         public DateTime? Since { get; set; }\r
  68         public IEnumerable<string> Batch { get; set; }\r
  69     }*/\r
  70 \r
  71 \r
  72     /// <summary>\r
  73     /// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all\r
  74     /// objects and compares it with a previously cached version to detect differences. \r
  75     /// New files are downloaded, missing files are deleted from the local file system and common files are compared\r
  76     /// to determine the appropriate action\r
  77     /// </summary>\r
  78     [Export]\r
  79     public class PollAgent\r
  80     {\r
  81         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  82 \r
  83         [System.ComponentModel.Composition.Import]\r
  84         public IStatusKeeper StatusKeeper { get; set; }\r
  85 \r
  86         [System.ComponentModel.Composition.Import]\r
  87         public IPithosSettings Settings { get; set; }\r
  88 \r
  89         [System.ComponentModel.Composition.Import]\r
  90         public NetworkAgent NetworkAgent { get; set; }\r
  91 \r
  92         [System.ComponentModel.Composition.Import]\r
  93         public Selectives Selectives { get; set; }\r
  94 \r
  95         public IStatusNotification StatusNotification { get; set; }\r
  96 \r
  97         private CancellationTokenSource _currentOperationCancellation = new CancellationTokenSource();\r
  98 \r
  99         public void CancelCurrentOperation()\r
 100         {\r
 101             //What does it mean to cancel the current upload/download?\r
 102             //Obviously, the current operation will be cancelled by throwing\r
 103             //a cancellation exception.\r
 104             //\r
 105             //The default behavior is to retry any operations that throw.\r
 106             //Obviously this is not what we want in this situation.\r
 107             //The cancelled operation should NOT bea retried. \r
 108             //\r
 109             //This can be done by catching the cancellation exception\r
 110             //and avoiding the retry.\r
 111             //\r
 112 \r
 113             //Have to reset the cancellation source - it is not possible to reset the source\r
 114             //Have to prevent a case where an operation requests a token from the old source\r
 115             var oldSource = Interlocked.Exchange(ref _currentOperationCancellation, new CancellationTokenSource());\r
 116             oldSource.Cancel();\r
 117 \r
 118         }\r
 119 \r
 120         public bool Pause\r
 121         {\r
 122             get {\r
 123                 return _pause;\r
 124             }\r
 125             set {\r
 126                 _pause = value;                \r
 127                 if (!_pause)\r
 128                     _unPauseEvent.Set();\r
 129                 else\r
 130                 {\r
 131                     _unPauseEvent.Reset();\r
 132                 }\r
 133             }\r
 134         }\r
 135 \r
 136         public CancellationToken CancellationToken\r
 137         {\r
 138             get { return _currentOperationCancellation.Token; }\r
 139         }\r
 140 \r
 141         private bool _firstPoll = true;\r
 142 \r
 143         //The Sync Event signals a manual synchronisation\r
 144         private readonly AsyncManualResetEvent _syncEvent = new AsyncManualResetEvent();\r
 145 \r
 146         private readonly AsyncManualResetEvent _unPauseEvent = new AsyncManualResetEvent(true);\r
 147 \r
 148         private readonly ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>();\r
 149         private readonly ConcurrentDictionary<Uri, AccountInfo> _accounts = new ConcurrentDictionary<Uri,AccountInfo>();\r
 150 \r
 151         //private readonly ActionBlock<PollRequest>  _pollAction;\r
 152 \r
 153         readonly HashSet<string> _knownContainers = new HashSet<string>();\r
 154 \r
 155         \r
 156         /// <summary>\r
 157         /// Start a manual synchronization\r
 158         /// </summary>\r
 159         public void SynchNow(IEnumerable<string> paths=null)\r
 160         {\r
 161             _batchQueue.Enqueue(paths);\r
 162             _syncEvent.Set();                \r
 163 \r
 164             //_pollAction.Post(new PollRequest {Batch = paths});\r
 165         }\r
 166 \r
 167         readonly ConcurrentQueue<IEnumerable<string>> _batchQueue=new ConcurrentQueue<IEnumerable<string>>();\r
 168 \r
 169         ConcurrentDictionary<string,MovedEventArgs> _moves=new ConcurrentDictionary<string, MovedEventArgs>(); \r
 170 \r
 171         public void PostMove(MovedEventArgs args)\r
 172         {\r
 173             TaskEx.Run(() => _moves.AddOrUpdate(args.OldFullPath, args,(s,e)=>e));            \r
 174         }\r
 175 \r
 176         /// <summary>\r
 177         /// Remote files are polled periodically. Any changes are processed\r
 178         /// </summary>\r
 179         /// <param name="since"></param>\r
 180         /// <returns></returns>\r
 181         public  void PollRemoteFiles(DateTime? since = null)\r
 182         {\r
 183             if (Log.IsDebugEnabled)\r
 184                 Log.DebugFormat("Polling changes after [{0}]",since);\r
 185 \r
 186             Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
 187 \r
 188             //GC.Collect();\r
 189 \r
 190             using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
 191             {\r
 192                 //If this poll fails, we will retry with the same since value\r
 193                 var nextSince = since;\r
 194                 try\r
 195                 {\r
 196                     _unPauseEvent.Wait();\r
 197                     UpdateStatus(PithosStatus.PollSyncing);\r
 198 \r
 199                     var accountBatches=new Dictionary<Uri, IEnumerable<string>>();\r
 200                     IEnumerable<string> batch = null;\r
 201                     if (_batchQueue.TryDequeue(out batch) && batch != null)\r
 202                         foreach (var account in _accounts.Values)\r
 203                         {\r
 204                             var accountBatch = batch.Where(path => path.IsAtOrBelow(account.AccountPath));\r
 205                             accountBatches[account.AccountKey] = accountBatch;\r
 206                         }\r
 207                     \r
 208                     var moves=Interlocked.Exchange(ref _moves, new ConcurrentDictionary<string, MovedEventArgs>());\r
 209 \r
 210                     var tasks = new List<Task<DateTime?>>();\r
 211                     foreach(var accountInfo in _accounts.Values)\r
 212                     {\r
 213                         IEnumerable<string> accountBatch ;\r
 214                         accountBatches.TryGetValue(accountInfo.AccountKey,out accountBatch);\r
 215                         var t=ProcessAccountFiles (accountInfo, accountBatch, moves,since);\r
 216                         tasks.Add(t);\r
 217                     }\r
 218 \r
 219                     var nextTimes=TaskEx.WhenAll(tasks.ToList()).Result;\r
 220 \r
 221                     _firstPoll = false;\r
 222                     //Reschedule the poll with the current timestamp as a "since" value\r
 223 \r
 224                     if (nextTimes.Length>0)\r
 225                         nextSince = nextTimes.Min();\r
 226                     if (Log.IsDebugEnabled)\r
 227                         Log.DebugFormat("Next Poll at [{0}]",nextSince);\r
 228                 }\r
 229                 catch (Exception ex)\r
 230                 {\r
 231                     Log.ErrorFormat("Error while processing accounts\r\n{0}", ex);\r
 232                     //In case of failure retry with the same "since" value\r
 233                 }\r
 234 \r
 235                 UpdateStatus(PithosStatus.PollComplete);\r
 236                 //The multiple try blocks are required because we can't have an await call\r
 237                 //inside a finally block\r
 238                 //TODO: Find a more elegant solution for reschedulling in the event of an exception\r
 239                 try\r
 240                 {\r
 241                     //Wait for the polling interval to pass or the Sync event to be signalled\r
 242                     nextSince = WaitForScheduledOrManualPoll(nextSince).Result;\r
 243                 }\r
 244                 finally\r
 245                 {\r
 246                     //Ensure polling is scheduled even in case of error\r
 247                     TaskEx.Run(()=>PollRemoteFiles(nextSince));\r
 248                     //_pollAction.Post(new PollRequest {Since = nextSince});\r
 249                 }\r
 250             }\r
 251         }\r
 252 \r
 253         /// <summary>\r
 254         /// Wait for the polling period to expire or a manual sync request\r
 255         /// </summary>\r
 256         /// <param name="since"></param>\r
 257         /// <returns></returns>\r
 258         private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
 259         {\r
 260             var sync = _syncEvent.WaitAsync();\r
 261             var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval));\r
 262 \r
 263             var signaledTask = await TaskEx.WhenAny(sync, wait).ConfigureAwait(false);\r
 264             \r
 265             //Pausing takes precedence over manual sync or awaiting\r
 266             _unPauseEvent.Wait();\r
 267             \r
 268             //Wait for network processing to finish before polling\r
 269             var pauseTask=NetworkAgent.ProceedEvent.WaitAsync();\r
 270             await TaskEx.WhenAll(signaledTask, pauseTask).ConfigureAwait(false);\r
 271 \r
 272             //If polling is signalled by SynchNow, ignore the since tag\r
 273             if (sync.IsCompleted)\r
 274             {                \r
 275                 _syncEvent.Reset();\r
 276                 return null;\r
 277             }\r
 278             return since;\r
 279         }\r
 280 \r
 281         \r
 282 \r
 283         public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, IEnumerable<string> accountBatch, ConcurrentDictionary<string, MovedEventArgs> moves, DateTime? since = null)\r
 284         {\r
 285             if (accountInfo == null)\r
 286                 throw new ArgumentNullException("accountInfo");\r
 287             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
 288                 throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
 289             Contract.EndContractBlock();\r
 290 \r
 291 \r
 292             using (ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))\r
 293             {\r
 294 \r
 295                 await NetworkAgent.GetDeleteAwaiter().ConfigureAwait(false);\r
 296 \r
 297                 Log.Info("Scheduled");\r
 298                 var client = new CloudFilesClient(accountInfo);\r
 299 \r
 300                 //We don't need to check the trash container\r
 301                 var containers = client.ListContainers(accountInfo.UserName)\r
 302                     .Where(c=>c.Name!="trash")\r
 303                     .ToList();\r
 304 \r
 305 \r
 306                 CreateContainerFolders(accountInfo, containers);\r
 307 \r
 308                 //The nextSince time fallback time is the same as the current.\r
 309                 //If polling succeeds, the next Since time will be the smallest of the maximum modification times\r
 310                 //of the shared and account objects\r
 311                 var nextSince = since;\r
 312 \r
 313                 try\r
 314                 {\r
 315                     //Wait for any deletions to finish\r
 316                     await NetworkAgent.GetDeleteAwaiter().ConfigureAwait(false);\r
 317                     //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
 318                     //than delete a file that was created while we were executing the poll                    \r
 319 \r
 320                     //Get the list of server objects changed since the last check\r
 321                     //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
 322                     var listObjects = (from container in containers\r
 323                                        select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
 324                                              client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
 325 \r
 326                     var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
 327                         client.ListSharedObjects(_knownContainers,since), "shared");\r
 328                     listObjects.Add(listShared);\r
 329                     var listTasks = await Task.Factory.WhenAll(listObjects.ToArray()).ConfigureAwait(false);\r
 330 \r
 331                     using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
 332                     {\r
 333                         var dict = listTasks.ToDictionary(t => t.AsyncState);\r
 334 \r
 335                         //Get all non-trash objects. Remember, the container name is stored in AsyncState\r
 336                         var remoteObjects = (from objectList in listTasks\r
 337                                             where (string)objectList.AsyncState != "trash"\r
 338                                             from obj in objectList.Result\r
 339                                             orderby obj.Bytes ascending \r
 340                                             select obj).ToList();\r
 341                         \r
 342                         //Get the latest remote object modification date, only if it is after\r
 343                         //the original since date                        \r
 344                         nextSince = GetLatestDateAfter(nextSince, remoteObjects);\r
 345 \r
 346                         var sharedObjects = dict["shared"].Result;\r
 347 \r
 348                         //DON'T process trashed files\r
 349                         //If some files are deleted and added again to a folder, they will be deleted\r
 350                         //even though they are new.\r
 351                         //We would have to check file dates and hashes to ensure that a trashed file\r
 352                         //can be deleted safely from the local hard drive.\r
 353                         /*\r
 354                         //Items with the same name, hash may be both in the container and the trash\r
 355                         //Don't delete items that exist in the container\r
 356                         var realTrash = from trash in trashObjects\r
 357                                         where\r
 358                                             !remoteObjects.Any(\r
 359                                                 info => info.Name == trash.Name && info.Hash == trash.Hash)\r
 360                                    8     select trash;\r
 361                         ProcessTrashedFiles(accountInfo, realTrash);\r
 362 */\r
 363 \r
 364                         var cleanRemotes = (from info in remoteObjects.Union(sharedObjects)\r
 365                                             let name = info.Name??""\r
 366                                             where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&\r
 367                                                   !name.StartsWith(FolderConstants.CacheFolder + "/",\r
 368                                                                    StringComparison.InvariantCultureIgnoreCase)\r
 369                                             select info).ToList();\r
 370 \r
 371                         if (_firstPoll)\r
 372                             StatusKeeper.CleanupOrphanStates();\r
 373                         \r
 374                         var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
 375                         var currentRemotes = differencer.Current.ToList();\r
 376                         StatusKeeper.CleanupStaleStates(accountInfo, currentRemotes);\r
 377 \r
 378                         //var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
 379 \r
 380                         //May have to wait if the FileAgent has asked for a Pause, due to local changes\r
 381                         await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
 382 \r
 383                         //Get the local files here                        \r
 384                         var agent = AgentLocator<FileAgent>.Get(accountInfo.AccountPath);                                                \r
 385                         var files = LoadLocalFileTuples(accountInfo, accountBatch);\r
 386 \r
 387                         var states = FileState.Queryable.ToList();                        \r
 388                         \r
 389                         var infos = (from remote in currentRemotes\r
 390                                     let path = remote.RelativeUrlToFilePath(accountInfo.UserName)\r
 391                                     let info=agent.GetFileSystemInfo(path)\r
 392                                     select Tuple.Create(info.FullName,remote))\r
 393                                     .ToList();\r
 394 \r
 395                         var token = _currentOperationCancellation.Token;\r
 396 \r
 397                         var tuples = MergeSources(infos, files, states,moves).ToList();\r
 398 \r
 399                         var processedPaths = new HashSet<string>();\r
 400                         //Process only the changes in the batch file, if one exists\r
 401                         var stateTuples = accountBatch==null?tuples:tuples.Where(t => accountBatch.Contains(t.FilePath));\r
 402                         foreach (var tuple in stateTuples.Where(s=>!s.Locked))\r
 403                         {\r
 404                             await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
 405 \r
 406                             //Set the Merkle Hash\r
 407                             //SetMerkleHash(accountInfo, tuple);\r
 408 \r
 409                             await SyncSingleItem(accountInfo, tuple, agent, moves,processedPaths,token).ConfigureAwait(false);\r
 410 \r
 411                         }\r
 412 \r
 413 \r
 414                         //On the first run\r
 415 /*\r
 416                         if (_firstPoll)\r
 417                         {\r
 418                             MarkSuspectedDeletes(accountInfo, cleanRemotes);\r
 419                         }\r
 420 */\r
 421 \r
 422 \r
 423                         Log.Info("[LISTENER] End Processing");\r
 424                     }\r
 425                 }\r
 426                 catch (Exception ex)\r
 427                 {\r
 428                     Log.ErrorFormat("[FAIL] ListObjects for {0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);\r
 429                     return nextSince;\r
 430                 }\r
 431 \r
 432                 Log.Info("[LISTENER] Finished");\r
 433                 return nextSince;\r
 434             }\r
 435         }\r
 436 /*\r
 437 \r
 438         private static void SetMerkleHash(AccountInfo accountInfo, StateTuple tuple)\r
 439         {\r
 440             //The Merkle hash for directories is that of an empty buffer\r
 441             if (tuple.FileInfo is DirectoryInfo)\r
 442                 tuple.C = MERKLE_EMPTY;\r
 443             else if (tuple.FileState != null && tuple.MD5 == tuple.FileState.ETag)\r
 444             {\r
 445                 //If there is a state whose MD5 matches, load the merkle hash from the file state\r
 446                 //insteaf of calculating it\r
 447                 tuple.C = tuple.FileState.Checksum;                              \r
 448             }\r
 449             else\r
 450             {\r
 451                 tuple.Merkle = Signature.CalculateTreeHashAsync((FileInfo)tuple.FileInfo, accountInfo.BlockSize, accountInfo.BlockHash,1,progress);\r
 452                 //tuple.C=tuple.Merkle.TopHash.ToHashString();                \r
 453             }\r
 454         }\r
 455 */\r
 456 \r
 457         private IEnumerable<FileSystemInfo> LoadLocalFileTuples(AccountInfo accountInfo,IEnumerable<string> batch )\r
 458         {\r
 459             using (ThreadContext.Stacks["Account Files Hashing"].Push(accountInfo.UserName))\r
 460             {\r
 461                 var batchPaths = (batch==null)?new List<string>():batch.ToList();\r
 462                 IEnumerable<FileSystemInfo> localInfos=AgentLocator<FileAgent>.Get(accountInfo.AccountPath)\r
 463                                                         .EnumerateFileSystemInfos();\r
 464                 if (batchPaths.Count>0)\r
 465                     localInfos= localInfos.Where(fi => batchPaths.Contains(fi.FullName));\r
 466 \r
 467                 return localInfos;\r
 468             }\r
 469         }\r
 470 \r
 471         /// <summary>\r
 472         /// Wait and Pause the agent while waiting\r
 473         /// </summary>\r
 474         /// <param name="backoff"></param>\r
 475         /// <returns></returns>\r
 476         private async Task PauseFor(int backoff)\r
 477         {\r
 478 \r
 479             Pause = true;\r
 480             await TaskEx.Delay(backoff).ConfigureAwait(false);\r
 481             Pause = false;\r
 482         }\r
 483 \r
 484         private async Task SyncSingleItem(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves,HashSet<string> processedPaths, CancellationToken token)\r
 485         {\r
 486             Log.DebugFormat("Sync [{0}] C:[{1}] L:[{2}] S:[{3}]", tuple.FilePath, tuple.C, tuple.L, tuple.S);\r
 487 \r
 488             //If the processed paths already contain the current path, exit\r
 489             if (!processedPaths.Add(tuple.FilePath))\r
 490                 return;\r
 491 \r
 492             try\r
 493             {\r
 494                 bool isInferredParent = tuple.ObjectInfo != null && tuple.ObjectInfo.UUID.StartsWith("00000000-0000-0000");\r
 495 \r
 496                 var localFilePath = tuple.FilePath;\r
 497                 //Don't use the tuple info, it may have been deleted\r
 498                 var localInfo = FileInfoExtensions.FromPath(localFilePath);\r
 499 \r
 500 \r
 501                 var isUnselectedRootFolder = agent.IsUnselectedRootFolder(tuple.FilePath);\r
 502 \r
 503                 //Unselected root folders that have not yet been uploaded should be uploaded and added to the \r
 504                 //selective folders\r
 505 \r
 506                 if (!Selectives.IsSelected(accountInfo, localFilePath) &&\r
 507                     !(isUnselectedRootFolder && tuple.ObjectInfo == null))\r
 508                     return;\r
 509 \r
 510                 // Local file unchanged? If both C and L are null, make sure it's because \r
 511                 //both the file is missing and the state checksum is not missing\r
 512                 if (tuple.C == tuple.L /*&& (localInfo.Exists || tuple.FileState == null)*/)\r
 513                 {\r
 514                     //No local changes\r
 515                     //Server unchanged?\r
 516                     if (tuple.S == tuple.L)\r
 517                     {\r
 518                         // No server changes\r
 519                         //Has the file been renamed on the server?\r
 520                         MoveForServerMove(accountInfo, tuple);\r
 521                     }\r
 522                     else\r
 523                     {\r
 524                         //Different from server\r
 525                         //Does the server file exist?\r
 526                         if (tuple.S == null)\r
 527                         {\r
 528                             //Server file doesn't exist\r
 529                             //deleteObjectFromLocal()\r
 530                             using (\r
 531                                 StatusNotification.GetNotifier("Deleting local {0}", "Deleted local {0}",\r
 532                                                                localInfo.Name))\r
 533                             {\r
 534                                 DeleteLocalFile(agent, localFilePath);\r
 535                             }\r
 536                         }\r
 537                         else\r
 538                         {\r
 539                             //Server file exists\r
 540                             //downloadServerObject() // Result: L = S\r
 541                             //If the file has moved on the server, move it locally before downloading\r
 542                             using (\r
 543                                 StatusNotification.GetNotifier("Downloading {0}", "Downloaded {0}",\r
 544                                                                localInfo.Name))\r
 545                             {\r
 546                                 var targetPath = MoveForServerMove(accountInfo, tuple);\r
 547 \r
 548                                 await DownloadCloudFile(accountInfo, tuple, token, targetPath).ConfigureAwait(false);\r
 549 \r
 550                                 AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
 551                             }\r
 552 \r
 553                             /*\r
 554                                                             StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
 555                                                                                       FileOverlayStatus.Normal, "");\r
 556                                 */\r
 557                         }\r
 558                     }\r
 559                 }\r
 560                 else\r
 561                 {                   \r
 562 \r
 563                     //Local changes found\r
 564 \r
 565                     //Server unchanged?\r
 566                     if (tuple.S == tuple.L)\r
 567                     {\r
 568                         //The FileAgent selective sync checks for new root folder files\r
 569                         if (!agent.Ignore(localFilePath))\r
 570                         {\r
 571                             if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
 572                             {\r
 573                                 //deleteObjectFromServer()\r
 574                                 DeleteCloudFile(accountInfo, tuple);\r
 575                                 //updateRecord( Remove L, S)                  \r
 576                             }\r
 577                             else\r
 578                             {\r
 579                                 //uploadLocalObject() // Result: S = C, L = S                        \r
 580                                 var progress = new Progress<double>(d =>\r
 581                                     StatusNotification.Notify(new StatusNotification(String.Format("Merkle Hashing for Upload {0:p} of {1}", d, localInfo.Name))));\r
 582 \r
 583                                 //Is it an unselected root folder\r
 584                                 var isCreation = isUnselectedRootFolder ||//or a new folder under a selected parent?\r
 585                                         (localInfo is DirectoryInfo && Selectives.IsSelected(accountInfo, localInfo) && tuple.FileState == null && tuple.ObjectInfo == null);\r
 586 \r
 587 \r
 588                                 //Is this a result of a FILE move with no modifications? Then try to move it,\r
 589                                 //to avoid an expensive hash\r
 590                                 if (!MoveForLocalMove(accountInfo, tuple))\r
 591                                 {\r
 592                                     await UploadLocalFile(accountInfo, tuple, token, isCreation, localInfo,processedPaths, progress).ConfigureAwait(false);\r
 593                                 }\r
 594 \r
 595                                 //updateRecord( S = C )\r
 596                                 //State updated by the uploader\r
 597                                 \r
 598                                 if (isCreation )\r
 599                                 {                                    \r
 600                                     ProcessChildren(accountInfo, tuple, agent, moves,processedPaths,token);\r
 601                                 }\r
 602                             }\r
 603                         }\r
 604                     }\r
 605                     else\r
 606                     {\r
 607                         if (tuple.C == tuple.S)\r
 608                         {\r
 609                             // (Identical Changes) Result: L = S\r
 610                             //doNothing()\r
 611                             \r
 612                             //Don't update anything for nonexistend server files\r
 613                             if (tuple.S != null)\r
 614                             {\r
 615                                 //Detect server moves\r
 616                                 var targetPath = MoveForServerMove(accountInfo, tuple);\r
 617                                 StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
 618                                 \r
 619                                 AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
 620                             }\r
 621                             else\r
 622                             {\r
 623                                 //At this point, C==S==NULL and we have a stale state (L)\r
 624                                 //Log the stale tuple for investigation\r
 625                                 Log.WarnFormat("Stale tuple detected FilePathPath:[{0}], State:[{1}], LocalFile:[{2}]", tuple.FilePath, tuple.FileState, tuple.FileInfo);\r
 626 \r
 627                                 //And remove it\r
 628                                 if (!String.IsNullOrWhiteSpace(tuple.FilePath))\r
 629                                     StatusKeeper.ClearFileStatus(tuple.FilePath);\r
 630                             }\r
 631                         }\r
 632                         else\r
 633                         {\r
 634                             if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
 635                             {\r
 636                                 //deleteObjectFromServer()\r
 637                                 DeleteCloudFile(accountInfo, tuple);\r
 638                                 //updateRecord(Remove L, S)                  \r
 639                             }\r
 640                                 //If both the local and server files are missing, the state is stale\r
 641                             else if (!localInfo.Exists && (tuple.S == null || tuple.ObjectInfo == null))\r
 642                             {\r
 643                                 StatusKeeper.ClearFileStatus(localInfo.FullName);\r
 644                             }\r
 645                             else\r
 646                             {\r
 647                                 ReportConflictForMismatch(localFilePath);\r
 648                                 //identifyAsConflict() // Manual action required\r
 649                             }\r
 650                         }\r
 651                     }\r
 652                 }\r
 653             }\r
 654             catch (Exception exc)\r
 655             {\r
 656                 //In case of error log and retry with the next poll\r
 657                 Log.ErrorFormat("[SYNC] Failed for file {0}. Will Retry.\r\n{1}",tuple.FilePath,exc);\r
 658             }\r
 659         }\r
 660 \r
 661         private void DeleteLocalFile(FileAgent agent, string localFilePath)\r
 662         {\r
 663             StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
 664                                       FileOverlayStatus.Deleted, "");\r
 665             using (NetworkGate.Acquire(localFilePath, NetworkOperation.Deleting))\r
 666             {\r
 667                 agent.Delete(localFilePath);\r
 668             }\r
 669             //updateRecord(Remove C, L)\r
 670             StatusKeeper.ClearFileStatus(localFilePath);\r
 671         }\r
 672 \r
 673         private async Task DownloadCloudFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token, string targetPath)\r
 674         {\r
 675             StatusKeeper.SetFileState(targetPath, FileStatus.Modified, FileOverlayStatus.Modified,\r
 676                                       "");\r
 677 \r
 678             await\r
 679                 NetworkAgent.Downloader.DownloadCloudFile(accountInfo, tuple.ObjectInfo, targetPath,\r
 680                                                           token)\r
 681                     .ConfigureAwait(false);\r
 682             //updateRecord( L = S )\r
 683             StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
 684                                             tuple.ObjectInfo.X_Object_Hash);\r
 685 \r
 686             StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
 687         }\r
 688 \r
 689         private async Task UploadLocalFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token,\r
 690                                      bool isUnselectedRootFolder, FileSystemInfo localInfo, HashSet<string> processedPaths, Progress<double> progress)\r
 691         {\r
 692             var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
 693                                                accountInfo.BlockSize, accountInfo.BlockHash,\r
 694                                                "Poll", isUnselectedRootFolder, token, progress);\r
 695             using (StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}",\r
 696                                                   localInfo.Name))\r
 697             {\r
 698                 await NetworkAgent.Uploader.UploadCloudFile(action, token).ConfigureAwait(false);\r
 699             }\r
 700 \r
 701             if (isUnselectedRootFolder)\r
 702             {\r
 703                 var dirActions =(\r
 704                     from dir in ((DirectoryInfo) localInfo).EnumerateDirectories("*", SearchOption.AllDirectories)\r
 705                     let subAction = new CloudUploadAction(accountInfo, dir, null,\r
 706                                                           accountInfo.BlockSize, accountInfo.BlockHash,\r
 707                                                           "Poll", true, token, progress)\r
 708                     select subAction).ToList();\r
 709                 foreach (var dirAction in dirActions)\r
 710                 {\r
 711                     processedPaths.Add(dirAction.LocalFile.FullName);\r
 712                 }\r
 713                 \r
 714                 await TaskEx.WhenAll(dirActions.Select(a=>NetworkAgent.Uploader.UploadCloudFile(a,token)).ToArray());\r
 715             }\r
 716         }\r
 717 \r
 718         private bool MoveForLocalMove(AccountInfo accountInfo, StateTuple tuple)\r
 719         {\r
 720             if (!(tuple.FileInfo is FileInfo) || String.IsNullOrWhiteSpace(tuple.OldFullPath) || tuple.OldMD5 != tuple.C)\r
 721                 return false;\r
 722 \r
 723             try\r
 724             {\r
 725 \r
 726                 var client = new CloudFilesClient(accountInfo);\r
 727                 var objectInfo = CloudAction.CreateObjectInfoFor(accountInfo, tuple.FileInfo);\r
 728                 var containerPath = Path.Combine(accountInfo.AccountPath, objectInfo.Container);\r
 729                 var oldName = tuple.OldFullPath.AsRelativeTo(containerPath);\r
 730                 //Then execute a move instead of an upload\r
 731                 using (StatusNotification.GetNotifier("Moving {0}", "Moved {0}", tuple.FileInfo.Name))\r
 732                 {\r
 733                     client.MoveObject(objectInfo.Account, objectInfo.Container, oldName,\r
 734                                                           objectInfo.Container, objectInfo.Name);\r
 735                 }\r
 736                 return true;\r
 737             }\r
 738             catch (Exception exc)\r
 739             {\r
 740                 Log.ErrorFormat("[MOVE] Failed for [{0}],:\r\n{1}", tuple.FilePath, exc);\r
 741                 //Return false to force an upload of the file\r
 742                 return false;\r
 743             }\r
 744 \r
 745         }\r
 746 \r
 747         private void AddOwnFolderToSelectives(AccountInfo accountInfo, StateTuple tuple, string targetPath)\r
 748         {\r
 749             //Not for shared folders\r
 750             if (tuple.ObjectInfo.IsShared==true)\r
 751                 return;\r
 752             //Also ensure that any newly created folders are added to the selectives, if the original folder was selected\r
 753             var containerPath = Path.Combine(accountInfo.AccountPath, tuple.ObjectInfo.Container);\r
 754 \r
 755             //If this is a root folder encountered for the first time\r
 756             if (tuple.L == null && Directory.Exists(tuple.FileInfo.FullName) \r
 757                 && (tuple.FileInfo.FullName.IsAtOrBelow(containerPath)))\r
 758             {\r
 759                 \r
 760                 var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 761                 var initialPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
 762 \r
 763                 //var hasMoved = true;// !initialPath.Equals(targetPath);\r
 764                 //If the new path is under a selected folder, add it to the selectives as well\r
 765                 if (Selectives.IsSelected(accountInfo, initialPath))\r
 766                 {\r
 767                     Selectives.AddUri(accountInfo, tuple.ObjectInfo.Uri);\r
 768                     Selectives.Save(accountInfo);\r
 769                 }\r
 770             }\r
 771         }\r
 772 \r
 773         private string MoveForServerMove(AccountInfo accountInfo, StateTuple tuple)\r
 774         {\r
 775             if (tuple.ObjectInfo == null)\r
 776                 return null;\r
 777             var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 778             var serverPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
 779             \r
 780             //Compare Case Insensitive\r
 781             if (String.Equals(tuple.FilePath ,serverPath,StringComparison.InvariantCultureIgnoreCase)) \r
 782                 return serverPath;\r
 783 \r
 784             tuple.FileInfo.Refresh();\r
 785             //The file/folder may not exist if it was moved because its parent moved\r
 786             if (!tuple.FileInfo.Exists)\r
 787             {\r
 788                 var target=FileInfoExtensions.FromPath(serverPath);\r
 789                 if (!target.Exists)\r
 790                 {\r
 791                     Log.ErrorFormat("No source or target found while trying to move {0} to {1}", tuple.FileInfo.FullName, serverPath);\r
 792                 }\r
 793                 return serverPath;\r
 794             }\r
 795 \r
 796             using (StatusNotification.GetNotifier("Moving local {0}", "Moved local {0}", Path.GetFileName(tuple.FilePath)))\r
 797             using(NetworkGate.Acquire(tuple.FilePath,NetworkOperation.Renaming))\r
 798             {\r
 799                     \r
 800                 var fi = tuple.FileInfo as FileInfo;\r
 801                 if (fi != null)\r
 802                 {\r
 803                     var targetFile = new FileInfo(serverPath);\r
 804                     if (!targetFile.Directory.Exists)\r
 805                         targetFile.Directory.Create();\r
 806                     fi.MoveTo(serverPath);\r
 807                 }\r
 808                 var di = tuple.FileInfo as DirectoryInfo;\r
 809                 if (di != null)\r
 810                 {\r
 811                     var targetDir = new DirectoryInfo(serverPath);\r
 812                     if (!targetDir.Parent.Exists)\r
 813                         targetDir.Parent.Create();\r
 814                     di.MoveTo(serverPath);\r
 815                 }\r
 816             }\r
 817             StatusKeeper.StoreInfo(serverPath, tuple.ObjectInfo);\r
 818 \r
 819             return serverPath;\r
 820         }\r
 821 \r
 822         private void DeleteCloudFile(AccountInfo accountInfo, StateTuple tuple)\r
 823         {\r
 824             using (StatusNotification.GetNotifier("Deleting server {0}", "Deleted server {0}", Path.GetFileName(tuple.FilePath)))\r
 825             {\r
 826 \r
 827                 StatusKeeper.SetFileState(tuple.FilePath, FileStatus.Deleted,\r
 828                                           FileOverlayStatus.Deleted, "");\r
 829                 NetworkAgent.DeleteAgent.DeleteCloudFile(accountInfo, tuple.ObjectInfo);\r
 830                 StatusKeeper.ClearFileStatus(tuple.FilePath);\r
 831             }\r
 832         }\r
 833 \r
 834         private void ProcessChildren(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, ConcurrentDictionary<string, MovedEventArgs> moves,HashSet<string> processedPaths,CancellationToken token)\r
 835         {\r
 836 \r
 837             var dirInfo = tuple.FileInfo as DirectoryInfo;\r
 838             var folderTuples = from folder in dirInfo.EnumerateDirectories("*", SearchOption.AllDirectories)\r
 839                                select new StateTuple(folder){C=Signature.MD5_EMPTY};\r
 840             var fileTuples = from file in dirInfo.EnumerateFiles("*", SearchOption.AllDirectories)\r
 841                              select new StateTuple(file){C=file.ComputeShortHash(this.StatusNotification)};\r
 842             \r
 843             //Process folders first, to ensure folders appear on the sever as soon as possible\r
 844             folderTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, moves, processedPaths,token).Wait());\r
 845             \r
 846             fileTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, moves,processedPaths, token).Wait());\r
 847         }\r
 848 \r
 849 \r
 850         /*\r
 851          *                 //Use the queue to retry locked file hashing\r
 852                 var fileQueue = new ConcurrentQueue<FileSystemInfo>(localInfos);\r
 853                 \r
 854 \r
 855                 var results = new List<Tuple<FileSystemInfo, string>>();\r
 856                 var backoff = 0;\r
 857                 while (fileQueue.Count > 0)\r
 858                 {\r
 859                     FileSystemInfo file;\r
 860                     fileQueue.TryDequeue(out file);\r
 861                     using (ThreadContext.Stacks["File"].Push(file.FullName))\r
 862                     {\r
 863                         try\r
 864                         {\r
 865                             //Replace MD5 here, do the calc while syncing individual files\r
 866                             string hash ;\r
 867                             if (file is DirectoryInfo)\r
 868                                 hash = MD5_EMPTY;\r
 869                             else\r
 870                             {\r
 871                                 //Wait in case the FileAgent has requested a Pause\r
 872                                 await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
 873                                 \r
 874                                 using (StatusNotification.GetNotifier("Hashing {0}", "", file.Name))\r
 875                                 {\r
 876                                     hash = ((FileInfo)file).ComputeShortHash(StatusNotification);\r
 877                                     backoff = 0;\r
 878                                 }\r
 879                             }                            \r
 880                             results.Add(Tuple.Create(file, hash));\r
 881                         }\r
 882                         catch (IOException exc)\r
 883                         {\r
 884                             Log.WarnFormat("[HASH] File in use, will retry [{0}]", exc);\r
 885                             fileQueue.Enqueue(file);\r
 886                             //If this is the only enqueued file                            \r
 887                             if (fileQueue.Count != 1) continue;\r
 888                             \r
 889                             \r
 890                             //Increase delay\r
 891                             if (backoff<60000)\r
 892                                 backoff += 10000;\r
 893                             //Pause Polling for the specified time\r
 894                         }\r
 895                         if (backoff>0)\r
 896                             await PauseFor(backoff).ConfigureAwait(false);\r
 897                     }\r
 898                 }\r
 899 \r
 900                 return results;\r
 901 \r
 902          */\r
 903         private IEnumerable<StateTuple> MergeSources(IEnumerable<Tuple<string, ObjectInfo>> infos, IEnumerable<FileSystemInfo> files, List<FileState> states, ConcurrentDictionary<string, MovedEventArgs> moves)\r
 904         {\r
 905             var tuplesByPath = new Dictionary<string, StateTuple>();\r
 906             foreach (var info in files)\r
 907             {\r
 908                 var tuple = new StateTuple(info);\r
 909                 //Is this the target of a move event?\r
 910                 var moveArg = moves.Values.FirstOrDefault(arg => info.FullName.Equals(arg.FullPath, StringComparison.InvariantCultureIgnoreCase) \r
 911                     || info.FullName.IsAtOrBelow(arg.FullPath));\r
 912                 if (moveArg != null)\r
 913                 {\r
 914                     tuple.NewFullPath = info.FullName;\r
 915                     var relativePath = info.AsRelativeTo(moveArg.FullPath);                    \r
 916                     tuple.OldFullPath = Path.Combine(moveArg.OldFullPath, relativePath);\r
 917                     tuple.OldMD5 = states.FirstOrDefault(st => st.FilePath.Equals(tuple.OldFullPath, StringComparison.InvariantCultureIgnoreCase))\r
 918                                 .NullSafe(st => st.LastMD5);\r
 919                 }\r
 920 \r
 921                 tuplesByPath[tuple.FilePath] = tuple;\r
 922             }\r
 923             \r
 924 \r
 925             \r
 926             \r
 927             //For files that have state\r
 928             foreach (var state in states)\r
 929             {\r
 930                 StateTuple hashTuple;\r
 931 \r
 932                 \r
 933                 if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple))\r
 934                 {\r
 935                     hashTuple.FileState = state;\r
 936                     UpdateMD5(hashTuple);\r
 937                 }\r
 938                 else\r
 939                 {\r
 940                     var fsInfo = FileInfoExtensions.FromPath(state.FilePath);\r
 941                     hashTuple = new StateTuple {FileInfo = fsInfo, FileState = state};\r
 942 \r
 943                     //Is the source of a moved item?\r
 944                     var moveArg = moves.Values.FirstOrDefault(arg => state.FilePath.Equals(arg.OldFullPath,StringComparison.InvariantCultureIgnoreCase) \r
 945                         || state.FilePath.IsAtOrBelow(arg.OldFullPath));\r
 946                     if (moveArg != null)\r
 947                     {\r
 948                         var relativePath = state.FilePath.AsRelativeTo(moveArg.OldFullPath);\r
 949                         hashTuple.NewFullPath = Path.Combine(moveArg.FullPath,relativePath);\r
 950                         hashTuple.OldFullPath = state.FilePath;\r
 951                         //Do we have the old MD5?\r
 952                         hashTuple.OldMD5 = state.LastMD5;\r
 953                     }\r
 954 \r
 955 \r
 956                     tuplesByPath[state.FilePath] = hashTuple;\r
 957                 }\r
 958             }\r
 959             //for files that don't have state\r
 960             foreach (var tuple in tuplesByPath.Values.Where(t => t.FileState == null))\r
 961             {\r
 962                 UpdateMD5(tuple);\r
 963             }\r
 964 \r
 965             var tuplesByID = tuplesByPath.Values\r
 966                 .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null)\r
 967                 .ToDictionary(tuple=>tuple.FileState.ObjectID,tuple=>tuple);//new Dictionary<Guid, StateTuple>();\r
 968 \r
 969             foreach (var info in infos)\r
 970             {\r
 971                 StateTuple hashTuple;\r
 972                 var filePath = info.Item1;\r
 973                 var objectInfo = info.Item2;\r
 974                 var objectID = objectInfo.UUID;\r
 975 \r
 976                 if (objectID != _emptyGuid &&  tuplesByID.TryGetValue(objectID, out hashTuple))\r
 977                 {\r
 978                     hashTuple.ObjectInfo = objectInfo;                    \r
 979                 }\r
 980                 else if (tuplesByPath.TryGetValue(filePath, out hashTuple))\r
 981                 {\r
 982                     hashTuple.ObjectInfo = objectInfo;\r
 983                 }\r
 984                 else\r
 985                 {\r
 986 \r
 987                     \r
 988                     var fsInfo = FileInfoExtensions.FromPath(filePath);\r
 989                     hashTuple= new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo};\r
 990                     tuplesByPath[filePath] = hashTuple;\r
 991                     \r
 992                     if (objectInfo.UUID!=_emptyGuid)\r
 993                         tuplesByID[objectInfo.UUID] = hashTuple;\r
 994                 }\r
 995             }\r
 996             Debug.Assert(tuplesByPath.Values.All(t => t.HashesValid()));\r
 997             return tuplesByPath.Values;\r
 998         }\r
 999 \r
1000         private void  UpdateMD5(StateTuple hashTuple)\r
1001         {\r
1002             \r
1003             try\r
1004             {\r
1005                 var hash = Signature.MD5_EMPTY;\r
1006                 if (hashTuple.FileInfo is FileInfo)\r
1007                 {\r
1008                     var file = hashTuple.FileInfo as FileInfo;\r
1009                     var stateDate = hashTuple.NullSafe(h => h.FileState).NullSafe(s => s.LastWriteDate) ??\r
1010                                     DateTime.MinValue;\r
1011                     if (file.LastWriteTime - stateDate < TimeSpan.FromSeconds(1) &&\r
1012                         hashTuple.FileState.LastLength == file.Length)\r
1013                     {\r
1014                         hash = hashTuple.FileState.LastMD5;\r
1015                     }\r
1016                     else\r
1017                     {\r
1018                         //Modified, must calculate hash\r
1019                         hash = file.ComputeShortHash(StatusNotification);\r
1020                         StatusKeeper.UpdateLastMD5(file, hash);\r
1021                     }\r
1022                 }\r
1023                 hashTuple.C = hash;\r
1024                 hashTuple.MD5 = hash;\r
1025             }\r
1026             catch (IOException)\r
1027             {\r
1028                 hashTuple.Locked = true;\r
1029             }            \r
1030         }\r
1031 \r
1032         /// <summary>\r
1033         /// Returns the latest LastModified date from the list of objects, but only if it is before\r
1034         /// than the threshold value\r
1035         /// </summary>\r
1036         /// <param name="threshold"></param>\r
1037         /// <param name="cloudObjects"></param>\r
1038         /// <returns></returns>\r
1039         private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
1040         {\r
1041             DateTime? maxDate = null;\r
1042             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
1043                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
1044             if (maxDate == null || maxDate == DateTime.MinValue)\r
1045                 return threshold;\r
1046             if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate)\r
1047                 return maxDate;\r
1048             return threshold;\r
1049         }\r
1050 \r
1051         /// <summary>\r
1052         /// Returns the latest LastModified date from the list of objects, but only if it is after\r
1053         /// the threshold value\r
1054         /// </summary>\r
1055         /// <param name="threshold"></param>\r
1056         /// <param name="cloudObjects"></param>\r
1057         /// <returns></returns>\r
1058         private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
1059         {\r
1060             DateTime? maxDate = null;\r
1061             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
1062                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
1063             if (maxDate == null || maxDate == DateTime.MinValue)\r
1064                 return threshold;\r
1065             if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate)\r
1066                 return maxDate;\r
1067             return threshold;\r
1068         }\r
1069 \r
1070         readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
1071         private bool _pause;\r
1072         private readonly string _emptyGuid = Guid.Empty.ToString();\r
1073 \r
1074 \r
1075 \r
1076         private void ReportConflictForMismatch(string localFilePath)\r
1077         {\r
1078             if (String.IsNullOrWhiteSpace(localFilePath))\r
1079                 throw new ArgumentNullException("localFilePath");\r
1080             Contract.EndContractBlock();\r
1081 \r
1082             StatusKeeper.SetFileState(localFilePath, FileStatus.Conflict, FileOverlayStatus.Conflict, "File changed at the server");\r
1083             UpdateStatus(PithosStatus.HasConflicts);\r
1084             var message = String.Format("Conflict detected for file {0}", localFilePath);\r
1085             Log.Warn(message);\r
1086             StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
1087         }\r
1088 \r
1089 \r
1090         /// <summary>\r
1091         /// Notify the UI to update the visual status\r
1092         /// </summary>\r
1093         /// <param name="status"></param>\r
1094         private void UpdateStatus(PithosStatus status)\r
1095         {\r
1096             try\r
1097             {\r
1098                 StatusNotification.SetPithosStatus(status);\r
1099                 //StatusNotification.Notify(new Notification());\r
1100             }\r
1101             catch (Exception exc)\r
1102             {\r
1103                 //Failure is not critical, just log it\r
1104                 Log.Warn("Error while updating status", exc);\r
1105             }\r
1106         }\r
1107 \r
1108         private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)\r
1109         {\r
1110             var containerPaths = from container in containers\r
1111                                  let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)\r
1112                                  where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)\r
1113                                  select containerPath;\r
1114 \r
1115             foreach (var path in containerPaths)\r
1116             {\r
1117                 Directory.CreateDirectory(path);\r
1118             }\r
1119         }\r
1120 \r
1121         public void AddAccount(AccountInfo accountInfo)\r
1122         {\r
1123             //Avoid adding a duplicate accountInfo\r
1124             _accounts.TryAdd(accountInfo.AccountKey, accountInfo);\r
1125         }\r
1126 \r
1127         public void RemoveAccount(AccountInfo accountInfo)\r
1128         {\r
1129             AccountInfo account;\r
1130             _accounts.TryRemove(accountInfo.AccountKey, out account);\r
1131 \r
1132             SnapshotDifferencer differencer;\r
1133             _differencer.Differencers.TryRemove(accountInfo.AccountKey, out differencer);\r
1134         }\r
1135        \r
1136     }\r
1137 }\r