code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/PollAgent.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="PollAgent.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 \r
  43 using System.Collections.Concurrent;\r
  44 using System.ComponentModel.Composition;\r
  45 using System.Diagnostics;\r
  46 using System.Diagnostics.Contracts;\r
  47 using System.IO;\r
  48 using System.Linq.Expressions;\r
  49 using System.Reflection;\r
  50 using System.Security.Cryptography;\r
  51 using System.Threading;\r
  52 using System.Threading.Tasks;\r
  53 using System.Threading.Tasks.Dataflow;\r
  54 using Castle.ActiveRecord;\r
  55 using Pithos.Interfaces;\r
  56 using Pithos.Network;\r
  57 using log4net;\r
  58 \r
  59 namespace Pithos.Core.Agents\r
  60 {\r
  61     using System;\r
  62     using System.Collections.Generic;\r
  63     using System.Linq;\r
  64 \r
  65     public class PollRequest\r
  66     {\r
  67         public DateTime? Since { get; set; }\r
  68         public IEnumerable<string> Batch { get; set; }\r
  69     }\r
  70 \r
  71     [DebuggerDisplay("{FilePath} C:{C} L:{L} S:{S}")]\r
  72     public class StateTuple\r
  73     {\r
  74         public string FilePath { get; private set; }\r
  75 \r
  76         public string MD5 { get; set; }\r
  77 \r
  78         public string L\r
  79         {\r
  80             get { return FileState==null?null:FileState.Checksum; }\r
  81         }\r
  82 \r
  83         private string _c;\r
  84         public string C\r
  85         {\r
  86             get { return _c; }\r
  87             set {\r
  88                 _c = String.IsNullOrWhiteSpace(value) ? null : value;\r
  89             }\r
  90         }\r
  91 \r
  92         public string S\r
  93         {\r
  94             get { return ObjectInfo == null ? null : ObjectInfo.X_Object_Hash; }\r
  95         }\r
  96 \r
  97         private FileSystemInfo _fileInfo;\r
  98         private TreeHash _merkle;\r
  99 \r
 100         public FileSystemInfo FileInfo\r
 101         {\r
 102             get { return _fileInfo; }\r
 103             set\r
 104             {\r
 105                 _fileInfo = value;\r
 106                 FilePath = value.FullName;\r
 107             }\r
 108         }\r
 109 \r
 110         public FileState FileState { get; set; }\r
 111         public ObjectInfo ObjectInfo{ get; set; }\r
 112 \r
 113 \r
 114         public TreeHash Merkle\r
 115         {\r
 116             get {\r
 117                 return _merkle;\r
 118             }\r
 119             set {\r
 120                 _merkle = value;\r
 121                 C = _merkle.TopHash.ToHashString();\r
 122             }\r
 123         }\r
 124 \r
 125         public StateTuple() { }\r
 126 \r
 127         public StateTuple(FileSystemInfo info)\r
 128         {\r
 129             FileInfo = info;\r
 130         }\r
 131 \r
 132 \r
 133     }\r
 134 \r
 135 \r
 136     /// <summary>\r
 137     /// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all\r
 138     /// objects and compares it with a previously cached version to detect differences. \r
 139     /// New files are downloaded, missing files are deleted from the local file system and common files are compared\r
 140     /// to determine the appropriate action\r
 141     /// </summary>\r
 142     [Export]\r
 143     public class PollAgent\r
 144     {\r
 145         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
 146 \r
 147         [System.ComponentModel.Composition.Import]\r
 148         public IStatusKeeper StatusKeeper { get; set; }\r
 149 \r
 150         [System.ComponentModel.Composition.Import]\r
 151         public IPithosSettings Settings { get; set; }\r
 152 \r
 153         [System.ComponentModel.Composition.Import]\r
 154         public NetworkAgent NetworkAgent { get; set; }\r
 155 \r
 156         [System.ComponentModel.Composition.Import]\r
 157         public Selectives Selectives { get; set; }\r
 158 \r
 159         public IStatusNotification StatusNotification { get; set; }\r
 160 \r
 161         private CancellationTokenSource _currentOperationCancellation = new CancellationTokenSource();\r
 162 \r
 163         public void CancelCurrentOperation()\r
 164         {\r
 165             //What does it mean to cancel the current upload/download?\r
 166             //Obviously, the current operation will be cancelled by throwing\r
 167             //a cancellation exception.\r
 168             //\r
 169             //The default behavior is to retry any operations that throw.\r
 170             //Obviously this is not what we want in this situation.\r
 171             //The cancelled operation should NOT bea retried. \r
 172             //\r
 173             //This can be done by catching the cancellation exception\r
 174             //and avoiding the retry.\r
 175             //\r
 176 \r
 177             //Have to reset the cancellation source - it is not possible to reset the source\r
 178             //Have to prevent a case where an operation requests a token from the old source\r
 179             var oldSource = Interlocked.Exchange(ref _currentOperationCancellation, new CancellationTokenSource());\r
 180             oldSource.Cancel();\r
 181 \r
 182         }\r
 183 \r
 184         public bool Pause\r
 185         {\r
 186             get {\r
 187                 return _pause;\r
 188             }\r
 189             set {\r
 190                 _pause = value;                \r
 191                 if (!_pause)\r
 192                     _unPauseEvent.Set();\r
 193                 else\r
 194                 {\r
 195                     _unPauseEvent.Reset();\r
 196                 }\r
 197             }\r
 198         }\r
 199 \r
 200         private bool _firstPoll = true;\r
 201 \r
 202         //The Sync Event signals a manual synchronisation\r
 203         private readonly AsyncManualResetEvent _syncEvent = new AsyncManualResetEvent();\r
 204 \r
 205         private readonly AsyncManualResetEvent _unPauseEvent = new AsyncManualResetEvent(true);\r
 206 \r
 207         private readonly ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>();\r
 208         private readonly ConcurrentDictionary<Uri, AccountInfo> _accounts = new ConcurrentDictionary<Uri,AccountInfo>();\r
 209 \r
 210         //private readonly ActionBlock<PollRequest>  _pollAction;\r
 211 \r
 212         public PollAgent()\r
 213         {\r
 214             //_pollAction=new ActionBlock<PollRequest>(p=>ProcessPollRequest(p));\r
 215         }\r
 216 \r
 217 \r
 218         /*private void ProcessPollRequest(PollRequest request)\r
 219         {\r
 220 \r
 221             if (request.Since == null && request.Batch != null)\r
 222             {\r
 223                 _batchQueue.Enqueue(request.Batch);\r
 224                 _syncEvent.Set();                \r
 225             }\r
 226             else \r
 227             {\r
 228                 PollRemoteFiles(request.Since).Wait();\r
 229             }\r
 230         }*/\r
 231         /// <summary>\r
 232         /// Start a manual synchronization\r
 233         /// </summary>\r
 234         public void SynchNow(IEnumerable<string> paths=null)\r
 235         {\r
 236             _batchQueue.Enqueue(paths);\r
 237             _syncEvent.Set();                \r
 238 \r
 239             //_pollAction.Post(new PollRequest {Batch = paths});\r
 240         }\r
 241 \r
 242         readonly ConcurrentQueue<IEnumerable<string>> _batchQueue=new ConcurrentQueue<IEnumerable<string>>();\r
 243 \r
 244         /// <summary>\r
 245         /// Remote files are polled periodically. Any changes are processed\r
 246         /// </summary>\r
 247         /// <param name="since"></param>\r
 248         /// <returns></returns>\r
 249         public  void PollRemoteFiles(DateTime? since = null)\r
 250         {\r
 251             if (Log.IsDebugEnabled)\r
 252                 Log.DebugFormat("Polling changes after [{0}]",since);\r
 253 \r
 254             Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
 255 \r
 256             //GC.Collect();\r
 257 \r
 258             using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
 259             {\r
 260                 //If this poll fails, we will retry with the same since value\r
 261                 var nextSince = since;\r
 262                 try\r
 263                 {\r
 264                     _unPauseEvent.Wait();\r
 265                     UpdateStatus(PithosStatus.PollSyncing);\r
 266 \r
 267                     var accountBatches=new Dictionary<Uri, IEnumerable<string>>();\r
 268                     IEnumerable<string> batch = null;\r
 269                     if (_batchQueue.TryDequeue(out batch) && batch != null)\r
 270                         foreach (var account in _accounts.Values)\r
 271                         {\r
 272                             var accountBatch = batch.Where(path => path.IsAtOrBelow(account.AccountPath));\r
 273                             accountBatches[account.AccountKey] = accountBatch;\r
 274                         }\r
 275 \r
 276                     var tasks = new List<Task<DateTime?>>();\r
 277                     foreach(var accountInfo in _accounts.Values)\r
 278                     {\r
 279                         IEnumerable<string> accountBatch ;\r
 280                         accountBatches.TryGetValue(accountInfo.AccountKey,out accountBatch);\r
 281                         var t=ProcessAccountFiles (accountInfo, accountBatch, since);\r
 282                         tasks.Add(t);\r
 283                     }\r
 284 \r
 285                     var nextTimes=TaskEx.WhenAll(tasks.ToList()).Result;\r
 286 \r
 287                     _firstPoll = false;\r
 288                     //Reschedule the poll with the current timestamp as a "since" value\r
 289 \r
 290                     if (nextTimes.Length>0)\r
 291                         nextSince = nextTimes.Min();\r
 292                     if (Log.IsDebugEnabled)\r
 293                         Log.DebugFormat("Next Poll at [{0}]",nextSince);\r
 294                 }\r
 295                 catch (Exception ex)\r
 296                 {\r
 297                     Log.ErrorFormat("Error while processing accounts\r\n{0}", ex);\r
 298                     //In case of failure retry with the same "since" value\r
 299                 }\r
 300 \r
 301                 UpdateStatus(PithosStatus.PollComplete);\r
 302                 //The multiple try blocks are required because we can't have an await call\r
 303                 //inside a finally block\r
 304                 //TODO: Find a more elegant solution for reschedulling in the event of an exception\r
 305                 try\r
 306                 {\r
 307                     //Wait for the polling interval to pass or the Sync event to be signalled\r
 308                     nextSince = WaitForScheduledOrManualPoll(nextSince).Result;\r
 309                 }\r
 310                 finally\r
 311                 {\r
 312                     //Ensure polling is scheduled even in case of error\r
 313                     PollRemoteFiles(nextSince);\r
 314                     //_pollAction.Post(new PollRequest {Since = nextSince});\r
 315                 }\r
 316             }\r
 317         }\r
 318 \r
 319         /// <summary>\r
 320         /// Wait for the polling period to expire or a manual sync request\r
 321         /// </summary>\r
 322         /// <param name="since"></param>\r
 323         /// <returns></returns>\r
 324         private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
 325         {\r
 326             var sync = _syncEvent.WaitAsync();\r
 327             var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval));\r
 328             \r
 329             var signaledTask = await TaskEx.WhenAny(sync, wait);\r
 330             \r
 331             //Pausing takes precedence over manual sync or awaiting\r
 332             _unPauseEvent.Wait();\r
 333             \r
 334             //Wait for network processing to finish before polling\r
 335             var pauseTask=NetworkAgent.ProceedEvent.WaitAsync();\r
 336             await TaskEx.WhenAll(signaledTask, pauseTask);\r
 337 \r
 338             //If polling is signalled by SynchNow, ignore the since tag\r
 339             if (sync.IsCompleted)\r
 340             {\r
 341                 //TODO: Must convert to AutoReset\r
 342                 _syncEvent.Reset();\r
 343                 return null;\r
 344             }\r
 345             return since;\r
 346         }\r
 347 \r
 348         public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, IEnumerable<string> accountBatch, DateTime? since = null)\r
 349         {\r
 350             if (accountInfo == null)\r
 351                 throw new ArgumentNullException("accountInfo");\r
 352             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
 353                 throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
 354             Contract.EndContractBlock();\r
 355 \r
 356 \r
 357             using (ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))\r
 358             {\r
 359 \r
 360                 await NetworkAgent.GetDeleteAwaiter();\r
 361 \r
 362                 Log.Info("Scheduled");\r
 363                 var client = new CloudFilesClient(accountInfo);\r
 364 \r
 365                 //We don't need to check the trash container\r
 366                 var containers = client.ListContainers(accountInfo.UserName)\r
 367                     .Where(c=>c.Name!="trash")\r
 368                     .ToList();\r
 369 \r
 370 \r
 371                 CreateContainerFolders(accountInfo, containers);\r
 372 \r
 373                 //The nextSince time fallback time is the same as the current.\r
 374                 //If polling succeeds, the next Since time will be the smallest of the maximum modification times\r
 375                 //of the shared and account objects\r
 376                 var nextSince = since;\r
 377 \r
 378                 try\r
 379                 {\r
 380                     //Wait for any deletions to finish\r
 381                     await NetworkAgent.GetDeleteAwaiter();\r
 382                     //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
 383                     //than delete a file that was created while we were executing the poll                    \r
 384 \r
 385                     //Get the list of server objects changed since the last check\r
 386                     //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
 387                     var listObjects = (from container in containers\r
 388                                        select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
 389                                              client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
 390 \r
 391                     var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
 392                         client.ListSharedObjects(since), "shared");\r
 393                     listObjects.Add(listShared);\r
 394                     var listTasks = await Task.Factory.WhenAll(listObjects.ToArray());\r
 395 \r
 396                     using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
 397                     {\r
 398                         var dict = listTasks.ToDictionary(t => t.AsyncState);\r
 399 \r
 400                         //Get all non-trash objects. Remember, the container name is stored in AsyncState\r
 401                         var remoteObjects = (from objectList in listTasks\r
 402                                             where (string)objectList.AsyncState != "trash"\r
 403                                             from obj in objectList.Result\r
 404                                             orderby obj.Bytes ascending \r
 405                                             select obj).ToList();\r
 406                         \r
 407                         //Get the latest remote object modification date, only if it is after\r
 408                         //the original since date\r
 409                         nextSince = GetLatestDateAfter(nextSince, remoteObjects);\r
 410 \r
 411                         var sharedObjects = dict["shared"].Result;\r
 412 \r
 413                         //DON'T process trashed files\r
 414                         //If some files are deleted and added again to a folder, they will be deleted\r
 415                         //even though they are new.\r
 416                         //We would have to check file dates and hashes to ensure that a trashed file\r
 417                         //can be deleted safely from the local hard drive.\r
 418                         /*\r
 419                         //Items with the same name, hash may be both in the container and the trash\r
 420                         //Don't delete items that exist in the container\r
 421                         var realTrash = from trash in trashObjects\r
 422                                         where\r
 423                                             !remoteObjects.Any(\r
 424                                                 info => info.Name == trash.Name && info.Hash == trash.Hash)\r
 425                                    8     select trash;\r
 426                         ProcessTrashedFiles(accountInfo, realTrash);\r
 427 */\r
 428 \r
 429                         var cleanRemotes = (from info in remoteObjects.Union(sharedObjects)\r
 430                                             let name = info.Name??""\r
 431                                             where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&\r
 432                                                   !name.StartsWith(FolderConstants.CacheFolder + "/",\r
 433                                                                    StringComparison.InvariantCultureIgnoreCase)\r
 434                                             select info).ToList();\r
 435 \r
 436                         if (_firstPoll)\r
 437                             StatusKeeper.CleanupOrphanStates();\r
 438                         \r
 439                         var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
 440                         var currentRemotes = differencer.Current.ToList();\r
 441                         StatusKeeper.CleanupStaleStates(accountInfo, currentRemotes);\r
 442 \r
 443                         //var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
 444 \r
 445                         //May have to wait if the FileAgent has asked for a Pause, due to local changes\r
 446                         await _unPauseEvent.WaitAsync();\r
 447 \r
 448                         //Get the local files here                        \r
 449                         var agent = AgentLocator<FileAgent>.Get(accountInfo.AccountPath);                        \r
 450                         //TODO: Pass the batch here as well\r
 451                         var files = await LoadLocalFileTuples(accountInfo, accountBatch);\r
 452 \r
 453                         var states = FileState.Queryable.ToList();                        \r
 454                         \r
 455                         var infos = (from remote in currentRemotes\r
 456                                     let path = remote.RelativeUrlToFilePath(accountInfo.UserName)\r
 457                                     let info=agent.GetFileSystemInfo(path)\r
 458                                     select Tuple.Create(info.FullName,remote))\r
 459                                     .ToList();\r
 460 \r
 461                         var token = _currentOperationCancellation.Token;\r
 462 \r
 463                         var tuples = MergeSources(infos, files, states).ToList();\r
 464 \r
 465                         //Process only the changes in the batch file, if one exists\r
 466                         var stateTuples = accountBatch==null?tuples:tuples.Where(t => accountBatch.Contains(t.FilePath));\r
 467                         foreach (var tuple in stateTuples)\r
 468                         {\r
 469                             await _unPauseEvent.WaitAsync();\r
 470 \r
 471                             //Set the Merkle Hash\r
 472                             SetMerkleHash(accountInfo, tuple);\r
 473 \r
 474                             SyncSingleItem(accountInfo, tuple, agent, token);\r
 475 \r
 476                         }\r
 477 \r
 478 \r
 479                         //On the first run\r
 480 /*\r
 481                         if (_firstPoll)\r
 482                         {\r
 483                             MarkSuspectedDeletes(accountInfo, cleanRemotes);\r
 484                         }\r
 485 */\r
 486 \r
 487 \r
 488                         Log.Info("[LISTENER] End Processing");\r
 489                     }\r
 490                 }\r
 491                 catch (Exception ex)\r
 492                 {\r
 493                     Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);\r
 494                     return nextSince;\r
 495                 }\r
 496 \r
 497                 Log.Info("[LISTENER] Finished");\r
 498                 return nextSince;\r
 499             }\r
 500         }\r
 501 \r
 502         private static void SetMerkleHash(AccountInfo accountInfo, StateTuple tuple)\r
 503         {\r
 504             //The Merkle hash for directories is that of an empty buffer\r
 505             if (tuple.FileInfo is DirectoryInfo)\r
 506                 tuple.C = MERKLE_EMPTY;\r
 507             else if (tuple.FileState != null && tuple.MD5 == tuple.FileState.ShortHash)\r
 508             {\r
 509                 //If there is a state whose MD5 matches, load the merkle hash from the file state\r
 510                 //insteaf of calculating it\r
 511                 tuple.C = tuple.FileState.Checksum;                              \r
 512             }\r
 513             else\r
 514             {\r
 515                 tuple.Merkle = TaskEx.Run(()=> Signature.CalculateTreeHash(tuple.FileInfo, accountInfo.BlockSize, accountInfo.BlockHash)).Result;\r
 516                 //tuple.C=tuple.Merkle.TopHash.ToHashString();                \r
 517             }\r
 518         }\r
 519 \r
 520         private async Task<List<Tuple<FileSystemInfo, string>>> LoadLocalFileTuples(AccountInfo accountInfo,IEnumerable<string> batch )\r
 521         {\r
 522             using (ThreadContext.Stacks["Account Files Hashing"].Push(accountInfo.UserName))\r
 523             {\r
 524                 var batchPaths = (batch==null)?new List<string>():batch.ToList();\r
 525                 IEnumerable<FileSystemInfo> localInfos=AgentLocator<FileAgent>.Get(accountInfo.AccountPath)\r
 526                                                         .EnumerateFileSystemInfos();\r
 527                 if (batchPaths.Count>0)\r
 528                     localInfos= localInfos.Where(fi => batchPaths.Contains(fi.FullName));\r
 529                 \r
 530                 //Use the queue to retry locked file hashing\r
 531                 var fileQueue = new Queue<FileSystemInfo>(localInfos);\r
 532                 var hasher = MD5.Create();\r
 533 \r
 534                 var results = new List<Tuple<FileSystemInfo, string>>();\r
 535                 var backoff = 0;\r
 536                 while (fileQueue.Count > 0)\r
 537                 {\r
 538                     var file = fileQueue.Dequeue();\r
 539                     using (ThreadContext.Stacks["File"].Push(file.FullName))\r
 540                     {\r
 541                         /*\r
 542                                                 Signature.CalculateTreeHash(file, accountInfo.BlockSize,\r
 543                                                                                                  accountInfo.BlockHash).\r
 544                                                                          TopHash.ToHashString()\r
 545                         */\r
 546                         try\r
 547                         {\r
 548                             //Replace MD5 here, do the calc while syncing individual files\r
 549                             string hash ;\r
 550                             if (file is DirectoryInfo)\r
 551                                 hash = MERKLE_EMPTY;\r
 552                             else\r
 553                             {\r
 554                                 //Wait in case the FileAgent has requested a Pause\r
 555                                 await _unPauseEvent.WaitAsync();\r
 556                                 \r
 557                                 using (StatusNotification.GetNotifier("Hashing {0}", "Finished hashing {0}", file.Name))\r
 558                                 using (var stream = (file as FileInfo).OpenRead())\r
 559                                 {                                    \r
 560                                     hash = hasher.ComputeHash(stream).ToHashString();\r
 561                                     backoff = 0;\r
 562                                 }\r
 563                             }                            \r
 564                             results.Add(Tuple.Create(file, hash));\r
 565                         }\r
 566                         catch (IOException exc)\r
 567                         {\r
 568                             Log.WarnFormat("[HASH] File in use, will retry [{0}]", exc);\r
 569                             fileQueue.Enqueue(file);\r
 570                             //If this is the only enqueued file                            \r
 571                             if (fileQueue.Count != 1) continue;\r
 572                             \r
 573                             \r
 574                             //Increase delay\r
 575                             if (backoff<60000)\r
 576                                 backoff += 10000;\r
 577                             //Pause Polling for the specified time\r
 578                         }\r
 579                         if (backoff>0)\r
 580                             await PauseFor(backoff);\r
 581                     }\r
 582                 }\r
 583 \r
 584                 return results;\r
 585             }\r
 586         }\r
 587 \r
 588         /// <summary>\r
 589         /// Wait and Pause the agent while waiting\r
 590         /// </summary>\r
 591         /// <param name="backoff"></param>\r
 592         /// <returns></returns>\r
 593         private async Task PauseFor(int backoff)\r
 594         {\r
 595 \r
 596             Pause = true;\r
 597             await TaskEx.Delay(backoff);\r
 598             Pause = false;\r
 599         }\r
 600 \r
 601         private void SyncSingleItem(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, CancellationToken token)\r
 602         {\r
 603             Log.DebugFormat("Sync [{0}] C:[{1}] L:[{2}] S:[{3}]",tuple.FilePath,tuple.C,tuple.L,tuple.S);\r
 604 \r
 605             var localFilePath = tuple.FilePath;\r
 606             //Don't use the tuple info, it may have been deleted\r
 607             var localInfo = FileInfoExtensions.FromPath(localFilePath);\r
 608 \r
 609 \r
 610             var isUnselectedRootFolder = agent.IsUnselectedRootFolder(tuple.FilePath);\r
 611 \r
 612             //Unselected root folders that have not yet been uploaded should be uploaded and added to the \r
 613             //selective folders\r
 614 \r
 615             if (!Selectives.IsSelected(accountInfo, localFilePath) && !(isUnselectedRootFolder && tuple.ObjectInfo==null) )                \r
 616                 return;\r
 617 \r
 618             // Local file unchanged? If both C and L are null, make sure it's because \r
 619             //both the file is missing and the state checksum is not missing\r
 620             if (tuple.C == tuple.L /*&& (localInfo.Exists || tuple.FileState == null)*/)\r
 621             {\r
 622                 //No local changes\r
 623                 //Server unchanged?\r
 624                 if (tuple.S == tuple.L)\r
 625                 {\r
 626                     // No server changes\r
 627                     //Has the file been renamed on the server?\r
 628                     MoveForServerMove(accountInfo, tuple);\r
 629                 }\r
 630                 else\r
 631                 {\r
 632                     //Different from server\r
 633                     //Does the server file exist?\r
 634                     if (tuple.S == null)\r
 635                     {\r
 636                         //Server file doesn't exist\r
 637                         //deleteObjectFromLocal()\r
 638                         StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
 639                                                   FileOverlayStatus.Deleted, "");\r
 640                         agent.Delete(localFilePath);\r
 641                         //updateRecord(Remove C, L)\r
 642                         StatusKeeper.ClearFileStatus(localFilePath);\r
 643                     }\r
 644                     else\r
 645                     {\r
 646                         //Server file exists\r
 647                         //downloadServerObject() // Result: L = S\r
 648                         //If the file has moved on the server, move it locally before downloading\r
 649                         var targetPath = MoveForServerMove(accountInfo, tuple);\r
 650 \r
 651                         StatusKeeper.SetFileState(targetPath, FileStatus.Modified,\r
 652                                                   FileOverlayStatus.Modified, "");\r
 653                         NetworkAgent.Downloader.DownloadCloudFile(accountInfo,\r
 654                                                                   tuple.ObjectInfo,\r
 655                                                                   targetPath, tuple.Merkle, token).Wait(token);\r
 656                         //updateRecord( L = S )\r
 657                         StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
 658                                                         tuple.ObjectInfo.X_Object_Hash);\r
 659 \r
 660                         StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
 661 \r
 662                         /*\r
 663                                                         StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
 664                                                                                   FileOverlayStatus.Normal, "");\r
 665                             */\r
 666                     }\r
 667                 }\r
 668 \r
 669             }\r
 670             else\r
 671             {\r
 672                 //Local changes found\r
 673 \r
 674                 //Server unchanged?\r
 675                 if (tuple.S == tuple.L)\r
 676                 {\r
 677                     //The FileAgent selective sync checks for new root folder files\r
 678                     if (!agent.Ignore(localFilePath))\r
 679                     {\r
 680                         if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
 681                         {\r
 682                             //deleteObjectFromServer()\r
 683                             DeleteCloudFile(accountInfo, tuple);\r
 684                             //updateRecord( Remove L, S)                  \r
 685                         }\r
 686                         else\r
 687                         {\r
 688                             //uploadLocalObject() // Result: S = C, L = S                        \r
 689 \r
 690                             //Debug.Assert(tuple.FileState !=null);\r
 691                             var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
 692                                                                accountInfo.BlockSize, accountInfo.BlockHash,\r
 693                                                                "Poll", isUnselectedRootFolder);\r
 694                             NetworkAgent.Uploader.UploadCloudFile(action, tuple.Merkle, token).Wait(token);\r
 695 \r
 696                             //updateRecord( S = C )\r
 697                             //State updated by the uploader\r
 698 \r
 699                             if (isUnselectedRootFolder)\r
 700                             {\r
 701                                 ProcessChildren(accountInfo, tuple, agent, token);\r
 702                             }\r
 703                         }\r
 704                     }\r
 705                 }\r
 706                 else\r
 707                 {\r
 708                     if (tuple.C == tuple.S)\r
 709                     {\r
 710                         // (Identical Changes) Result: L = S\r
 711                         //doNothing()\r
 712                         //Detect server moves\r
 713                         var targetPath = MoveForServerMove(accountInfo, tuple);\r
 714                         StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
 715                     }\r
 716                     else\r
 717                     {\r
 718                         if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
 719                         {\r
 720                             //deleteObjectFromServer()\r
 721                             DeleteCloudFile(accountInfo, tuple);\r
 722                             //updateRecord(Remove L, S)                  \r
 723                         }\r
 724                             //If both the local and server files are missing, the state is stale\r
 725                         else if (!localInfo.Exists && (tuple.S == null || tuple.ObjectInfo == null))\r
 726                         {\r
 727                             StatusKeeper.ClearFileStatus(localInfo.FullName);\r
 728                         }\r
 729                         else\r
 730                         {\r
 731                             ReportConflictForMismatch(localFilePath);\r
 732                             //identifyAsConflict() // Manual action required\r
 733                         }\r
 734                     }\r
 735                 }\r
 736             }\r
 737         }\r
 738 \r
 739         private string MoveForServerMove(AccountInfo accountInfo, StateTuple tuple)\r
 740         {\r
 741             if (tuple.ObjectInfo == null)\r
 742                 return null;\r
 743             var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 744             var serverPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
 745             \r
 746             //Compare Case Insensitive\r
 747             if (String.Equals(tuple.FilePath ,serverPath,StringComparison.InvariantCultureIgnoreCase)) return serverPath;\r
 748 \r
 749             if (tuple.FileInfo.Exists)\r
 750             {                    \r
 751                 var fi = tuple.FileInfo as FileInfo;\r
 752                 if (fi != null)\r
 753                     fi.MoveTo(serverPath);\r
 754                 var di = tuple.FileInfo as DirectoryInfo;\r
 755                 if (di != null)\r
 756                     di.MoveTo(serverPath);\r
 757                 StatusKeeper.StoreInfo(serverPath, tuple.ObjectInfo);\r
 758             }\r
 759             else\r
 760             {\r
 761                 Debug.Assert(false, "File does not exist");\r
 762             }\r
 763             return serverPath;\r
 764         }\r
 765 \r
 766         private void DeleteCloudFile(AccountInfo accountInfo, StateTuple tuple)\r
 767         {\r
 768             StatusKeeper.SetFileState(tuple.FilePath, FileStatus.Deleted,\r
 769                                       FileOverlayStatus.Deleted, "");\r
 770             NetworkAgent.DeleteAgent.DeleteCloudFile(accountInfo, tuple.ObjectInfo);\r
 771             StatusKeeper.ClearFileStatus(tuple.FilePath);\r
 772         }\r
 773 \r
 774         private void ProcessChildren(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, CancellationToken token)\r
 775         {\r
 776 \r
 777             var dirInfo = tuple.FileInfo as DirectoryInfo;\r
 778             var folderTuples = from folder in dirInfo.EnumerateDirectories("*", SearchOption.AllDirectories)\r
 779                                select new StateTuple(folder);\r
 780             var fileTuples = from file in dirInfo.EnumerateFiles("*", SearchOption.AllDirectories)\r
 781                              select new StateTuple(file);\r
 782             \r
 783             //Process folders first, to ensure folders appear on the sever as soon as possible\r
 784             folderTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, token));\r
 785             \r
 786             fileTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, token));\r
 787         }\r
 788 \r
 789         private static IEnumerable<StateTuple> MergeSources(\r
 790             IEnumerable<Tuple<string, ObjectInfo>> infos, \r
 791             IEnumerable<Tuple<FileSystemInfo, string>> files, \r
 792             IEnumerable<FileState> states)\r
 793         {\r
 794             var tuplesByPath = new Dictionary<string, StateTuple>();\r
 795             foreach (var file in files)\r
 796             {\r
 797                 var fsInfo = file.Item1;\r
 798                 var fileHash = fsInfo is DirectoryInfo? MERKLE_EMPTY:file.Item2;\r
 799 \r
 800                 tuplesByPath[fsInfo.FullName] = new StateTuple {FileInfo = fsInfo, MD5 = fileHash};\r
 801             }\r
 802             foreach (var state in states)\r
 803             {\r
 804                 StateTuple hashTuple;\r
 805                 if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple))\r
 806                 {\r
 807                     hashTuple.FileState = state;\r
 808                 }\r
 809                 else\r
 810                 {\r
 811                     var fsInfo = FileInfoExtensions.FromPath(state.FilePath);\r
 812                     tuplesByPath[state.FilePath] = new StateTuple {FileInfo = fsInfo, FileState = state};\r
 813                 }\r
 814             }\r
 815 \r
 816             var tuplesByID = tuplesByPath.Values\r
 817                 .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null)\r
 818                 .ToDictionary(tuple=>tuple.FileState.ObjectID,tuple=>tuple);//new Dictionary<Guid, StateTuple>();\r
 819 \r
 820             foreach (var info in infos)\r
 821             {\r
 822                 StateTuple hashTuple;\r
 823                 var filePath = info.Item1;\r
 824                 var objectInfo = info.Item2;\r
 825                 var objectID = objectInfo.UUID;\r
 826 \r
 827                 if (tuplesByID.TryGetValue(objectID, out hashTuple))\r
 828                 {\r
 829                     hashTuple.ObjectInfo = objectInfo;                    \r
 830                 }\r
 831                 else if (tuplesByPath.TryGetValue(filePath, out hashTuple))\r
 832                 {\r
 833                     hashTuple.ObjectInfo = objectInfo;\r
 834                 }\r
 835                 else\r
 836                 {\r
 837                     var fsInfo = FileInfoExtensions.FromPath(filePath);\r
 838                     var tuple = new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo};\r
 839                     tuplesByPath[filePath] = tuple;\r
 840                     tuplesByID[objectInfo.UUID] = tuple;\r
 841                 }\r
 842             }\r
 843             return tuplesByPath.Values;\r
 844         }\r
 845 \r
 846         /// <summary>\r
 847         /// Returns the latest LastModified date from the list of objects, but only if it is before\r
 848         /// than the threshold value\r
 849         /// </summary>\r
 850         /// <param name="threshold"></param>\r
 851         /// <param name="cloudObjects"></param>\r
 852         /// <returns></returns>\r
 853         private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
 854         {\r
 855             DateTime? maxDate = null;\r
 856             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
 857                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
 858             if (maxDate == null || maxDate == DateTime.MinValue)\r
 859                 return threshold;\r
 860             if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate)\r
 861                 return maxDate;\r
 862             return threshold;\r
 863         }\r
 864 \r
 865         /// <summary>\r
 866         /// Returns the latest LastModified date from the list of objects, but only if it is after\r
 867         /// the threshold value\r
 868         /// </summary>\r
 869         /// <param name="threshold"></param>\r
 870         /// <param name="cloudObjects"></param>\r
 871         /// <returns></returns>\r
 872         private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
 873         {\r
 874             DateTime? maxDate = null;\r
 875             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
 876                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
 877             if (maxDate == null || maxDate == DateTime.MinValue)\r
 878                 return threshold;\r
 879             if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate)\r
 880                 return maxDate;\r
 881             return threshold;\r
 882         }\r
 883 \r
 884         readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
 885         private Dictionary<Uri, List<Uri>> _selectiveUris = new Dictionary<Uri, List<Uri>>();\r
 886         private bool _pause;\r
 887         private static string MERKLE_EMPTY = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";\r
 888 \r
 889         /// <summary>\r
 890         /// Deletes local files that are not found in the list of cloud files\r
 891         /// </summary>\r
 892         /// <param name="accountInfo"></param>\r
 893         /// <param name="cloudFiles"></param>\r
 894         private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles)\r
 895         {\r
 896             if (accountInfo == null)\r
 897                 throw new ArgumentNullException("accountInfo");\r
 898             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
 899                 throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
 900             if (cloudFiles == null)\r
 901                 throw new ArgumentNullException("cloudFiles");\r
 902             Contract.EndContractBlock();\r
 903 \r
 904             var deletedFiles = new List<FileSystemInfo>();\r
 905             foreach (var objectInfo in cloudFiles)\r
 906             {\r
 907                 if (Log.IsDebugEnabled)\r
 908                     Log.DebugFormat("Handle deleted [{0}]", objectInfo.Uri);\r
 909                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 910                 var item = FileAgent.GetFileAgent(accountInfo).GetFileSystemInfo(relativePath);\r
 911                 if (Log.IsDebugEnabled)\r
 912                     Log.DebugFormat("Will delete [{0}] for [{1}]", item.FullName, objectInfo.Uri);\r
 913                 if (item.Exists)\r
 914                 {\r
 915                     if ((item.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)\r
 916                     {\r
 917                         item.Attributes = item.Attributes & ~FileAttributes.ReadOnly;\r
 918 \r
 919                     }\r
 920 \r
 921 \r
 922                     Log.DebugFormat("Deleting {0}", item.FullName);\r
 923 \r
 924                     var directory = item as DirectoryInfo;\r
 925                     if (directory != null)\r
 926                         directory.Delete(true);\r
 927                     else\r
 928                         item.Delete();\r
 929                     Log.DebugFormat("Deleted [{0}] for [{1}]", item.FullName, objectInfo.Uri);\r
 930                     DateTime lastDate;\r
 931                     _lastSeen.TryRemove(item.FullName, out lastDate);\r
 932                     deletedFiles.Add(item);\r
 933                 }\r
 934                 StatusKeeper.SetFileState(item.FullName, FileStatus.Deleted, FileOverlayStatus.Deleted, "File Deleted");\r
 935             }\r
 936             Log.InfoFormat("[{0}] files were deleted", deletedFiles.Count);\r
 937             StatusNotification.NotifyForFiles(deletedFiles, String.Format("{0} files were deleted", deletedFiles.Count),\r
 938                                               TraceLevel.Info);\r
 939 \r
 940         }\r
 941 \r
 942         private void MarkSuspectedDeletes(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles)\r
 943         {\r
 944 //Only consider files that are not being modified, ie they are in the Unchanged state            \r
 945             var deleteCandidates = FileState.Queryable.Where(state =>\r
 946                                                              state.FilePath.StartsWith(accountInfo.AccountPath)\r
 947                                                              && state.FileStatus == FileStatus.Unchanged).ToList();\r
 948 \r
 949 \r
 950             //TODO: filesToDelete must take into account the Others container            \r
 951             var filesToDelete = (from deleteCandidate in deleteCandidates\r
 952                                  let localFile = FileInfoExtensions.FromPath(deleteCandidate.FilePath)\r
 953                                  let relativeFilePath = localFile.AsRelativeTo(accountInfo.AccountPath)\r
 954                                  where\r
 955                                      !cloudFiles.Any(r => r.RelativeUrlToFilePath(accountInfo.UserName) == relativeFilePath)\r
 956                                  select localFile).ToList();\r
 957 \r
 958 \r
 959             //Set the status of missing files to Conflict\r
 960             foreach (var item in filesToDelete)\r
 961             {\r
 962                 //Try to acquire a gate on the file, to take into account files that have been dequeued\r
 963                 //and are being processed\r
 964                 using (var gate = NetworkGate.Acquire(item.FullName, NetworkOperation.Deleting))\r
 965                 {\r
 966                     if (gate.Failed)\r
 967                         continue;\r
 968                     StatusKeeper.SetFileState(item.FullName, FileStatus.Conflict, FileOverlayStatus.Deleted,\r
 969                                               "Local file missing from server");\r
 970                 }\r
 971             }\r
 972             UpdateStatus(PithosStatus.HasConflicts);\r
 973             StatusNotification.NotifyConflicts(filesToDelete,\r
 974                                                String.Format(\r
 975                                                    "{0} local files are missing from Pithos, possibly because they were deleted",\r
 976                                                    filesToDelete.Count));\r
 977             StatusNotification.NotifyForFiles(filesToDelete, String.Format("{0} files were deleted", filesToDelete.Count),\r
 978                                               TraceLevel.Info);\r
 979         }\r
 980 \r
 981         private void ReportConflictForMismatch(string localFilePath)\r
 982         {\r
 983             if (String.IsNullOrWhiteSpace(localFilePath))\r
 984                 throw new ArgumentNullException("localFilePath");\r
 985             Contract.EndContractBlock();\r
 986 \r
 987             StatusKeeper.SetFileState(localFilePath, FileStatus.Conflict, FileOverlayStatus.Conflict, "File changed at the server");\r
 988             UpdateStatus(PithosStatus.HasConflicts);\r
 989             var message = String.Format("Conflict detected for file {0}", localFilePath);\r
 990             Log.Warn(message);\r
 991             StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
 992         }\r
 993 \r
 994 \r
 995 \r
 996         /// <summary>\r
 997         /// Creates a Sync action for each changed server file\r
 998         /// </summary>\r
 999         /// <param name="accountInfo"></param>\r
1000         /// <param name="changes"></param>\r
1001         /// <returns></returns>\r
1002         private IEnumerable<CloudAction> ChangesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> changes)\r
1003         {\r
1004             if (changes == null)\r
1005                 throw new ArgumentNullException();\r
1006             Contract.EndContractBlock();\r
1007             var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
1008 \r
1009             //In order to avoid multiple iterations over the files, we iterate only once\r
1010             //over the remote files\r
1011             foreach (var objectInfo in changes)\r
1012             {\r
1013                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
1014                 //If a directory object already exists, we may need to sync it\r
1015                 if (fileAgent.Exists(relativePath))\r
1016                 {\r
1017                     var localFile = fileAgent.GetFileSystemInfo(relativePath);\r
1018                     //We don't need to sync directories\r
1019                     if (objectInfo.IsDirectory && localFile is DirectoryInfo)\r
1020                         continue;\r
1021                     using (new SessionScope(FlushAction.Never))\r
1022                     {\r
1023                         var state = StatusKeeper.GetStateByFilePath(localFile.FullName);\r
1024                         _lastSeen[localFile.FullName] = DateTime.Now;\r
1025                         //Common files should be checked on a per-case basis to detect differences, which is newer\r
1026 \r
1027                         yield return new CloudAction(accountInfo, CloudActionType.MustSynch,\r
1028                                                      localFile, objectInfo, state, accountInfo.BlockSize,\r
1029                                                      accountInfo.BlockHash,"Poll Changes");\r
1030                     }\r
1031                 }\r
1032                 else\r
1033                 {\r
1034                     //Remote files should be downloaded\r
1035                     yield return new CloudDownloadAction(accountInfo, objectInfo,"Poll Changes");\r
1036                 }\r
1037             }\r
1038         }\r
1039 \r
1040         /// <summary>\r
1041         /// Creates a Local Move action for each moved server file\r
1042         /// </summary>\r
1043         /// <param name="accountInfo"></param>\r
1044         /// <param name="moves"></param>\r
1045         /// <returns></returns>\r
1046         private IEnumerable<CloudAction> MovesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> moves)\r
1047         {\r
1048             if (moves == null)\r
1049                 throw new ArgumentNullException();\r
1050             Contract.EndContractBlock();\r
1051             var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
1052 \r
1053             //In order to avoid multiple iterations over the files, we iterate only once\r
1054             //over the remote files\r
1055             foreach (var objectInfo in moves)\r
1056             {\r
1057                 var previousRelativepath = objectInfo.Previous.RelativeUrlToFilePath(accountInfo.UserName);\r
1058                 //If the previous file already exists, we can execute a Move operation\r
1059                 if (fileAgent.Exists(previousRelativepath))\r
1060                 {\r
1061                     var previousFile = fileAgent.GetFileSystemInfo(previousRelativepath);\r
1062                     using (new SessionScope(FlushAction.Never))\r
1063                     {\r
1064                         var state = StatusKeeper.GetStateByFilePath(previousFile.FullName);\r
1065                         _lastSeen[previousFile.FullName] = DateTime.Now;\r
1066 \r
1067                         //For each moved object we need to move both the local file and update                                                \r
1068                         yield return new CloudAction(accountInfo, CloudActionType.RenameLocal,\r
1069                                                      previousFile, objectInfo, state, accountInfo.BlockSize,\r
1070                                                      accountInfo.BlockHash,"Poll Moves");\r
1071                         //For modified files, we need to download the changes as well\r
1072                         if (objectInfo.X_Object_Hash != objectInfo.PreviousHash)\r
1073                             yield return new CloudDownloadAction(accountInfo,objectInfo, "Poll Moves");\r
1074                     }\r
1075                 }\r
1076                 //If the previous file does not exist, we need to download it in the new location\r
1077                 else\r
1078                 {\r
1079                     //Remote files should be downloaded\r
1080                     yield return new CloudDownloadAction(accountInfo, objectInfo, "Poll Moves");\r
1081                 }\r
1082             }\r
1083         }\r
1084 \r
1085 \r
1086         /// <summary>\r
1087         /// Creates a download action for each new server file\r
1088         /// </summary>\r
1089         /// <param name="accountInfo"></param>\r
1090         /// <param name="creates"></param>\r
1091         /// <returns></returns>\r
1092         private IEnumerable<CloudAction> CreatesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> creates)\r
1093         {\r
1094             if (creates == null)\r
1095                 throw new ArgumentNullException();\r
1096             Contract.EndContractBlock();\r
1097             var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
1098 \r
1099             //In order to avoid multiple iterations over the files, we iterate only once\r
1100             //over the remote files\r
1101             foreach (var objectInfo in creates)\r
1102             {\r
1103                 if (Log.IsDebugEnabled)\r
1104                     Log.DebugFormat("[NEW INFO] {0}",objectInfo.Uri);\r
1105 \r
1106                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
1107 \r
1108                 //If the object already exists, we should check before uploading or downloading\r
1109                 if (fileAgent.Exists(relativePath))\r
1110                 {\r
1111                     var localFile= fileAgent.GetFileSystemInfo(relativePath);\r
1112                     var state = StatusKeeper.GetStateByFilePath(localFile.WithProperCapitalization().FullName);\r
1113                     yield return new CloudAction(accountInfo, CloudActionType.MustSynch,\r
1114                                                      localFile, objectInfo, state, accountInfo.BlockSize,\r
1115                                                      accountInfo.BlockHash,"Poll Creates");                    \r
1116                 }\r
1117                 else\r
1118                 {\r
1119                     //Remote files should be downloaded\r
1120                     yield return new CloudDownloadAction(accountInfo, objectInfo,"Poll Creates");\r
1121                 }\r
1122 \r
1123             }\r
1124         }\r
1125 \r
1126         /// <summary>\r
1127         /// Notify the UI to update the visual status\r
1128         /// </summary>\r
1129         /// <param name="status"></param>\r
1130         private void UpdateStatus(PithosStatus status)\r
1131         {\r
1132             try\r
1133             {\r
1134                 StatusNotification.SetPithosStatus(status);\r
1135                 //StatusNotification.Notify(new Notification());\r
1136             }\r
1137             catch (Exception exc)\r
1138             {\r
1139                 //Failure is not critical, just log it\r
1140                 Log.Warn("Error while updating status", exc);\r
1141             }\r
1142         }\r
1143 \r
1144         private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)\r
1145         {\r
1146             var containerPaths = from container in containers\r
1147                                  let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)\r
1148                                  where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)\r
1149                                  select containerPath;\r
1150 \r
1151             foreach (var path in containerPaths)\r
1152             {\r
1153                 Directory.CreateDirectory(path);\r
1154             }\r
1155         }\r
1156 \r
1157         public void AddAccount(AccountInfo accountInfo)\r
1158         {\r
1159             //Avoid adding a duplicate accountInfo\r
1160             _accounts.TryAdd(accountInfo.AccountKey, accountInfo);\r
1161         }\r
1162 \r
1163         public void RemoveAccount(AccountInfo accountInfo)\r
1164         {\r
1165             AccountInfo account;\r
1166             _accounts.TryRemove(accountInfo.AccountKey, out account);\r
1167 \r
1168             SnapshotDifferencer differencer;\r
1169             _differencer.Differencers.TryRemove(accountInfo.AccountKey, out differencer);\r
1170         }\r
1171 \r
1172         public void SetSelectivePaths(AccountInfo accountInfo,Uri[] added, Uri[] removed)\r
1173         {\r
1174             AbortRemovedPaths(accountInfo,removed);\r
1175             //DownloadNewPaths(accountInfo,added);\r
1176         }\r
1177 \r
1178 /*\r
1179         private void DownloadNewPaths(AccountInfo accountInfo, Uri[] added)\r
1180         {\r
1181             var client = new CloudFilesClient(accountInfo);\r
1182             foreach (var folderUri in added)\r
1183             {\r
1184                 try\r
1185                 {\r
1186 \r
1187                     string account;\r
1188                     string container;\r
1189                     var segmentsCount = folderUri.Segments.Length;\r
1190                     //Is this an account URL?\r
1191                     if (segmentsCount < 3)\r
1192                         continue;\r
1193                     //Is this a container or  folder URL?\r
1194                     if (segmentsCount == 3)\r
1195                     {\r
1196                         account = folderUri.Segments[1].TrimEnd('/');\r
1197                         container = folderUri.Segments[2].TrimEnd('/');\r
1198                     }\r
1199                     else\r
1200                     {\r
1201                         account = folderUri.Segments[2].TrimEnd('/');\r
1202                         container = folderUri.Segments[3].TrimEnd('/');\r
1203                     }\r
1204                     IList<ObjectInfo> items;\r
1205                     if (segmentsCount > 3)\r
1206                     {\r
1207                         //List folder\r
1208                         var folder = String.Join("", folderUri.Segments.Splice(4));\r
1209                         items = client.ListObjects(account, container, folder);\r
1210                     }\r
1211                     else\r
1212                     {\r
1213                         //List container\r
1214                         items = client.ListObjects(account, container);\r
1215                     }\r
1216                     var actions = CreatesToActions(accountInfo, items);\r
1217                     foreach (var action in actions)\r
1218                     {\r
1219                         NetworkAgent.Post(action);\r
1220                     }\r
1221                 }\r
1222                 catch (Exception exc)\r
1223                 {\r
1224                     Log.WarnFormat("Listing of new selective path [{0}] failed with \r\n{1}", folderUri, exc);\r
1225                 }\r
1226             }\r
1227 \r
1228             //Need to get a listing of each of the URLs, then post them to the NetworkAgent\r
1229             //CreatesToActions(accountInfo,)\r
1230 \r
1231 /*            NetworkAgent.Post();#1#\r
1232         }\r
1233 */\r
1234 \r
1235         private void AbortRemovedPaths(AccountInfo accountInfo, Uri[] removed)\r
1236         {\r
1237             /*this.NetworkAgent.*/\r
1238         }\r
1239     }\r
1240 }\r