code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/PollAgent.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="PollAgent.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 \r
  43 using System.Collections.Concurrent;\r
  44 using System.ComponentModel.Composition;\r
  45 using System.Diagnostics;\r
  46 using System.Diagnostics.Contracts;\r
  47 using System.IO;\r
  48 using System.Linq.Expressions;\r
  49 using System.Reflection;\r
  50 using System.Threading;\r
  51 using System.Threading.Tasks;\r
  52 using Castle.ActiveRecord;\r
  53 using Pithos.Interfaces;\r
  54 using Pithos.Network;\r
  55 using log4net;\r
  56 \r
  57 namespace Pithos.Core.Agents\r
  58 {\r
  59     using System;\r
  60     using System.Collections.Generic;\r
  61     using System.Linq;\r
  62 \r
  63     [DebuggerDisplay("{FilePath} C:{C} L:{L} S:{S}")]\r
  64     public class StateTuple\r
  65     {\r
  66         public string FilePath { get; private set; }\r
  67 \r
  68         public string L\r
  69         {\r
  70             get { return FileState==null?null:FileState.Checksum; }\r
  71         }\r
  72 \r
  73         public string C { get; set; }\r
  74 \r
  75         public string S\r
  76         {\r
  77             get { return ObjectInfo== null ? null : ObjectInfo.Hash; }\r
  78         }\r
  79 \r
  80         private FileSystemInfo _fileInfo;\r
  81         public FileSystemInfo FileInfo\r
  82         {\r
  83             get { return _fileInfo; }\r
  84             set\r
  85             {\r
  86                 _fileInfo = value;\r
  87                 FilePath = value.FullName;\r
  88             }\r
  89         }\r
  90 \r
  91         public FileState FileState { get; set; }\r
  92         public ObjectInfo ObjectInfo{ get; set; }\r
  93 \r
  94         public StateTuple() { }\r
  95 \r
  96         public StateTuple(FileSystemInfo info)\r
  97         {\r
  98             FileInfo = info;\r
  99         }\r
 100 \r
 101 \r
 102     }\r
 103 \r
 104 \r
 105     /// <summary>\r
 106     /// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all\r
 107     /// objects and compares it with a previously cached version to detect differences. \r
 108     /// New files are downloaded, missing files are deleted from the local file system and common files are compared\r
 109     /// to determine the appropriate action\r
 110     /// </summary>\r
 111     [Export]\r
 112     public class PollAgent\r
 113     {\r
 114         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
 115 \r
 116         [System.ComponentModel.Composition.Import]\r
 117         public IStatusKeeper StatusKeeper { get; set; }\r
 118 \r
 119         [System.ComponentModel.Composition.Import]\r
 120         public IPithosSettings Settings { get; set; }\r
 121 \r
 122         [System.ComponentModel.Composition.Import]\r
 123         public NetworkAgent NetworkAgent { get; set; }\r
 124 \r
 125         [System.ComponentModel.Composition.Import]\r
 126         public Selectives Selectives { get; set; }\r
 127 \r
 128         public IStatusNotification StatusNotification { get; set; }\r
 129 \r
 130         private CancellationTokenSource _currentOperationCancellation = new CancellationTokenSource();\r
 131 \r
 132         public void CancelCurrentOperation()\r
 133         {\r
 134             //What does it mean to cancel the current upload/download?\r
 135             //Obviously, the current operation will be cancelled by throwing\r
 136             //a cancellation exception.\r
 137             //\r
 138             //The default behavior is to retry any operations that throw.\r
 139             //Obviously this is not what we want in this situation.\r
 140             //The cancelled operation should NOT bea retried. \r
 141             //\r
 142             //This can be done by catching the cancellation exception\r
 143             //and avoiding the retry.\r
 144             //\r
 145 \r
 146             //Have to reset the cancellation source - it is not possible to reset the source\r
 147             //Have to prevent a case where an operation requests a token from the old source\r
 148             var oldSource = Interlocked.Exchange(ref _currentOperationCancellation, new CancellationTokenSource());\r
 149             oldSource.Cancel();\r
 150 \r
 151         }\r
 152 \r
 153         public bool Pause\r
 154         {\r
 155             get {\r
 156                 return _pause;\r
 157             }\r
 158             set {\r
 159                 _pause = value;                \r
 160                 if (!_pause)\r
 161                     _unPauseEvent.Set();\r
 162                 else\r
 163                 {\r
 164                     _unPauseEvent.Reset();\r
 165                 }\r
 166             }\r
 167         }\r
 168 \r
 169         private bool _firstPoll = true;\r
 170 \r
 171         //The Sync Event signals a manual synchronisation\r
 172         private readonly AsyncManualResetEvent _syncEvent = new AsyncManualResetEvent();\r
 173 \r
 174         private readonly AsyncManualResetEvent _unPauseEvent = new AsyncManualResetEvent(true);\r
 175 \r
 176         private readonly ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>();\r
 177         private readonly ConcurrentDictionary<Uri, AccountInfo> _accounts = new ConcurrentDictionary<Uri,AccountInfo>();\r
 178 \r
 179 \r
 180         /// <summary>\r
 181         /// Start a manual synchronization\r
 182         /// </summary>\r
 183         public void SynchNow()\r
 184         {            \r
 185             _syncEvent.Set();\r
 186         }\r
 187 \r
 188 \r
 189         /// <summary>\r
 190         /// Remote files are polled periodically. Any changes are processed\r
 191         /// </summary>\r
 192         /// <param name="since"></param>\r
 193         /// <returns></returns>\r
 194         public async Task PollRemoteFiles(DateTime? since = null)\r
 195         {\r
 196             if (Log.IsDebugEnabled)\r
 197                 Log.DebugFormat("Polling changes after [{0}]",since);\r
 198 \r
 199             Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
 200 \r
 201             //GC.Collect();\r
 202 \r
 203             using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
 204             {\r
 205                 //If this poll fails, we will retry with the same since value\r
 206                 var nextSince = since;\r
 207                 try\r
 208                 {\r
 209                     await _unPauseEvent.WaitAsync();\r
 210                     UpdateStatus(PithosStatus.PollSyncing);\r
 211 \r
 212                     var tasks = from accountInfo in _accounts.Values\r
 213                                 select ProcessAccountFiles(accountInfo, since);\r
 214 \r
 215                     var nextTimes=await TaskEx.WhenAll(tasks.ToList());\r
 216 \r
 217                     _firstPoll = false;\r
 218                     //Reschedule the poll with the current timestamp as a "since" value\r
 219 \r
 220                     if (nextTimes.Length>0)\r
 221                         nextSince = nextTimes.Min();\r
 222                     if (Log.IsDebugEnabled)\r
 223                         Log.DebugFormat("Next Poll at [{0}]",nextSince);\r
 224                 }\r
 225                 catch (Exception ex)\r
 226                 {\r
 227                     Log.ErrorFormat("Error while processing accounts\r\n{0}", ex);\r
 228                     //In case of failure retry with the same "since" value\r
 229                 }\r
 230 \r
 231                 UpdateStatus(PithosStatus.PollComplete);\r
 232                 //The multiple try blocks are required because we can't have an await call\r
 233                 //inside a finally block\r
 234                 //TODO: Find a more elegant solution for reschedulling in the event of an exception\r
 235                 try\r
 236                 {\r
 237                     //Wait for the polling interval to pass or the Sync event to be signalled\r
 238                     nextSince = await WaitForScheduledOrManualPoll(nextSince);\r
 239                 }\r
 240                 finally\r
 241                 {\r
 242                     //Ensure polling is scheduled even in case of error\r
 243                     TaskEx.Run(() => PollRemoteFiles(nextSince));                        \r
 244                 }\r
 245             }\r
 246         }\r
 247 \r
 248         /// <summary>\r
 249         /// Wait for the polling period to expire or a manual sync request\r
 250         /// </summary>\r
 251         /// <param name="since"></param>\r
 252         /// <returns></returns>\r
 253         private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
 254         {\r
 255             var sync = _syncEvent.WaitAsync();\r
 256             var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval), NetworkAgent.CancellationToken);\r
 257             \r
 258             var signaledTask = await TaskEx.WhenAny(sync, wait);\r
 259             \r
 260             //Pausing takes precedence over manual sync or awaiting\r
 261             _unPauseEvent.Wait();\r
 262             \r
 263             //Wait for network processing to finish before polling\r
 264             var pauseTask=NetworkAgent.ProceedEvent.WaitAsync();\r
 265             await TaskEx.WhenAll(signaledTask, pauseTask);\r
 266 \r
 267             //If polling is signalled by SynchNow, ignore the since tag\r
 268             if (sync.IsCompleted)\r
 269             {\r
 270                 //TODO: Must convert to AutoReset\r
 271                 _syncEvent.Reset();\r
 272                 return null;\r
 273             }\r
 274             return since;\r
 275         }\r
 276 \r
 277         public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, DateTime? since = null)\r
 278         {\r
 279             if (accountInfo == null)\r
 280                 throw new ArgumentNullException("accountInfo");\r
 281             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
 282                 throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
 283             Contract.EndContractBlock();\r
 284 \r
 285 \r
 286             using (ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))\r
 287             {\r
 288 \r
 289                 await NetworkAgent.GetDeleteAwaiter();\r
 290 \r
 291                 Log.Info("Scheduled");\r
 292                 var client = new CloudFilesClient(accountInfo);\r
 293 \r
 294                 //We don't need to check the trash container\r
 295                 var containers = client.ListContainers(accountInfo.UserName)\r
 296                     .Where(c=>c.Name!="trash")\r
 297                     .ToList();\r
 298 \r
 299 \r
 300                 CreateContainerFolders(accountInfo, containers);\r
 301 \r
 302                 //The nextSince time fallback time is the same as the current.\r
 303                 //If polling succeeds, the next Since time will be the smallest of the maximum modification times\r
 304                 //of the shared and account objects\r
 305                 var nextSince = since;\r
 306 \r
 307                 try\r
 308                 {\r
 309                     //Wait for any deletions to finish\r
 310                     await NetworkAgent.GetDeleteAwaiter();\r
 311                     //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
 312                     //than delete a file that was created while we were executing the poll                    \r
 313 \r
 314                     //Get the list of server objects changed since the last check\r
 315                     //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
 316                     var listObjects = (from container in containers\r
 317                                        select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
 318                                              client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
 319 \r
 320                     var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
 321                         client.ListSharedObjects(since), "shared");\r
 322                     listObjects.Add(listShared);\r
 323                     var listTasks = await Task.Factory.WhenAll(listObjects.ToArray());\r
 324 \r
 325                     using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
 326                     {\r
 327                         var dict = listTasks.ToDictionary(t => t.AsyncState);\r
 328 \r
 329                         //Get all non-trash objects. Remember, the container name is stored in AsyncState\r
 330                         var remoteObjects = (from objectList in listTasks\r
 331                                             where (string)objectList.AsyncState != "trash"\r
 332                                             from obj in objectList.Result\r
 333                                             orderby obj.Bytes ascending \r
 334                                             select obj).ToList();\r
 335                         \r
 336                         //Get the latest remote object modification date, only if it is after\r
 337                         //the original since date\r
 338                         nextSince = GetLatestDateAfter(nextSince, remoteObjects);\r
 339 \r
 340                         var sharedObjects = dict["shared"].Result;\r
 341                         nextSince = GetLatestDateBefore(nextSince, sharedObjects);\r
 342 \r
 343                         //DON'T process trashed files\r
 344                         //If some files are deleted and added again to a folder, they will be deleted\r
 345                         //even though they are new.\r
 346                         //We would have to check file dates and hashes to ensure that a trashed file\r
 347                         //can be deleted safely from the local hard drive.\r
 348                         /*\r
 349                         //Items with the same name, hash may be both in the container and the trash\r
 350                         //Don't delete items that exist in the container\r
 351                         var realTrash = from trash in trashObjects\r
 352                                         where\r
 353                                             !remoteObjects.Any(\r
 354                                                 info => info.Name == trash.Name && info.Hash == trash.Hash)\r
 355                                         select trash;\r
 356                         ProcessTrashedFiles(accountInfo, realTrash);\r
 357 */\r
 358 \r
 359                         var cleanRemotes = (from info in remoteObjects.Union(sharedObjects)\r
 360                                             let name = info.Name??""\r
 361                                             where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&\r
 362                                                   !name.StartsWith(FolderConstants.CacheFolder + "/",\r
 363                                                                    StringComparison.InvariantCultureIgnoreCase)\r
 364                                             select info).ToList();\r
 365 \r
 366                         if (_firstPoll)\r
 367                             StatusKeeper.CleanupOrphanStates();\r
 368                         StatusKeeper.CleanupStaleStates(accountInfo, cleanRemotes);\r
 369                         \r
 370                         //var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
 371 \r
 372                         var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
 373 \r
 374 \r
 375                         //Get the local files here                        \r
 376                         var agent = AgentLocator<FileAgent>.Get(accountInfo.AccountPath);\r
 377 \r
 378                         var files = LoadLocalFileTuples(accountInfo);\r
 379 \r
 380                         var states = FileState.Queryable.ToList();\r
 381 \r
 382                         var infos = (from remote in cleanRemotes\r
 383                                     let path = remote.RelativeUrlToFilePath(accountInfo.UserName)\r
 384                                     let info=agent.GetFileSystemInfo(path)\r
 385                                     select Tuple.Create(info.FullName,remote))\r
 386                                     .ToList();\r
 387 \r
 388                         var token = _currentOperationCancellation.Token;\r
 389 \r
 390                         var tuples = MergeSources(infos, files, states).ToList();\r
 391 \r
 392                         \r
 393                         foreach (var tuple in tuples)\r
 394                         {\r
 395                             await _unPauseEvent.WaitAsync();\r
 396                             \r
 397                             SyncSingleItem(accountInfo, tuple, agent, token);\r
 398                         }\r
 399 \r
 400 \r
 401                         //On the first run\r
 402 /*\r
 403                         if (_firstPoll)\r
 404                         {\r
 405                             MarkSuspectedDeletes(accountInfo, cleanRemotes);\r
 406                         }\r
 407 */\r
 408 \r
 409 \r
 410                         Log.Info("[LISTENER] End Processing");\r
 411                     }\r
 412                 }\r
 413                 catch (Exception ex)\r
 414                 {\r
 415                     Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);\r
 416                     return nextSince;\r
 417                 }\r
 418 \r
 419                 Log.Info("[LISTENER] Finished");\r
 420                 return nextSince;\r
 421             }\r
 422         }\r
 423 \r
 424         private static List<Tuple<FileSystemInfo, string>> LoadLocalFileTuples(AccountInfo accountInfo)\r
 425         {\r
 426             using (ThreadContext.Stacks["Account Files Hashing"].Push(accountInfo.UserName))\r
 427             {\r
 428 \r
 429                 var localInfos = AgentLocator<FileAgent>.Get(accountInfo.AccountPath).EnumerateFileSystemInfos();\r
 430                 //Use the queue to retry locked file hashing\r
 431                 var fileQueue = new Queue<FileSystemInfo>(localInfos);\r
 432 \r
 433                 var results = new List<Tuple<FileSystemInfo, string>>();\r
 434 \r
 435                 while (fileQueue.Count > 0)\r
 436                 {\r
 437                     var file = fileQueue.Dequeue();\r
 438                     using (ThreadContext.Stacks["File"].Push(file.FullName))\r
 439                     {\r
 440                         try\r
 441                         {\r
 442                             var hash = (file is DirectoryInfo)\r
 443                                            ? "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"\r
 444                                            : Signature.CalculateTreeHash(file, accountInfo.BlockSize,\r
 445                                                                          accountInfo.BlockHash)\r
 446                                                  .\r
 447                                                  TopHash.ToHashString();\r
 448                             results.Add(Tuple.Create(file, hash));\r
 449                         }\r
 450                         catch (IOException exc)\r
 451                         {\r
 452                             Log.WarnFormat("[HASH] File in use, will retry [{0}]", exc);\r
 453                             fileQueue.Enqueue(file);\r
 454                         }\r
 455                     }\r
 456                 }\r
 457 \r
 458                 return results;\r
 459             }\r
 460         }\r
 461 \r
 462         private void SyncSingleItem(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, CancellationToken token)\r
 463         {\r
 464             Log.DebugFormat("Sync [{0}] C:[{1}] L:[{2}] S:[{3}]",tuple.FilePath,tuple.C,tuple.L,tuple.S);\r
 465 \r
 466             var localFilePath = tuple.FilePath;\r
 467             //Don't use the tuple info, it may have been deleted\r
 468             var localInfo = FileInfoExtensions.FromPath(localFilePath);\r
 469 \r
 470             // Local file unchanged? If both C and L are null, make sure it's because \r
 471             //both the file is missing and the state checksum is not missing\r
 472             if (tuple.C == tuple.L && (localInfo.Exists || tuple.FileState==null))\r
 473             {\r
 474                 //No local changes\r
 475                 //Server unchanged?\r
 476                 if (tuple.S == tuple.L)\r
 477                 {\r
 478                     // No server changes\r
 479                     ;\r
 480                 }\r
 481                 else\r
 482                 {\r
 483                     //Different from server\r
 484                     if (Selectives.IsSelected(accountInfo, localFilePath))\r
 485                     {\r
 486                         //Does the server file exist?\r
 487                         if (tuple.S == null)\r
 488                         {\r
 489                             //Server file doesn't exist\r
 490                             //deleteObjectFromLocal()\r
 491                             StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
 492                                                       FileOverlayStatus.Deleted, "");\r
 493                             agent.Delete(localFilePath);\r
 494                             //updateRecord(Remove C, L)\r
 495                             StatusKeeper.ClearFileStatus(localFilePath);\r
 496                         }\r
 497                         else\r
 498                         {\r
 499                             //Server file exists\r
 500                             //downloadServerObject() // Result: L = S\r
 501                             StatusKeeper.SetFileState(localFilePath, FileStatus.Modified,\r
 502                                                       FileOverlayStatus.Modified, "");\r
 503                             NetworkAgent.Downloader.DownloadCloudFile(accountInfo,\r
 504                                                                             tuple.ObjectInfo,\r
 505                                                                             localFilePath, token).Wait(token);\r
 506                             //updateRecord( L = S )\r
 507                             StatusKeeper.UpdateFileChecksum(localFilePath, tuple.FileState==null?"":tuple.FileState.ShortHash,\r
 508                                                             tuple.ObjectInfo.Hash);\r
 509 \r
 510                             StatusKeeper.SetFileState(localFilePath, FileStatus.Unchanged,\r
 511                                                       FileOverlayStatus.Normal, "");\r
 512                         }\r
 513                     }\r
 514                 }\r
 515             }\r
 516             else\r
 517             {\r
 518                 //Local changes found\r
 519 \r
 520                 //Server unchanged?\r
 521                 if (tuple.S == tuple.L)\r
 522                 {\r
 523                     //The FileAgent selective sync checks for new root folder files\r
 524                     if (!agent.Ignore(localFilePath))\r
 525                     {\r
 526                         if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
 527                         {\r
 528                             //deleteObjectFromServer()\r
 529                             DeleteCloudFile(accountInfo, tuple);\r
 530                             //updateRecord( Remove L, S)                  \r
 531                         }\r
 532                         else\r
 533                         {\r
 534                             //uploadLocalObject() // Result: S = C, L = S                        \r
 535                             var isUnselected = agent.IsUnselectedRootFolder(tuple.FilePath);\r
 536 \r
 537                             //Debug.Assert(tuple.FileState !=null);\r
 538                             var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
 539                                                                accountInfo.BlockSize, accountInfo.BlockHash,\r
 540                                                                "Poll", isUnselected);\r
 541                             NetworkAgent.Uploader.UploadCloudFile(action, token).Wait(token);\r
 542 \r
 543 \r
 544                             //updateRecord( S = C )\r
 545                             StatusKeeper.SetFileState(localFilePath, FileStatus.Unchanged,\r
 546                                                       FileOverlayStatus.Normal, "");\r
 547                             if (isUnselected)\r
 548                             {\r
 549                                 ProcessChildren(accountInfo, tuple, agent, token);\r
 550                             }\r
 551                         }\r
 552                     }\r
 553                 }\r
 554                 else\r
 555                 {\r
 556                     if (Selectives.IsSelected(accountInfo, localFilePath))\r
 557                     {\r
 558                         if (tuple.C == tuple.S)\r
 559                         {\r
 560                             // (Identical Changes) Result: L = S\r
 561                             //doNothing()\r
 562                             StatusKeeper.UpdateFileChecksum(localFilePath, tuple.FileState == null ? "" : tuple.FileState.ShortHash,\r
 563                                                             tuple.ObjectInfo.Hash);\r
 564                             StatusKeeper.SetFileState(localFilePath, FileStatus.Unchanged,\r
 565                                                       FileOverlayStatus.Normal, "");\r
 566                         }\r
 567                         else\r
 568                         {\r
 569                             if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null )\r
 570                             {\r
 571                                 //deleteObjectFromServer()\r
 572                                 DeleteCloudFile(accountInfo, tuple);\r
 573                                 //updateRecord(Remove L, S)                  \r
 574                             }\r
 575                             else\r
 576                             {\r
 577                                 ReportConflictForMismatch(localFilePath);\r
 578                                 //identifyAsConflict() // Manual action required\r
 579                             }\r
 580                         }\r
 581                     }\r
 582                 }\r
 583             }\r
 584         }\r
 585 \r
 586         private void DeleteCloudFile(AccountInfo accountInfo, StateTuple tuple)\r
 587         {\r
 588             StatusKeeper.SetFileState(tuple.FilePath, FileStatus.Deleted,\r
 589                                       FileOverlayStatus.Deleted, "");\r
 590             NetworkAgent.DeleteAgent.DeleteCloudFile(accountInfo, tuple.ObjectInfo);\r
 591             StatusKeeper.ClearFileStatus(tuple.FilePath);\r
 592         }\r
 593 \r
 594         private void ProcessChildren(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, CancellationToken token)\r
 595         {\r
 596 \r
 597             var dirInfo = tuple.FileInfo as DirectoryInfo;\r
 598             var folderTuples = from folder in dirInfo.EnumerateDirectories("*", SearchOption.AllDirectories)\r
 599                                select new StateTuple(folder);\r
 600             var fileTuples = from file in dirInfo.EnumerateFiles("*", SearchOption.AllDirectories)\r
 601                              select new StateTuple(file);\r
 602             \r
 603             //Process folders first, to ensure folders appear on the sever as soon as possible\r
 604             folderTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, token));\r
 605             \r
 606             fileTuples.ApplyAction(t => SyncSingleItem(accountInfo, t, agent, token));\r
 607         }\r
 608 \r
 609         private static IEnumerable<StateTuple> MergeSources(\r
 610             IEnumerable<Tuple<string, ObjectInfo>> infos, \r
 611             IEnumerable<Tuple<FileSystemInfo, string>> files, \r
 612             IEnumerable<FileState> states)\r
 613         {\r
 614             var dct = new Dictionary<string, StateTuple>();\r
 615             foreach (var file in files)\r
 616             {\r
 617                 var fsInfo = file.Item1;\r
 618                 var fileHash = file.Item2;\r
 619                 dct[fsInfo.FullName] = new StateTuple {FileInfo = fsInfo, C = fileHash};\r
 620             }\r
 621             foreach (var state in states)\r
 622             {\r
 623                 StateTuple hashTuple;\r
 624                 if (dct.TryGetValue(state.FilePath, out hashTuple))\r
 625                 {\r
 626                     hashTuple.FileState = state;\r
 627                 }\r
 628                 else\r
 629                 {\r
 630                     var fsInfo = FileInfoExtensions.FromPath(state.FilePath);\r
 631                     dct[state.FilePath] = new StateTuple {FileInfo = fsInfo, FileState = state};\r
 632                 }\r
 633             }\r
 634             foreach (var info in infos)\r
 635             {\r
 636                 StateTuple hashTuple;\r
 637                 var filePath = info.Item1;\r
 638                 var objectInfo = info.Item2;\r
 639                 if (dct.TryGetValue(filePath, out hashTuple))\r
 640                 {\r
 641                     hashTuple.ObjectInfo = objectInfo;\r
 642                 }\r
 643                 else\r
 644                 {\r
 645                     var fsInfo = FileInfoExtensions.FromPath(filePath);\r
 646                     dct[filePath] = new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo};\r
 647                 }\r
 648             }\r
 649             return dct.Values;\r
 650         }\r
 651 \r
 652         /// <summary>\r
 653         /// Returns the latest LastModified date from the list of objects, but only if it is before\r
 654         /// than the threshold value\r
 655         /// </summary>\r
 656         /// <param name="threshold"></param>\r
 657         /// <param name="cloudObjects"></param>\r
 658         /// <returns></returns>\r
 659         private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
 660         {\r
 661             DateTime? maxDate = null;\r
 662             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
 663                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
 664             if (maxDate == null || maxDate == DateTime.MinValue)\r
 665                 return threshold;\r
 666             if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate)\r
 667                 return maxDate;\r
 668             return threshold;\r
 669         }\r
 670 \r
 671         /// <summary>\r
 672         /// Returns the latest LastModified date from the list of objects, but only if it is after\r
 673         /// the threshold value\r
 674         /// </summary>\r
 675         /// <param name="threshold"></param>\r
 676         /// <param name="cloudObjects"></param>\r
 677         /// <returns></returns>\r
 678         private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
 679         {\r
 680             DateTime? maxDate = null;\r
 681             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
 682                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
 683             if (maxDate == null || maxDate == DateTime.MinValue)\r
 684                 return threshold;\r
 685             if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate)\r
 686                 return maxDate;\r
 687             return threshold;\r
 688         }\r
 689 \r
 690         //readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
 691         private Dictionary<Uri, List<Uri>> _selectiveUris = new Dictionary<Uri, List<Uri>>();\r
 692         private bool _pause;\r
 693 \r
 694         /// <summary>\r
 695         /// Deletes local files that are not found in the list of cloud files\r
 696         /// </summary>\r
 697         /// <param name="accountInfo"></param>\r
 698         /// <param name="cloudFiles"></param>\r
 699         private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles)\r
 700         {\r
 701             if (accountInfo == null)\r
 702                 throw new ArgumentNullException("accountInfo");\r
 703             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
 704                 throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
 705             if (cloudFiles == null)\r
 706                 throw new ArgumentNullException("cloudFiles");\r
 707             Contract.EndContractBlock();\r
 708 \r
 709             var deletedFiles = new List<FileSystemInfo>();\r
 710             foreach (var objectInfo in cloudFiles)\r
 711             {\r
 712                 if (Log.IsDebugEnabled)\r
 713                     Log.DebugFormat("Handle deleted [{0}]", objectInfo.Uri);\r
 714                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 715                 var item = FileAgent.GetFileAgent(accountInfo).GetFileSystemInfo(relativePath);\r
 716                 if (Log.IsDebugEnabled)\r
 717                     Log.DebugFormat("Will delete [{0}] for [{1}]", item.FullName, objectInfo.Uri);\r
 718                 if (item.Exists)\r
 719                 {\r
 720                     if ((item.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)\r
 721                     {\r
 722                         item.Attributes = item.Attributes & ~FileAttributes.ReadOnly;\r
 723 \r
 724                     }\r
 725 \r
 726 \r
 727                     Log.DebugFormat("Deleting {0}", item.FullName);\r
 728 \r
 729                     var directory = item as DirectoryInfo;\r
 730                     if (directory != null)\r
 731                         directory.Delete(true);\r
 732                     else\r
 733                         item.Delete();\r
 734                     Log.DebugFormat("Deleted [{0}] for [{1}]", item.FullName, objectInfo.Uri);\r
 735                     DateTime lastDate;\r
 736                     _lastSeen.TryRemove(item.FullName, out lastDate);\r
 737                     deletedFiles.Add(item);\r
 738                 }\r
 739                 StatusKeeper.SetFileState(item.FullName, FileStatus.Deleted, FileOverlayStatus.Deleted, "File Deleted");\r
 740             }\r
 741             Log.InfoFormat("[{0}] files were deleted", deletedFiles.Count);\r
 742             StatusNotification.NotifyForFiles(deletedFiles, String.Format("{0} files were deleted", deletedFiles.Count),\r
 743                                               TraceLevel.Info);\r
 744 \r
 745         }\r
 746 \r
 747         private void MarkSuspectedDeletes(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles)\r
 748         {\r
 749 //Only consider files that are not being modified, ie they are in the Unchanged state            \r
 750             var deleteCandidates = FileState.Queryable.Where(state =>\r
 751                                                              state.FilePath.StartsWith(accountInfo.AccountPath)\r
 752                                                              && state.FileStatus == FileStatus.Unchanged).ToList();\r
 753 \r
 754 \r
 755             //TODO: filesToDelete must take into account the Others container            \r
 756             var filesToDelete = (from deleteCandidate in deleteCandidates\r
 757                                  let localFile = FileInfoExtensions.FromPath(deleteCandidate.FilePath)\r
 758                                  let relativeFilePath = localFile.AsRelativeTo(accountInfo.AccountPath)\r
 759                                  where\r
 760                                      !cloudFiles.Any(r => r.RelativeUrlToFilePath(accountInfo.UserName) == relativeFilePath)\r
 761                                  select localFile).ToList();\r
 762 \r
 763 \r
 764             //Set the status of missing files to Conflict\r
 765             foreach (var item in filesToDelete)\r
 766             {\r
 767                 //Try to acquire a gate on the file, to take into account files that have been dequeued\r
 768                 //and are being processed\r
 769                 using (var gate = NetworkGate.Acquire(item.FullName, NetworkOperation.Deleting))\r
 770                 {\r
 771                     if (gate.Failed)\r
 772                         continue;\r
 773                     StatusKeeper.SetFileState(item.FullName, FileStatus.Conflict, FileOverlayStatus.Deleted,\r
 774                                               "Local file missing from server");\r
 775                 }\r
 776             }\r
 777             UpdateStatus(PithosStatus.HasConflicts);\r
 778             StatusNotification.NotifyConflicts(filesToDelete,\r
 779                                                String.Format(\r
 780                                                    "{0} local files are missing from Pithos, possibly because they were deleted",\r
 781                                                    filesToDelete.Count));\r
 782             StatusNotification.NotifyForFiles(filesToDelete, String.Format("{0} files were deleted", filesToDelete.Count),\r
 783                                               TraceLevel.Info);\r
 784         }\r
 785 \r
 786         private void ReportConflictForMismatch(string localFilePath)\r
 787         {\r
 788             if (String.IsNullOrWhiteSpace(localFilePath))\r
 789                 throw new ArgumentNullException("localFilePath");\r
 790             Contract.EndContractBlock();\r
 791 \r
 792             StatusKeeper.SetFileState(localFilePath, FileStatus.Conflict, FileOverlayStatus.Conflict, "File changed at the server");\r
 793             UpdateStatus(PithosStatus.HasConflicts);\r
 794             var message = String.Format("Conflict detected for file {0}", localFilePath);\r
 795             Log.Warn(message);\r
 796             StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
 797         }\r
 798 \r
 799 \r
 800 \r
 801         /// <summary>\r
 802         /// Creates a Sync action for each changed server file\r
 803         /// </summary>\r
 804         /// <param name="accountInfo"></param>\r
 805         /// <param name="changes"></param>\r
 806         /// <returns></returns>\r
 807         private IEnumerable<CloudAction> ChangesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> changes)\r
 808         {\r
 809             if (changes == null)\r
 810                 throw new ArgumentNullException();\r
 811             Contract.EndContractBlock();\r
 812             var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
 813 \r
 814             //In order to avoid multiple iterations over the files, we iterate only once\r
 815             //over the remote files\r
 816             foreach (var objectInfo in changes)\r
 817             {\r
 818                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 819                 //If a directory object already exists, we may need to sync it\r
 820                 if (fileAgent.Exists(relativePath))\r
 821                 {\r
 822                     var localFile = fileAgent.GetFileSystemInfo(relativePath);\r
 823                     //We don't need to sync directories\r
 824                     if (objectInfo.IsDirectory && localFile is DirectoryInfo)\r
 825                         continue;\r
 826                     using (new SessionScope(FlushAction.Never))\r
 827                     {\r
 828                         var state = StatusKeeper.GetStateByFilePath(localFile.FullName);\r
 829                         _lastSeen[localFile.FullName] = DateTime.Now;\r
 830                         //Common files should be checked on a per-case basis to detect differences, which is newer\r
 831 \r
 832                         yield return new CloudAction(accountInfo, CloudActionType.MustSynch,\r
 833                                                      localFile, objectInfo, state, accountInfo.BlockSize,\r
 834                                                      accountInfo.BlockHash,"Poll Changes");\r
 835                     }\r
 836                 }\r
 837                 else\r
 838                 {\r
 839                     //Remote files should be downloaded\r
 840                     yield return new CloudDownloadAction(accountInfo, objectInfo,"Poll Changes");\r
 841                 }\r
 842             }\r
 843         }\r
 844 \r
 845         /// <summary>\r
 846         /// Creates a Local Move action for each moved server file\r
 847         /// </summary>\r
 848         /// <param name="accountInfo"></param>\r
 849         /// <param name="moves"></param>\r
 850         /// <returns></returns>\r
 851         private IEnumerable<CloudAction> MovesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> moves)\r
 852         {\r
 853             if (moves == null)\r
 854                 throw new ArgumentNullException();\r
 855             Contract.EndContractBlock();\r
 856             var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
 857 \r
 858             //In order to avoid multiple iterations over the files, we iterate only once\r
 859             //over the remote files\r
 860             foreach (var objectInfo in moves)\r
 861             {\r
 862                 var previousRelativepath = objectInfo.Previous.RelativeUrlToFilePath(accountInfo.UserName);\r
 863                 //If the previous file already exists, we can execute a Move operation\r
 864                 if (fileAgent.Exists(previousRelativepath))\r
 865                 {\r
 866                     var previousFile = fileAgent.GetFileSystemInfo(previousRelativepath);\r
 867                     using (new SessionScope(FlushAction.Never))\r
 868                     {\r
 869                         var state = StatusKeeper.GetStateByFilePath(previousFile.FullName);\r
 870                         _lastSeen[previousFile.FullName] = DateTime.Now;\r
 871 \r
 872                         //For each moved object we need to move both the local file and update                                                \r
 873                         yield return new CloudAction(accountInfo, CloudActionType.RenameLocal,\r
 874                                                      previousFile, objectInfo, state, accountInfo.BlockSize,\r
 875                                                      accountInfo.BlockHash,"Poll Moves");\r
 876                         //For modified files, we need to download the changes as well\r
 877                         if (objectInfo.Hash!=objectInfo.PreviousHash)\r
 878                             yield return new CloudDownloadAction(accountInfo,objectInfo, "Poll Moves");\r
 879                     }\r
 880                 }\r
 881                 //If the previous file does not exist, we need to download it in the new location\r
 882                 else\r
 883                 {\r
 884                     //Remote files should be downloaded\r
 885                     yield return new CloudDownloadAction(accountInfo, objectInfo, "Poll Moves");\r
 886                 }\r
 887             }\r
 888         }\r
 889 \r
 890 \r
 891         /// <summary>\r
 892         /// Creates a download action for each new server file\r
 893         /// </summary>\r
 894         /// <param name="accountInfo"></param>\r
 895         /// <param name="creates"></param>\r
 896         /// <returns></returns>\r
 897         private IEnumerable<CloudAction> CreatesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> creates)\r
 898         {\r
 899             if (creates == null)\r
 900                 throw new ArgumentNullException();\r
 901             Contract.EndContractBlock();\r
 902             var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
 903 \r
 904             //In order to avoid multiple iterations over the files, we iterate only once\r
 905             //over the remote files\r
 906             foreach (var objectInfo in creates)\r
 907             {\r
 908                 if (Log.IsDebugEnabled)\r
 909                     Log.DebugFormat("[NEW INFO] {0}",objectInfo.Uri);\r
 910 \r
 911                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 912 \r
 913                 //If the object already exists, we should check before uploading or downloading\r
 914                 if (fileAgent.Exists(relativePath))\r
 915                 {\r
 916                     var localFile= fileAgent.GetFileSystemInfo(relativePath);\r
 917                     var state = StatusKeeper.GetStateByFilePath(localFile.WithProperCapitalization().FullName);\r
 918                     yield return new CloudAction(accountInfo, CloudActionType.MustSynch,\r
 919                                                      localFile, objectInfo, state, accountInfo.BlockSize,\r
 920                                                      accountInfo.BlockHash,"Poll Creates");                    \r
 921                 }\r
 922                 else\r
 923                 {\r
 924                     //Remote files should be downloaded\r
 925                     yield return new CloudDownloadAction(accountInfo, objectInfo,"Poll Creates");\r
 926                 }\r
 927 \r
 928             }\r
 929         }\r
 930 \r
 931         /// <summary>\r
 932         /// Notify the UI to update the visual status\r
 933         /// </summary>\r
 934         /// <param name="status"></param>\r
 935         private void UpdateStatus(PithosStatus status)\r
 936         {\r
 937             try\r
 938             {\r
 939                 StatusNotification.SetPithosStatus(status);\r
 940                 //StatusNotification.Notify(new Notification());\r
 941             }\r
 942             catch (Exception exc)\r
 943             {\r
 944                 //Failure is not critical, just log it\r
 945                 Log.Warn("Error while updating status", exc);\r
 946             }\r
 947         }\r
 948 \r
 949         private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)\r
 950         {\r
 951             var containerPaths = from container in containers\r
 952                                  let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)\r
 953                                  where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)\r
 954                                  select containerPath;\r
 955 \r
 956             foreach (var path in containerPaths)\r
 957             {\r
 958                 Directory.CreateDirectory(path);\r
 959             }\r
 960         }\r
 961 \r
 962         public void AddAccount(AccountInfo accountInfo)\r
 963         {\r
 964             //Avoid adding a duplicate accountInfo\r
 965             _accounts.TryAdd(accountInfo.AccountKey, accountInfo);\r
 966         }\r
 967 \r
 968         public void RemoveAccount(AccountInfo accountInfo)\r
 969         {\r
 970             AccountInfo account;\r
 971             _accounts.TryRemove(accountInfo.AccountKey, out account);\r
 972 /*\r
 973             SnapshotDifferencer differencer;\r
 974             _differencer.Differencers.TryRemove(accountInfo.AccountKey, out differencer);\r
 975 */\r
 976         }\r
 977 \r
 978         public void SetSelectivePaths(AccountInfo accountInfo,Uri[] added, Uri[] removed)\r
 979         {\r
 980             AbortRemovedPaths(accountInfo,removed);\r
 981             DownloadNewPaths(accountInfo,added);\r
 982         }\r
 983 \r
 984         private void DownloadNewPaths(AccountInfo accountInfo, Uri[] added)\r
 985         {\r
 986             var client = new CloudFilesClient(accountInfo);\r
 987             foreach (var folderUri in added)\r
 988             {\r
 989                 try\r
 990                 {\r
 991 \r
 992                     string account;\r
 993                     string container;\r
 994                     var segmentsCount = folderUri.Segments.Length;\r
 995                     //Is this an account URL?\r
 996                     if (segmentsCount < 3)\r
 997                         continue;\r
 998                     //Is this a container or  folder URL?\r
 999                     if (segmentsCount == 3)\r
1000                     {\r
1001                         account = folderUri.Segments[1].TrimEnd('/');\r
1002                         container = folderUri.Segments[2].TrimEnd('/');\r
1003                     }\r
1004                     else\r
1005                     {\r
1006                         account = folderUri.Segments[2].TrimEnd('/');\r
1007                         container = folderUri.Segments[3].TrimEnd('/');\r
1008                     }\r
1009                     IList<ObjectInfo> items;\r
1010                     if (segmentsCount > 3)\r
1011                     {\r
1012                         //List folder\r
1013                         var folder = String.Join("", folderUri.Segments.Splice(4));\r
1014                         items = client.ListObjects(account, container, folder);\r
1015                     }\r
1016                     else\r
1017                     {\r
1018                         //List container\r
1019                         items = client.ListObjects(account, container);\r
1020                     }\r
1021                     var actions = CreatesToActions(accountInfo, items);\r
1022                     foreach (var action in actions)\r
1023                     {\r
1024                         NetworkAgent.Post(action);\r
1025                     }\r
1026                 }\r
1027                 catch (Exception exc)\r
1028                 {\r
1029                     Log.WarnFormat("Listing of new selective path [{0}] failed with \r\n{1}", folderUri, exc);\r
1030                 }\r
1031             }\r
1032 \r
1033             //Need to get a listing of each of the URLs, then post them to the NetworkAgent\r
1034             //CreatesToActions(accountInfo,)\r
1035 \r
1036 /*            NetworkAgent.Post();*/\r
1037         }\r
1038 \r
1039         private void AbortRemovedPaths(AccountInfo accountInfo, Uri[] removed)\r
1040         {\r
1041             /*this.NetworkAgent.*/\r
1042         }\r
1043     }\r
1044 }\r