code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/PollAgent.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="PollAgent.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 \r
  43 using System.Collections.Concurrent;\r
  44 using System.ComponentModel.Composition;\r
  45 using System.Diagnostics;\r
  46 using System.Diagnostics.Contracts;\r
  47 using System.IO;\r
  48 using System.Reflection;\r
  49 using System.Threading;\r
  50 using System.Threading.Tasks;\r
  51 using Castle.ActiveRecord;\r
  52 using Pithos.Interfaces;\r
  53 using Pithos.Network;\r
  54 using log4net;\r
  55 \r
  56 namespace Pithos.Core.Agents\r
  57 {\r
  58     using System;\r
  59     using System.Collections.Generic;\r
  60     using System.Linq;\r
  61 \r
  62     /// <summary>\r
  63     /// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all\r
  64     /// objects and compares it with a previously cached version to detect differences. \r
  65     /// New files are downloaded, missing files are deleted from the local file system and common files are compared\r
  66     /// to determine the appropriate action\r
  67     /// </summary>\r
  68     [Export]\r
  69     public class PollAgent\r
  70     {\r
  71         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  72 \r
  73         [System.ComponentModel.Composition.Import]\r
  74         public IStatusKeeper StatusKeeper { get; set; }\r
  75 \r
  76         [System.ComponentModel.Composition.Import]\r
  77         public IPithosSettings Settings { get; set; }\r
  78 \r
  79         [System.ComponentModel.Composition.Import]\r
  80         public NetworkAgent NetworkAgent { get; set; }\r
  81 \r
  82         [System.ComponentModel.Composition.Import]\r
  83         public Selectives Selectives { get; set; }\r
  84 \r
  85         public IStatusNotification StatusNotification { get; set; }\r
  86 \r
  87         public bool Pause\r
  88         {\r
  89             get {\r
  90                 return _pause;\r
  91             }\r
  92             set {\r
  93                 _pause = value;                \r
  94                 if (!_pause)\r
  95                     _unPauseEvent.Set();\r
  96                 else\r
  97                 {\r
  98                     _unPauseEvent.Reset();\r
  99                 }\r
 100             }\r
 101         }\r
 102 \r
 103         private bool _firstPoll = true;\r
 104 \r
 105         //The Sync Event signals a manual synchronisation\r
 106         private readonly AsyncManualResetEvent _syncEvent = new AsyncManualResetEvent();\r
 107 \r
 108         private readonly AsyncManualResetEvent _unPauseEvent = new AsyncManualResetEvent(true);\r
 109 \r
 110         private readonly ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>();\r
 111         private readonly ConcurrentDictionary<Uri, AccountInfo> _accounts = new ConcurrentDictionary<Uri,AccountInfo>();\r
 112 \r
 113 \r
 114         /// <summary>\r
 115         /// Start a manual synchronization\r
 116         /// </summary>\r
 117         public void SynchNow()\r
 118         {            \r
 119             _syncEvent.Set();\r
 120         }\r
 121 \r
 122         /// <summary>\r
 123         /// Remote files are polled periodically. Any changes are processed\r
 124         /// </summary>\r
 125         /// <param name="since"></param>\r
 126         /// <returns></returns>\r
 127         public async Task PollRemoteFiles(DateTime? since = null)\r
 128         {\r
 129             if (Log.IsDebugEnabled)\r
 130                 Log.DebugFormat("Polling changes after [{0}]",since);\r
 131 \r
 132             Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
 133             \r
 134 \r
 135             using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
 136             {\r
 137                 //If this poll fails, we will retry with the same since value\r
 138                 var nextSince = since;\r
 139                 try\r
 140                 {\r
 141                     await _unPauseEvent.WaitAsync();\r
 142                     UpdateStatus(PithosStatus.PollSyncing);\r
 143 \r
 144                     var tasks = from accountInfo in _accounts.Values\r
 145                                 select ProcessAccountFiles(accountInfo, since);\r
 146 \r
 147                     var nextTimes=await TaskEx.WhenAll(tasks.ToList());\r
 148 \r
 149                     _firstPoll = false;\r
 150                     //Reschedule the poll with the current timestamp as a "since" value\r
 151 \r
 152                     if (nextTimes.Length>0)\r
 153                         nextSince = nextTimes.Min();\r
 154                     if (Log.IsDebugEnabled)\r
 155                         Log.DebugFormat("Next Poll at [{0}]",nextSince);\r
 156                 }\r
 157                 catch (Exception ex)\r
 158                 {\r
 159                     Log.ErrorFormat("Error while processing accounts\r\n{0}", ex);\r
 160                     //In case of failure retry with the same "since" value\r
 161                 }\r
 162 \r
 163                 UpdateStatus(PithosStatus.PollComplete);\r
 164                 //The multiple try blocks are required because we can't have an await call\r
 165                 //inside a finally block\r
 166                 //TODO: Find a more elegant solution for reschedulling in the event of an exception\r
 167                 try\r
 168                 {\r
 169                     //Wait for the polling interval to pass or the Sync event to be signalled\r
 170                     nextSince = await WaitForScheduledOrManualPoll(nextSince);\r
 171                 }\r
 172                 finally\r
 173                 {\r
 174                     //Ensure polling is scheduled even in case of error\r
 175                     TaskEx.Run(() => PollRemoteFiles(nextSince));                        \r
 176                 }\r
 177             }\r
 178         }\r
 179 \r
 180         /// <summary>\r
 181         /// Wait for the polling period to expire or a manual sync request\r
 182         /// </summary>\r
 183         /// <param name="since"></param>\r
 184         /// <returns></returns>\r
 185         private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
 186         {\r
 187             var sync = _syncEvent.WaitAsync();\r
 188             var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval), NetworkAgent.CancellationToken);\r
 189             \r
 190             var signaledTask = await TaskEx.WhenAny(sync, wait);\r
 191             \r
 192             //Pausing takes precedence over manual sync or awaiting\r
 193             _unPauseEvent.Wait();\r
 194             \r
 195             //Wait for network processing to finish before polling\r
 196             var pauseTask=NetworkAgent.ProceedEvent.WaitAsync();\r
 197             await TaskEx.WhenAll(signaledTask, pauseTask);\r
 198 \r
 199             //If polling is signalled by SynchNow, ignore the since tag\r
 200             if (sync.IsCompleted)\r
 201             {\r
 202                 //TODO: Must convert to AutoReset\r
 203                 _syncEvent.Reset();\r
 204                 return null;\r
 205             }\r
 206             return since;\r
 207         }\r
 208 \r
 209         public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, DateTime? since = null)\r
 210         {\r
 211             if (accountInfo == null)\r
 212                 throw new ArgumentNullException("accountInfo");\r
 213             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
 214                 throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
 215             Contract.EndContractBlock();\r
 216 \r
 217 \r
 218             using (ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))\r
 219             {\r
 220 \r
 221                 await NetworkAgent.GetDeleteAwaiter();\r
 222 \r
 223                 Log.Info("Scheduled");\r
 224                 var client = new CloudFilesClient(accountInfo);\r
 225 \r
 226                 //We don't need to check the trash container\r
 227                 var containers = client.ListContainers(accountInfo.UserName)\r
 228                     .Where(c=>c.Name!="trash")\r
 229                     .ToList();\r
 230 \r
 231 \r
 232                 CreateContainerFolders(accountInfo, containers);\r
 233 \r
 234                 //The nextSince time fallback time is the same as the current.\r
 235                 //If polling succeeds, the next Since time will be the smallest of the maximum modification times\r
 236                 //of the shared and account objects\r
 237                 var nextSince = since;\r
 238 \r
 239                 try\r
 240                 {\r
 241                     //Wait for any deletions to finish\r
 242                     await NetworkAgent.GetDeleteAwaiter();\r
 243                     //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
 244                     //than delete a file that was created while we were executing the poll                    \r
 245 \r
 246                     //Get the list of server objects changed since the last check\r
 247                     //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
 248                     var listObjects = (from container in containers\r
 249                                        select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
 250                                              client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
 251 \r
 252                     var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
 253                         client.ListSharedObjects(since), "shared");\r
 254                     listObjects.Add(listShared);\r
 255                     var listTasks = await Task.Factory.WhenAll(listObjects.ToArray());\r
 256 \r
 257                     using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
 258                     {\r
 259                         var dict = listTasks.ToDictionary(t => t.AsyncState);\r
 260 \r
 261                         //Get all non-trash objects. Remember, the container name is stored in AsyncState\r
 262                         var remoteObjects = (from objectList in listTasks\r
 263                                             where (string)objectList.AsyncState != "trash"\r
 264                                             from obj in objectList.Result\r
 265                                             select obj).ToList();\r
 266                         \r
 267                         //Get the latest remote object modification date, only if it is after\r
 268                         //the original since date\r
 269                         nextSince = GetLatestDateAfter(nextSince, remoteObjects);\r
 270 \r
 271                         var sharedObjects = dict["shared"].Result;\r
 272                         nextSince = GetLatestDateBefore(nextSince, sharedObjects);\r
 273 \r
 274                         //DON'T process trashed files\r
 275                         //If some files are deleted and added again to a folder, they will be deleted\r
 276                         //even though they are new.\r
 277                         //We would have to check file dates and hashes to ensure that a trashed file\r
 278                         //can be deleted safely from the local hard drive.\r
 279                         /*\r
 280                         //Items with the same name, hash may be both in the container and the trash\r
 281                         //Don't delete items that exist in the container\r
 282                         var realTrash = from trash in trashObjects\r
 283                                         where\r
 284                                             !remoteObjects.Any(\r
 285                                                 info => info.Name == trash.Name && info.Hash == trash.Hash)\r
 286                                         select trash;\r
 287                         ProcessTrashedFiles(accountInfo, realTrash);\r
 288 */\r
 289 \r
 290                         var cleanRemotes = (from info in remoteObjects.Union(sharedObjects)\r
 291                                             let name = info.Name??""\r
 292                                             where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&\r
 293                                                   !name.StartsWith(FolderConstants.CacheFolder + "/",\r
 294                                                                    StringComparison.InvariantCultureIgnoreCase)\r
 295                                             select info).ToList();\r
 296 \r
 297                         if (_firstPoll)\r
 298                             StatusKeeper.CleanupOrphanStates();\r
 299                         StatusKeeper.CleanupStaleStates(accountInfo, cleanRemotes);\r
 300                         \r
 301                         var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
 302 \r
 303                         var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
 304 \r
 305                         ProcessDeletedFiles(accountInfo, differencer.Deleted.FilterDirectlyBelow(filterUris));\r
 306 \r
 307                         // @@@ NEED To add previous state here as well, To compare with previous hash\r
 308 \r
 309                         \r
 310 \r
 311                         //Create a list of actions from the remote files\r
 312                         \r
 313                         var allActions = MovesToActions(accountInfo,differencer.Moved.FilterDirectlyBelow(filterUris))\r
 314                                         .Union(\r
 315                                         ChangesToActions(accountInfo, differencer.Changed.FilterDirectlyBelow(filterUris)))\r
 316                                         .Union(\r
 317                                         CreatesToActions(accountInfo, differencer.Created.FilterDirectlyBelow(filterUris)));\r
 318 \r
 319                         //And remove those that are already being processed by the agent\r
 320                         var distinctActions = allActions\r
 321                             .Except(NetworkAgent.GetEnumerable(), new LocalFileComparer())\r
 322                             .ToList();\r
 323 \r
 324                         await _unPauseEvent.WaitAsync();\r
 325                         //Queue all the actions\r
 326                         foreach (var message in distinctActions)\r
 327                         {\r
 328                             NetworkAgent.Post(message);\r
 329                         }\r
 330 \r
 331                         Log.Info("[LISTENER] End Processing");\r
 332                     }\r
 333                 }\r
 334                 catch (Exception ex)\r
 335                 {\r
 336                     Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);\r
 337                     return nextSince;\r
 338                 }\r
 339 \r
 340                 Log.Info("[LISTENER] Finished");\r
 341                 return nextSince;\r
 342             }\r
 343         }\r
 344 \r
 345         /// <summary>\r
 346         /// Returns the latest LastModified date from the list of objects, but only if it is before\r
 347         /// than the threshold value\r
 348         /// </summary>\r
 349         /// <param name="threshold"></param>\r
 350         /// <param name="cloudObjects"></param>\r
 351         /// <returns></returns>\r
 352         private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
 353         {\r
 354             DateTime? maxDate = null;\r
 355             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
 356                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
 357             if (maxDate == null || maxDate == DateTime.MinValue)\r
 358                 return threshold;\r
 359             if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate)\r
 360                 return maxDate;\r
 361             return threshold;\r
 362         }\r
 363 \r
 364         /// <summary>\r
 365         /// Returns the latest LastModified date from the list of objects, but only if it is after\r
 366         /// the threshold value\r
 367         /// </summary>\r
 368         /// <param name="threshold"></param>\r
 369         /// <param name="cloudObjects"></param>\r
 370         /// <returns></returns>\r
 371         private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
 372         {\r
 373             DateTime? maxDate = null;\r
 374             if (cloudObjects!=null &&  cloudObjects.Count > 0)\r
 375                 maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
 376             if (maxDate == null || maxDate == DateTime.MinValue)\r
 377                 return threshold;\r
 378             if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate)\r
 379                 return maxDate;\r
 380             return threshold;\r
 381         }\r
 382 \r
 383         readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
 384         private Dictionary<Uri, List<Uri>> _selectiveUris = new Dictionary<Uri, List<Uri>>();\r
 385         private bool _pause;\r
 386 \r
 387         /// <summary>\r
 388         /// Deletes local files that are not found in the list of cloud files\r
 389         /// </summary>\r
 390         /// <param name="accountInfo"></param>\r
 391         /// <param name="cloudFiles"></param>\r
 392         private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles)\r
 393         {\r
 394             if (accountInfo == null)\r
 395                 throw new ArgumentNullException("accountInfo");\r
 396             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))\r
 397                 throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo");\r
 398             if (cloudFiles == null)\r
 399                 throw new ArgumentNullException("cloudFiles");\r
 400             Contract.EndContractBlock();\r
 401 \r
 402             //On the first run\r
 403             if (_firstPoll)\r
 404             {\r
 405                 //Only consider files that are not being modified, ie they are in the Unchanged state            \r
 406                 var deleteCandidates = FileState.Queryable.Where(state =>\r
 407                     state.FilePath.StartsWith(accountInfo.AccountPath)\r
 408                     && state.FileStatus == FileStatus.Unchanged).ToList();\r
 409 \r
 410 \r
 411                 //TODO: filesToDelete must take into account the Others container            \r
 412                 var filesToDelete = (from deleteCandidate in deleteCandidates\r
 413                                      let localFile = FileInfoExtensions.FromPath(deleteCandidate.FilePath)\r
 414                                      let relativeFilePath = localFile.AsRelativeTo(accountInfo.AccountPath)\r
 415                                      where\r
 416                                          !cloudFiles.Any(r => r.RelativeUrlToFilePath(accountInfo.UserName) == relativeFilePath)\r
 417                                      select localFile).ToList();\r
 418 \r
 419 \r
 420 \r
 421                 //Set the status of missing files to Conflict\r
 422                 foreach (var item in filesToDelete)\r
 423                 {\r
 424                     //Try to acquire a gate on the file, to take into account files that have been dequeued\r
 425                     //and are being processed\r
 426                     using (var gate = NetworkGate.Acquire(item.FullName, NetworkOperation.Deleting))\r
 427                     {\r
 428                         if (gate.Failed)\r
 429                             continue;\r
 430                         StatusKeeper.SetFileState(item.FullName, FileStatus.Conflict, FileOverlayStatus.Deleted,"Local file missing from server");\r
 431                     }\r
 432                 }\r
 433                 UpdateStatus(PithosStatus.HasConflicts);\r
 434                 StatusNotification.NotifyConflicts(filesToDelete, String.Format("{0} local files are missing from Pithos, possibly because they were deleted", filesToDelete.Count));\r
 435                 StatusNotification.NotifyForFiles(filesToDelete, String.Format("{0} files were deleted", filesToDelete.Count), TraceLevel.Info);\r
 436             }\r
 437             else\r
 438             {\r
 439                 var deletedFiles = new List<FileSystemInfo>();\r
 440                 foreach (var objectInfo in cloudFiles)\r
 441                 {\r
 442                     if (Log.IsDebugEnabled)\r
 443                         Log.DebugFormat("Handle deleted [{0}]",objectInfo.Uri);\r
 444                     var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 445                     var item = FileAgent.GetFileAgent(accountInfo).GetFileSystemInfo(relativePath);\r
 446                     if (Log.IsDebugEnabled)\r
 447                         Log.DebugFormat("Will delete [{0}] for [{1}]", item.FullName,objectInfo.Uri);\r
 448                     if (item.Exists)\r
 449                     {\r
 450                         if ((item.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)\r
 451                         {\r
 452                             item.Attributes = item.Attributes & ~FileAttributes.ReadOnly;\r
 453 \r
 454                         }\r
 455                         \r
 456                         \r
 457                         Log.DebugFormat("Deleting {0}", item.FullName);\r
 458 \r
 459                         var directory = item as DirectoryInfo;\r
 460                         if (directory!=null)\r
 461                             directory.Delete(true);\r
 462                         else\r
 463                             item.Delete();\r
 464                         Log.DebugFormat("Deleted [{0}] for [{1}]", item.FullName, objectInfo.Uri);\r
 465                         DateTime lastDate;\r
 466                         _lastSeen.TryRemove(item.FullName, out lastDate);\r
 467                         deletedFiles.Add(item);\r
 468                     }\r
 469                     StatusKeeper.SetFileState(item.FullName, FileStatus.Deleted, FileOverlayStatus.Deleted, "File Deleted");\r
 470                 }\r
 471                 Log.InfoFormat("[{0}] files were deleted",deletedFiles.Count);\r
 472                 StatusNotification.NotifyForFiles(deletedFiles, String.Format("{0} files were deleted", deletedFiles.Count), TraceLevel.Info);\r
 473             }\r
 474 \r
 475         }\r
 476 \r
 477         /// <summary>\r
 478         /// Creates a Sync action for each changed server file\r
 479         /// </summary>\r
 480         /// <param name="accountInfo"></param>\r
 481         /// <param name="changes"></param>\r
 482         /// <returns></returns>\r
 483         private IEnumerable<CloudAction> ChangesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> changes)\r
 484         {\r
 485             if (changes == null)\r
 486                 throw new ArgumentNullException();\r
 487             Contract.EndContractBlock();\r
 488             var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
 489 \r
 490             //In order to avoid multiple iterations over the files, we iterate only once\r
 491             //over the remote files\r
 492             foreach (var objectInfo in changes)\r
 493             {\r
 494                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 495                 //If a directory object already exists, we may need to sync it\r
 496                 if (fileAgent.Exists(relativePath))\r
 497                 {\r
 498                     var localFile = fileAgent.GetFileSystemInfo(relativePath);\r
 499                     //We don't need to sync directories\r
 500                     if (objectInfo.IsDirectory && localFile is DirectoryInfo)\r
 501                         continue;\r
 502                     using (new SessionScope(FlushAction.Never))\r
 503                     {\r
 504                         var state = StatusKeeper.GetStateByFilePath(localFile.FullName);\r
 505                         _lastSeen[localFile.FullName] = DateTime.Now;\r
 506                         //Common files should be checked on a per-case basis to detect differences, which is newer\r
 507 \r
 508                         yield return new CloudAction(accountInfo, CloudActionType.MustSynch,\r
 509                                                      localFile, objectInfo, state, accountInfo.BlockSize,\r
 510                                                      accountInfo.BlockHash,"Poll Changes");\r
 511                     }\r
 512                 }\r
 513                 else\r
 514                 {\r
 515                     //Remote files should be downloaded\r
 516                     yield return new CloudDownloadAction(accountInfo, objectInfo,"Poll Changes");\r
 517                 }\r
 518             }\r
 519         }\r
 520 \r
 521         /// <summary>\r
 522         /// Creates a Local Move action for each moved server file\r
 523         /// </summary>\r
 524         /// <param name="accountInfo"></param>\r
 525         /// <param name="moves"></param>\r
 526         /// <returns></returns>\r
 527         private IEnumerable<CloudAction> MovesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> moves)\r
 528         {\r
 529             if (moves == null)\r
 530                 throw new ArgumentNullException();\r
 531             Contract.EndContractBlock();\r
 532             var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
 533 \r
 534             //In order to avoid multiple iterations over the files, we iterate only once\r
 535             //over the remote files\r
 536             foreach (var objectInfo in moves)\r
 537             {\r
 538                 var previousRelativepath = objectInfo.Previous.RelativeUrlToFilePath(accountInfo.UserName);\r
 539                 //If the previous file already exists, we can execute a Move operation\r
 540                 if (fileAgent.Exists(previousRelativepath))\r
 541                 {\r
 542                     var previousFile = fileAgent.GetFileSystemInfo(previousRelativepath);\r
 543                     using (new SessionScope(FlushAction.Never))\r
 544                     {\r
 545                         var state = StatusKeeper.GetStateByFilePath(previousFile.FullName);\r
 546                         _lastSeen[previousFile.FullName] = DateTime.Now;\r
 547 \r
 548                         //For each moved object we need to move both the local file and update                                                \r
 549                         yield return new CloudAction(accountInfo, CloudActionType.RenameLocal,\r
 550                                                      previousFile, objectInfo, state, accountInfo.BlockSize,\r
 551                                                      accountInfo.BlockHash,"Poll Moves");\r
 552                         //For modified files, we need to download the changes as well\r
 553                         if (objectInfo.Hash!=objectInfo.PreviousHash)\r
 554                             yield return new CloudDownloadAction(accountInfo,objectInfo, "Poll Moves");\r
 555                     }\r
 556                 }\r
 557                 //If the previous file does not exist, we need to download it in the new location\r
 558                 else\r
 559                 {\r
 560                     //Remote files should be downloaded\r
 561                     yield return new CloudDownloadAction(accountInfo, objectInfo, "Poll Moves");\r
 562                 }\r
 563             }\r
 564         }\r
 565 \r
 566 \r
 567         /// <summary>\r
 568         /// Creates a download action for each new server file\r
 569         /// </summary>\r
 570         /// <param name="accountInfo"></param>\r
 571         /// <param name="creates"></param>\r
 572         /// <returns></returns>\r
 573         private IEnumerable<CloudAction> CreatesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> creates)\r
 574         {\r
 575             if (creates == null)\r
 576                 throw new ArgumentNullException();\r
 577             Contract.EndContractBlock();\r
 578             var fileAgent = FileAgent.GetFileAgent(accountInfo);\r
 579 \r
 580             //In order to avoid multiple iterations over the files, we iterate only once\r
 581             //over the remote files\r
 582             foreach (var objectInfo in creates)\r
 583             {\r
 584                 if (Log.IsDebugEnabled)\r
 585                     Log.DebugFormat("[NEW INFO] {0}",objectInfo.Uri);\r
 586 \r
 587                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
 588 \r
 589                 //If the object already exists, we should check before uploading or downloading\r
 590                 if (fileAgent.Exists(relativePath))\r
 591                 {\r
 592                     var localFile= fileAgent.GetFileSystemInfo(relativePath);\r
 593                     var state = StatusKeeper.GetStateByFilePath(localFile.WithProperCapitalization().FullName);\r
 594                     yield return new CloudAction(accountInfo, CloudActionType.MustSynch,\r
 595                                                      localFile, objectInfo, state, accountInfo.BlockSize,\r
 596                                                      accountInfo.BlockHash,"Poll Creates");                    \r
 597                 }\r
 598                 else\r
 599                 {\r
 600                     //Remote files should be downloaded\r
 601                     yield return new CloudDownloadAction(accountInfo, objectInfo,"Poll Creates");\r
 602                 }\r
 603 \r
 604             }\r
 605         }\r
 606 \r
 607         /// <summary>\r
 608         /// Notify the UI to update the visual status\r
 609         /// </summary>\r
 610         /// <param name="status"></param>\r
 611         private void UpdateStatus(PithosStatus status)\r
 612         {\r
 613             try\r
 614             {\r
 615                 StatusNotification.SetPithosStatus(status);\r
 616                 //StatusNotification.Notify(new Notification());\r
 617             }\r
 618             catch (Exception exc)\r
 619             {\r
 620                 //Failure is not critical, just log it\r
 621                 Log.Warn("Error while updating status", exc);\r
 622             }\r
 623         }\r
 624 \r
 625         private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)\r
 626         {\r
 627             var containerPaths = from container in containers\r
 628                                  let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)\r
 629                                  where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)\r
 630                                  select containerPath;\r
 631 \r
 632             foreach (var path in containerPaths)\r
 633             {\r
 634                 Directory.CreateDirectory(path);\r
 635             }\r
 636         }\r
 637 \r
 638         public void AddAccount(AccountInfo accountInfo)\r
 639         {\r
 640             //Avoid adding a duplicate accountInfo\r
 641             _accounts.TryAdd(accountInfo.AccountKey, accountInfo);\r
 642         }\r
 643 \r
 644         public void RemoveAccount(AccountInfo accountInfo)\r
 645         {\r
 646             AccountInfo account;\r
 647             _accounts.TryRemove(accountInfo.AccountKey, out account);\r
 648             SnapshotDifferencer differencer;\r
 649             _differencer.Differencers.TryRemove(accountInfo.AccountKey, out differencer);\r
 650         }\r
 651 \r
 652         public void SetSelectivePaths(AccountInfo accountInfo,Uri[] added, Uri[] removed)\r
 653         {\r
 654             AbortRemovedPaths(accountInfo,removed);\r
 655             DownloadNewPaths(accountInfo,added);\r
 656         }\r
 657 \r
 658         private void DownloadNewPaths(AccountInfo accountInfo, Uri[] added)\r
 659         {\r
 660             var client = new CloudFilesClient(accountInfo);\r
 661             foreach (var folderUri in added)\r
 662             {\r
 663                 try\r
 664                 {\r
 665 \r
 666                     string account;\r
 667                     string container;\r
 668                     var segmentsCount = folderUri.Segments.Length;\r
 669                     //Is this an account URL?\r
 670                     if (segmentsCount < 3)\r
 671                         continue;\r
 672                     //Is this a container or  folder URL?\r
 673                     if (segmentsCount == 3)\r
 674                     {\r
 675                         account = folderUri.Segments[1].TrimEnd('/');\r
 676                         container = folderUri.Segments[2].TrimEnd('/');\r
 677                     }\r
 678                     else\r
 679                     {\r
 680                         account = folderUri.Segments[2].TrimEnd('/');\r
 681                         container = folderUri.Segments[3].TrimEnd('/');\r
 682                     }\r
 683                     IList<ObjectInfo> items;\r
 684                     if (segmentsCount > 3)\r
 685                     {\r
 686                         //List folder\r
 687                         var folder = String.Join("", folderUri.Segments.Splice(4));\r
 688                         items = client.ListObjects(account, container, folder);\r
 689                     }\r
 690                     else\r
 691                     {\r
 692                         //List container\r
 693                         items = client.ListObjects(account, container);\r
 694                     }\r
 695                     var actions = CreatesToActions(accountInfo, items);\r
 696                     foreach (var action in actions)\r
 697                     {\r
 698                         NetworkAgent.Post(action);\r
 699                     }\r
 700                 }\r
 701                 catch (Exception exc)\r
 702                 {\r
 703                     Log.WarnFormat("Listing of new selective path [{0}] failed with \r\n{1}", folderUri, exc);\r
 704                 }\r
 705             }\r
 706 \r
 707             //Need to get a listing of each of the URLs, then post them to the NetworkAgent\r
 708             //CreatesToActions(accountInfo,)\r
 709 \r
 710 /*            NetworkAgent.Post();*/\r
 711         }\r
 712 \r
 713         private void AbortRemovedPaths(AccountInfo accountInfo, Uri[] removed)\r
 714         {\r
 715             /*this.NetworkAgent.*/\r
 716         }\r
 717     }\r
 718 }\r