code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/NetworkAgent.cs

   1 // -----------------------------------------------------------------------
   2 // <copyright file="NetworkAgent.cs" company="GRNET">
   3 // Copyright 2011 GRNET S.A. All rights reserved.
   4 //
   5 // Redistribution and use in source and binary forms, with or
   6 // without modification, are permitted provided that the following
   7 // conditions are met:
   8 //
   9 //   1. Redistributions of source code must retain the above
  10 //      copyright notice, this list of conditions and the following
  11 //      disclaimer.
  12 //
  13 //   2. Redistributions in binary form must reproduce the above
  14 //      copyright notice, this list of conditions and the following
  15 //      disclaimer in the documentation and/or other materials
  16 //      provided with the distribution.
  17 //
  18 // THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  19 // OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  21 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  22 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  25 // USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  26 // AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27 // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  28 // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29 // POSSIBILITY OF SUCH DAMAGE.
  30 //
  31 // The views and conclusions contained in the software and
  32 // documentation are those of the authors and should not be
  33 // interpreted as representing official policies, either expressed
  34 // or implied, of GRNET S.A.
  35 // </copyright>
  36 // -----------------------------------------------------------------------
  37
  38 using System;
  39 using System.Collections.Concurrent;
  40 using System.Collections.Generic;
  41 using System.ComponentModel.Composition;
  42 using System.Diagnostics;
  43 using System.Diagnostics.Contracts;
  44 using System.IO;
  45 using System.Linq;
  46 using System.Net;
  47 using System.Threading;
  48 using System.Threading.Tasks;
  49 using System.Threading.Tasks.Dataflow;
  50 using Castle.ActiveRecord;
  51 using Pithos.Interfaces;
  52 using Pithos.Network;
  53 using log4net;
  54
  55 namespace Pithos.Core.Agents
  56 {
  57     //TODO: Ensure all network operations use exact casing. Pithos is case sensitive
  58     [Export]
  59     public class NetworkAgent
  60     {
  61         private Agent<CloudAction> _agent;
  62
  63         //A separate agent is used to execute delete actions immediatelly;
  64         private ActionBlock<CloudDeleteAction> _deleteAgent;
  65         readonly ConcurrentDictionary<string,DateTime> _deletedFiles=new ConcurrentDictionary<string, DateTime>();
  66
  67
  68         private readonly AutoResetEvent _pauseAgent = new AutoResetEvent(false);
  69
  70         [System.ComponentModel.Composition.Import]
  71         public IStatusKeeper StatusKeeper { get; set; }
  72
  73         public IStatusNotification StatusNotification { get; set; }
  74
  75         private static readonly ILog Log = LogManager.GetLogger("NetworkAgent");
  76
  77         private readonly ConcurrentBag<AccountInfo> _accounts = new ConcurrentBag<AccountInfo>();
  78
  79         public void Start()
  80         {
  81
  82             _agent = Agent<CloudAction>.Start(inbox =>
  83             {
  84                 Action loop = null;
  85                 loop = () =>
  86                 {
  87                     _pauseAgent.WaitOne();
  88                     var message = inbox.Receive();
  89                     var process=message.Then(Process,inbox.CancellationToken);
  90                     inbox.LoopAsync(process, loop);
  91                 };
  92                 loop();
  93             });
  94
  95             _deleteAgent = new ActionBlock<CloudDeleteAction>(message =>ProcessDelete(message),new ExecutionDataflowBlockOptions{MaxDegreeOfParallelism=4});
  96             /*
  97                 Action loop = null;
  98                 loop = () =>
  99                             {
 100                                 var message = inbox.Receive();
 101                                 var process = message.Then(ProcessDelete,inbox.CancellationToken);
 102                                 inbox.LoopAsync(process, loop);
 103                             };
 104                 loop();
 105 */
 106
 107         }
 108
 109         private async Task Process(CloudAction action)
 110         {
 111             if (action == null)
 112                 throw new ArgumentNullException("action");
 113             if (action.AccountInfo==null)
 114                 throw new ArgumentException("The action.AccountInfo is empty","action");
 115             Contract.EndContractBlock();
 116
 117             var accountInfo = action.AccountInfo;
 118
 119             using (log4net.ThreadContext.Stacks["NETWORK"].Push("PROCESS"))
 120             {
 121                 Log.InfoFormat("[ACTION] Start Processing {0}", action);
 122
 123                 var cloudFile = action.CloudFile;
 124                 var downloadPath = action.GetDownloadPath();
 125
 126                 try
 127                 {
 128
 129                     if (action.Action == CloudActionType.DeleteCloud)
 130                     {
 131                         //Redirect deletes to the delete agent
 132                         _deleteAgent.Post((CloudDeleteAction)action);
 133                     }
 134                     if (IsDeletedFile(action))
 135                     {
 136                         //Clear the status of already deleted files to avoid reprocessing
 137                         if (action.LocalFile != null)
 138                             this.StatusKeeper.ClearFileStatus(action.LocalFile.FullName);
 139                     }
 140                     else
 141                     {
 142                         switch (action.Action)
 143                         {
 144                             case CloudActionType.UploadUnconditional:
 145                                 //Abort if the file was deleted before we reached this point
 146                                 await UploadCloudFile(action);
 147                                 break;
 148                             case CloudActionType.DownloadUnconditional:
 149                                 await DownloadCloudFile(accountInfo, cloudFile, downloadPath);
 150                                 break;
 151                             case CloudActionType.RenameCloud:
 152                                 var moveAction = (CloudMoveAction) action;
 153                                 RenameCloudFile(accountInfo, moveAction);
 154                                 break;
 155                             case CloudActionType.MustSynch:
 156                                 if (!File.Exists(downloadPath) && !Directory.Exists(downloadPath))
 157                                 {
 158                                     await DownloadCloudFile(accountInfo, cloudFile, downloadPath);
 159                                 }
 160                                 else
 161                                 {
 162                                     await SyncFiles(accountInfo, action);
 163                                 }
 164                                 break;
 165                         }
 166                     }
 167                     Log.InfoFormat("[ACTION] End Processing {0}:{1}->{2}", action.Action, action.LocalFile,
 168                                            action.CloudFile.Name);
 169                 }
 170                 catch (WebException exc)
 171                 {
 172                     Log.ErrorFormat("[WEB ERROR] {0} : {1} -> {2} due to exception\r\n{3}", action.Action, action.LocalFile, action.CloudFile, exc);
 173                 }
 174                 catch (OperationCanceledException)
 175                 {
 176                     throw;
 177                 }
 178                 catch (DirectoryNotFoundException)
 179                 {
 180                     Log.ErrorFormat("{0} : {1} -> {2}  failed because the directory was not found.\n Rescheduling a delete",
 181                         action.Action, action.LocalFile, action.CloudFile);
 182                     //Post a delete action for the missing file
 183                     Post(new CloudDeleteAction(action));
 184                 }
 185                 catch (FileNotFoundException)
 186                 {
 187                     Log.ErrorFormat("{0} : {1} -> {2}  failed because the file was not found.\n Rescheduling a delete",
 188                         action.Action, action.LocalFile, action.CloudFile);
 189                     //Post a delete action for the missing file
 190                     Post(new CloudDeleteAction(action));
 191                 }
 192                 catch (Exception exc)
 193                 {
 194                     Log.ErrorFormat("[REQUEUE] {0} : {1} -> {2} due to exception\r\n{3}",
 195                                      action.Action, action.LocalFile, action.CloudFile, exc);
 196
 197                     _agent.Post(action);
 198                 }
 199             }
 200         }
 201
 202         /// <summary>
 203         /// Processes cloud delete actions
 204         /// </summary>
 205         /// <param name="action">The delete action to execute</param>
 206         /// <returns></returns>
 207         /// <remarks>
 208         /// When a file/folder is deleted locally, we must delete it ASAP from the server and block any download
 209         /// operations that may be in progress.
 210         /// <para>
 211         /// A separate agent is used to process deletes because the main agent may be busy with a long operation.
 212         /// </para>
 213         /// </remarks>
 214         private async Task ProcessDelete(CloudDeleteAction action)
 215         {
 216             if (action == null)
 217                 throw new ArgumentNullException("action");
 218             if (action.AccountInfo==null)
 219                 throw new ArgumentException("The action.AccountInfo is empty","action");
 220             Contract.EndContractBlock();
 221
 222             var accountInfo = action.AccountInfo;
 223
 224             using (log4net.ThreadContext.Stacks["NETWORK"].Push("PROCESS"))
 225             {
 226                 Log.InfoFormat("[ACTION] Start Processing {0}", action);
 227
 228                 var cloudFile = action.CloudFile;
 229
 230                 try
 231                 {
 232                     //Acquire a lock on the deleted file to prevent uploading/downloading operations from the normal
 233                     //agent
 234                     using (var gate = NetworkGate.Acquire(action.LocalFile.FullName, NetworkOperation.Deleting))
 235                     {
 236
 237                         //Add the file URL to the deleted files list
 238                         var key = GetFileKey(action.CloudFile);
 239                         _deletedFiles[key] = DateTime.Now;
 240
 241                         _pauseAgent.Reset();
 242                         // and then delete the file from the server
 243                         DeleteCloudFile(accountInfo, cloudFile);
 244
 245                         Log.InfoFormat("[ACTION] End Delete {0}:{1}->{2}", action.Action, action.LocalFile,
 246                                        action.CloudFile.Name);
 247                     }
 248                 }
 249                 catch (WebException exc)
 250                 {
 251                     Log.ErrorFormat("[WEB ERROR] {0} : {1} -> {2} due to exception\r\n{3}", action.Action, action.LocalFile, action.CloudFile, exc);
 252                 }
 253                 catch (OperationCanceledException)
 254                 {
 255                     throw;
 256                 }
 257                 catch (DirectoryNotFoundException)
 258                 {
 259                     Log.ErrorFormat("{0} : {1} -> {2}  failed because the directory was not found.\n Rescheduling a delete",
 260                         action.Action, action.LocalFile, action.CloudFile);
 261                     //Repost a delete action for the missing file
 262                     _deleteAgent.Post(action);
 263                 }
 264                 catch (FileNotFoundException)
 265                 {
 266                     Log.ErrorFormat("{0} : {1} -> {2}  failed because the file was not found.\n Rescheduling a delete",
 267                         action.Action, action.LocalFile, action.CloudFile);
 268                     //Post a delete action for the missing file
 269                     _deleteAgent.Post(action);
 270                 }
 271                 catch (Exception exc)
 272                 {
 273                     Log.ErrorFormat("[REQUEUE] {0} : {1} -> {2} due to exception\r\n{3}",
 274                                      action.Action, action.LocalFile, action.CloudFile, exc);
 275
 276                     _deleteAgent.Post(action);
 277                 }
 278                 finally
 279                 {
 280                     if (_deleteAgent.InputCount == 0)
 281                         _pauseAgent.Set();
 282
 283                 }
 284             }
 285         }
 286
 287         private static string GetFileKey(ObjectInfo info)
 288         {
 289             var key = String.Format("{0}/{1}/{2}", info.Account, info.Container,info.Name);
 290             return key;
 291         }
 292
 293         private async Task SyncFiles(AccountInfo accountInfo,CloudAction action)
 294         {
 295             if (accountInfo == null)
 296                 throw new ArgumentNullException("accountInfo");
 297             if (action==null)
 298                 throw new ArgumentNullException("action");
 299             if (action.LocalFile==null)
 300                 throw new ArgumentException("The action's local file is not specified","action");
 301             if (!Path.IsPathRooted(action.LocalFile.FullName))
 302                 throw new ArgumentException("The action's local file path must be absolute","action");
 303             if (action.CloudFile== null)
 304                 throw new ArgumentException("The action's cloud file is not specified", "action");
 305             Contract.EndContractBlock();
 306
 307             var localFile = action.LocalFile;
 308             var cloudFile = action.CloudFile;
 309             var downloadPath=action.LocalFile.GetProperCapitalization();
 310
 311             var cloudHash = cloudFile.Hash.ToLower();
 312             var localHash = action.LocalHash.Value.ToLower();
 313             var topHash = action.TopHash.Value.ToLower();
 314
 315             //Not enough to compare only the local hashes, also have to compare the tophashes
 316
 317             //If any of the hashes match, we are done
 318             if ((cloudHash == localHash || cloudHash == topHash))
 319             {
 320                 Log.InfoFormat("Skipping {0}, hashes match",downloadPath);
 321                 return;
 322             }
 323
 324             //The hashes DON'T match. We need to sync
 325             var lastLocalTime = localFile.LastWriteTime;
 326             var lastUpTime = cloudFile.Last_Modified;
 327
 328             //If the local file is newer upload it
 329             if (lastUpTime <= lastLocalTime)
 330             {
 331                 //It probably means it was changed while the app was down
 332                 UploadCloudFile(action);
 333             }
 334             else
 335             {
 336                 //It the cloud file has a later date, it was modified by another user or computer.
 337                 //We need to check the local file's status
 338                 var status = StatusKeeper.GetFileStatus(downloadPath);
 339                 switch (status)
 340                 {
 341                     case FileStatus.Unchanged:
 342                         //If the local file's status is Unchanged, we can go on and download the newer cloud file
 343                         await DownloadCloudFile(accountInfo,cloudFile,downloadPath);
 344                         break;
 345                     case FileStatus.Modified:
 346                         //If the local file is Modified, we may have a conflict. In this case we should mark the file as Conflict
 347                         //We can't ensure that a file modified online since the last time will appear as Modified, unless we
 348                         //index all files before we start listening.
 349                     case FileStatus.Created:
 350                         //If the local file is Created, it means that the local and cloud files aren't related,
 351                         // yet they have the same name.
 352
 353                         //In both cases we must mark the file as in conflict
 354                         ReportConflict(downloadPath);
 355                         break;
 356                     default:
 357                         //Other cases should never occur. Mark them as Conflict as well but log a warning
 358                         ReportConflict(downloadPath);
 359                         Log.WarnFormat("Unexcepted status {0} for file {1}->{2}", status,
 360                                        downloadPath, action.CloudFile.Name);
 361                         break;
 362                 }
 363             }
 364         }
 365
 366         private void ReportConflict(string downloadPath)
 367         {
 368             if (String.IsNullOrWhiteSpace(downloadPath))
 369                 throw new ArgumentNullException("downloadPath");
 370             Contract.EndContractBlock();
 371
 372             StatusKeeper.SetFileOverlayStatus(downloadPath, FileOverlayStatus.Conflict);
 373             var message = String.Format("Conflict detected for file {0}", downloadPath);
 374             Log.Warn(message);
 375             StatusNotification.NotifyChange(message, TraceLevel.Warning);
 376         }
 377
 378         public void Post(CloudAction cloudAction)
 379         {
 380             if (cloudAction == null)
 381                 throw new ArgumentNullException("cloudAction");
 382             if (cloudAction.AccountInfo==null)
 383                 throw new ArgumentException("The CloudAction.AccountInfo is empty","cloudAction");
 384             Contract.EndContractBlock();
 385
 386             //If the action targets a local file, add a treehash calculation
 387             if (!(cloudAction is CloudDeleteAction) && cloudAction.LocalFile as FileInfo != null)
 388             {
 389                 var accountInfo = cloudAction.AccountInfo;
 390                 var localFile = (FileInfo) cloudAction.LocalFile;
 391                 if (localFile.Length > accountInfo.BlockSize)
 392                     cloudAction.TopHash =
 393                         new Lazy<string>(() => Signature.CalculateTreeHashAsync(localFile,
 394                                                                                 accountInfo.BlockSize,
 395                                                                                 accountInfo.BlockHash).Result
 396                                                     .TopHash.ToHashString());
 397                 else
 398                 {
 399                     cloudAction.TopHash = new Lazy<string>(() => cloudAction.LocalHash.Value);
 400                 }
 401             }
 402             else
 403             {
 404                 //The hash for a directory is the empty string
 405                 cloudAction.TopHash = new Lazy<string>(() => String.Empty);
 406             }
 407
 408             if (cloudAction is CloudDeleteAction)
 409                 _deleteAgent.Post((CloudDeleteAction)cloudAction);
 410             else
 411                 _agent.Post(cloudAction);
 412         }
 413
 414        /* class ObjectInfoByNameComparer:IEqualityComparer<ObjectInfo>
 415         {
 416             public bool Equals(ObjectInfo x, ObjectInfo y)
 417             {
 418                 return x.Name.Equals(y.Name,StringComparison.InvariantCultureIgnoreCase);
 419             }
 420
 421             public int GetHashCode(ObjectInfo obj)
 422             {
 423                 return obj.Name.ToLower().GetHashCode();
 424             }
 425         }*/
 426
 427
 428
 429         //Remote files are polled periodically. Any changes are processed
 430         public async Task ProcessRemoteFiles(DateTime? since = null)
 431         {
 432             await TaskEx.Delay(TimeSpan.FromSeconds(10),_agent.CancellationToken);
 433
 434             using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))
 435             {
 436
 437                 try
 438                 {
 439                     //Next time we will check for all changes since the current check minus 1 second
 440                     //This is done to ensure there are no discrepancies due to clock differences
 441                     DateTime nextSince = DateTime.Now.AddSeconds(-1);
 442
 443                     var tasks = from accountInfo in _accounts
 444                                 select ProcessAccountFiles(accountInfo, since);
 445
 446                     await TaskEx.WhenAll(tasks.ToList());
 447
 448                     ProcessRemoteFiles(nextSince);
 449                 }
 450                 catch (Exception ex)
 451                 {
 452                     Log.ErrorFormat("Error while processing accounts\r\n{0}",ex);
 453                     //In case of failure retry with the same parameter
 454                     ProcessRemoteFiles(since);
 455                 }
 456
 457
 458             }
 459         }
 460
 461         public async Task ProcessAccountFiles(AccountInfo accountInfo,DateTime? since=null)
 462         {
 463             if (accountInfo==null)
 464                 throw new ArgumentNullException("accountInfo");
 465             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))
 466                 throw new ArgumentException("The AccountInfo.AccountPath is empty","accountInfo");
 467             Contract.EndContractBlock();
 468
 469             using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))
 470             {
 471                 Log.Info("Scheduled");
 472                 var client=new CloudFilesClient(accountInfo);
 473
 474                 var containers = client.ListContainers(accountInfo.UserName);
 475
 476                 CreateContainerFolders(accountInfo, containers);
 477
 478                 try
 479                 {
 480
 481                     //Get the list of server objects changed since the last check
 482                     //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step
 483                     var listObjects = from container in containers
 484                                       select  Task<IList<ObjectInfo>>.Factory.StartNew(_ =>
 485                                             client.ListObjects(accountInfo.UserName,container.Name, since),container.Name);
 486
 487
 488                     var listTasks = await Task.Factory.WhenAll(listObjects.ToArray());
 489
 490                     using (log4net.ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))
 491                     {
 492                         var dict = listTasks.ToDictionary(t => t.AsyncState);
 493
 494                         //Get all non-trash objects. Remember, the container name is stored in AsyncState
 495                         var remoteObjects = from objectList in listTasks
 496                                             where (string) objectList.AsyncState != "trash"
 497                                             from obj in objectList.Result
 498                                             select obj;
 499
 500                         var trashObjects = dict["trash"].Result;
 501                         //var sharedObjects = ((Task<IList<ObjectInfo>>) task.Result[2]).Result;
 502
 503                         //Items with the same name, hash may be both in the container and the trash
 504                         //Don't delete items that exist in the container
 505                         var realTrash = from trash in trashObjects
 506                                         where
 507                                             !remoteObjects.Any(
 508                                                 info => info.Name == trash.Name && info.Hash == trash.Hash)
 509                                         select trash;
 510                         ProcessDeletedFiles(accountInfo, realTrash);
 511
 512
 513                         var remote = from info in remoteObjects
 514                                      //.Union(sharedObjects)
 515                                      let name = info.Name
 516                                      where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&
 517                                            !name.StartsWith(FolderConstants.CacheFolder + "/",
 518                                                             StringComparison.InvariantCultureIgnoreCase)
 519                                      select info;
 520
 521                         //Create a list of actions from the remote files
 522                         var allActions = ObjectsToActions(accountInfo, remote);
 523
 524                         //And remove those that are already being processed by the agent
 525                         var distinctActions = allActions
 526                             .Except(_agent.GetEnumerable(), new PithosMonitor.LocalFileComparer())
 527                             .ToList();
 528
 529                         //Queue all the actions
 530                         foreach (var message in distinctActions)
 531                         {
 532                             Post(message);
 533                         }
 534
 535                         Log.Info("[LISTENER] End Processing");
 536                     }
 537                 }
 538                 catch (Exception ex)
 539                 {
 540                     Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex);
 541                     return;
 542                 }
 543
 544                 Log.Info("[LISTENER] Finished");
 545
 546             }
 547         }
 548
 549         private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)
 550         {
 551             var containerPaths = from container in containers
 552                                  let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)
 553                                  where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)
 554                                  select containerPath;
 555
 556             foreach (var path in containerPaths)
 557             {
 558                 Directory.CreateDirectory(path);
 559             }
 560         }
 561
 562         //Creates an appropriate action for each server file
 563         private IEnumerable<CloudAction> ObjectsToActions(AccountInfo accountInfo,IEnumerable<ObjectInfo> remote)
 564         {
 565             if (remote==null)
 566                 throw new ArgumentNullException();
 567             Contract.EndContractBlock();
 568             var fileAgent = GetFileAgent(accountInfo);
 569
 570             //In order to avoid multiple iterations over the files, we iterate only once
 571             //over the remote files
 572             foreach (var objectInfo in remote)
 573             {
 574                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);
 575                 //and remove any matching objects from the list, adding them to the commonObjects list
 576
 577                 if (fileAgent.Exists(relativePath))
 578                 {
 579                     //If a directory object already exists, we don't need to perform any other action
 580                     var localFile = fileAgent.GetFileSystemInfo(relativePath);
 581                     if (objectInfo.Content_Type == @"application/directory" && localFile is DirectoryInfo)
 582                         continue;
 583                     using (new SessionScope(FlushAction.Never))
 584                     {
 585                         var state =  StatusKeeper.GetStateByFilePath(localFile.FullName);
 586                         //FileState.FindByFilePath(localFile.FullName);
 587                         //Common files should be checked on a per-case basis to detect differences, which is newer
 588
 589                         yield return new CloudAction(accountInfo, CloudActionType.MustSynch,
 590                                                      localFile, objectInfo, state, accountInfo.BlockSize,
 591                                                      accountInfo.BlockHash);
 592                     }
 593                 }
 594                 else
 595                 {
 596                     //If there is no match we add them to the localFiles list
 597                     //but only if the file is not marked for deletion
 598                     var targetFile = Path.Combine(accountInfo.AccountPath, relativePath);
 599                     var fileStatus = StatusKeeper.GetFileStatus(targetFile);
 600                     if (fileStatus != FileStatus.Deleted)
 601                     {
 602                         //Remote files should be downloaded
 603                         yield return new CloudDownloadAction(accountInfo,objectInfo);
 604                     }
 605                 }
 606             }
 607         }
 608
 609         private static FileAgent GetFileAgent(AccountInfo accountInfo)
 610         {
 611             return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);
 612         }
 613
 614         private void ProcessDeletedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects)
 615         {
 616             var fileAgent = GetFileAgent(accountInfo);
 617             foreach (var trashObject in trashObjects)
 618             {
 619                 var barePath = trashObject.RelativeUrlToFilePath(accountInfo.UserName);
 620                 //HACK: Assume only the "pithos" container is used. Must find out what happens when
 621                 //deleting a file from a different container
 622                 var relativePath = Path.Combine("pithos", barePath);
 623                 fileAgent.Delete(relativePath);
 624             }
 625         }
 626
 627
 628         private void RenameCloudFile(AccountInfo accountInfo,CloudMoveAction action)
 629         {
 630             if (accountInfo==null)
 631                 throw new ArgumentNullException("accountInfo");
 632             if (action==null)
 633                 throw new ArgumentNullException("action");
 634             if (action.CloudFile==null)
 635                 throw new ArgumentException("CloudFile","action");
 636             if (action.LocalFile==null)
 637                 throw new ArgumentException("LocalFile","action");
 638             if (action.OldLocalFile==null)
 639                 throw new ArgumentException("OldLocalFile","action");
 640             if (action.OldCloudFile==null)
 641                 throw new ArgumentException("OldCloudFile","action");
 642             Contract.EndContractBlock();
 643
 644
 645             var newFilePath = action.LocalFile.FullName;
 646
 647             //How do we handle concurrent renames and deletes/uploads/downloads?
 648             //* A conflicting upload means that a file was renamed before it had a chance to finish uploading
 649             //  This should never happen as the network agent executes only one action at a time
 650             //* A conflicting download means that the file was modified on the cloud. While we can go on and complete
 651             //  the rename, there may be a problem if the file is downloaded in blocks, as subsequent block requests for the
 652             //  same name will fail.
 653             //  This should never happen as the network agent executes only one action at a time.
 654             //* A conflicting delete can happen if the rename was followed by a delete action that didn't have the chance
 655             //  to remove the rename from the queue.
 656             //  We can probably ignore this case. It will result in an error which should be ignored
 657
 658
 659             //The local file is already renamed
 660             StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Modified);
 661
 662
 663             var account = action.CloudFile.Account ?? accountInfo.UserName;
 664             var container = action.CloudFile.Container;
 665
 666             var client = new CloudFilesClient(accountInfo);
 667             //TODO: What code is returned when the source file doesn't exist?
 668             client.MoveObject(account, container, action.OldCloudFile.Name, container, action.CloudFile.Name);
 669
 670             StatusKeeper.SetFileStatus(newFilePath, FileStatus.Unchanged);
 671             StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Normal);
 672             NativeMethods.RaiseChangeNotification(newFilePath);
 673         }
 674
 675         private void DeleteCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile)
 676         {
 677             if (accountInfo == null)
 678                 throw new ArgumentNullException("accountInfo");
 679             if (cloudFile==null)
 680                 throw new ArgumentNullException("cloudFile");
 681
 682             if (String.IsNullOrWhiteSpace(cloudFile.Container))
 683                 throw new ArgumentException("Invalid container", "cloudFile");
 684             Contract.EndContractBlock();
 685
 686             var fileAgent = GetFileAgent(accountInfo);
 687
 688             using ( log4net.ThreadContext.Stacks["DeleteCloudFile"].Push("Delete"))
 689             {
 690                 var fileName= cloudFile.RelativeUrlToFilePath(accountInfo.UserName);
 691                 var info = fileAgent.GetFileSystemInfo(fileName);
 692                 var fullPath = info.FullName.ToLower();
 693
 694                 StatusKeeper.SetFileOverlayStatus(fullPath, FileOverlayStatus.Modified);
 695
 696                 var account = cloudFile.Account ?? accountInfo.UserName;
 697                 var container = cloudFile.Container ;//?? FolderConstants.PithosContainer;
 698
 699                 var client = new CloudFilesClient(accountInfo);
 700                 client.DeleteObject(account, container, cloudFile.Name);
 701
 702                 StatusKeeper.ClearFileStatus(fullPath);
 703             }
 704         }
 705
 706         //Download a file.
 707         private async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile , string filePath)
 708         {
 709             if (accountInfo == null)
 710                 throw new ArgumentNullException("accountInfo");
 711             if (cloudFile == null)
 712                 throw new ArgumentNullException("cloudFile");
 713             if (String.IsNullOrWhiteSpace(cloudFile.Account))
 714                 throw new ArgumentNullException("cloudFile");
 715             if (String.IsNullOrWhiteSpace(cloudFile.Container))
 716                 throw new ArgumentNullException("cloudFile");
 717             if (String.IsNullOrWhiteSpace(filePath))
 718                 throw new ArgumentNullException("filePath");
 719             if (!Path.IsPathRooted(filePath))
 720                 throw new ArgumentException("The filePath must be rooted", "filePath");
 721             Contract.EndContractBlock();
 722
 723             var localPath = Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath);
 724             var relativeUrl = new Uri(cloudFile.Name, UriKind.Relative);
 725
 726             var url = relativeUrl.ToString();
 727             if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase))
 728                 return;
 729
 730
 731             //Are we already downloading or uploading the file?
 732             using (var gate=NetworkGate.Acquire(localPath, NetworkOperation.Downloading))
 733             {
 734                 if (gate.Failed)
 735                     return;
 736                 //The file's hashmap will be stored in the same location with the extension .hashmap
 737                 //var hashPath = Path.Combine(FileAgent.CachePath, relativePath + ".hashmap");
 738
 739                 var client = new CloudFilesClient(accountInfo);
 740                 var account = cloudFile.Account;
 741                 var container = cloudFile.Container;
 742
 743                 if (cloudFile.Content_Type == @"application/directory")
 744                 {
 745                     if (!Directory.Exists(localPath))
 746                         Directory.CreateDirectory(localPath);
 747                 }
 748                 else
 749                 {
 750                     //Retrieve the hashmap from the server
 751                     var serverHash = await client.GetHashMap(account, container, url);
 752                     //If it's a small file
 753                     if (serverHash.Hashes.Count == 1)
 754                         //Download it in one go
 755                         await
 756                             DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath, serverHash);
 757                         //Otherwise download it block by block
 758                     else
 759                         await DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath, serverHash);
 760
 761                     if (cloudFile.AllowedTo == "read")
 762                     {
 763                         var attributes = File.GetAttributes(localPath);
 764                         File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly);
 765                     }
 766                 }
 767
 768                 //Now we can store the object's metadata without worrying about ghost status entries
 769                 StatusKeeper.StoreInfo(localPath, cloudFile);
 770
 771             }
 772         }
 773
 774         //Download a small file with a single GET operation
 775         private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath,TreeHash serverHash)
 776         {
 777             if (client == null)
 778                 throw new ArgumentNullException("client");
 779             if (cloudFile==null)
 780                 throw new ArgumentNullException("cloudFile");
 781             if (relativeUrl == null)
 782                 throw new ArgumentNullException("relativeUrl");
 783             if (String.IsNullOrWhiteSpace(filePath))
 784                 throw new ArgumentNullException("filePath");
 785             if (!Path.IsPathRooted(filePath))
 786                 throw new ArgumentException("The localPath must be rooted", "filePath");
 787             Contract.EndContractBlock();
 788
 789             var localPath = Pithos.Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath);
 790             //If the file already exists
 791             if (File.Exists(localPath))
 792             {
 793                 //First check with MD5 as this is a small file
 794                 var localMD5 = Signature.CalculateMD5(localPath);
 795                 var cloudHash=serverHash.TopHash.ToHashString();
 796                 if (localMD5==cloudHash)
 797                     return;
 798                 //Then check with a treehash
 799                 var localTreeHash = Signature.CalculateTreeHash(localPath, serverHash.BlockSize, serverHash.BlockHash);
 800                 var localHash = localTreeHash.TopHash.ToHashString();
 801                 if (localHash==cloudHash)
 802                     return;
 803             }
 804
 805             var fileAgent = GetFileAgent(accountInfo);
 806             //Calculate the relative file path for the new file
 807             var relativePath = relativeUrl.RelativeUriToFilePath();
 808             //The file will be stored in a temporary location while downloading with an extension .download
 809             var tempPath = Path.Combine(fileAgent.CachePath, relativePath + ".download");
 810             //Make sure the target folder exists. DownloadFileTask will not create the folder
 811             var tempFolder = Path.GetDirectoryName(tempPath);
 812             if (!Directory.Exists(tempFolder))
 813                 Directory.CreateDirectory(tempFolder);
 814
 815             //Download the object to the temporary location
 816             await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl.ToString(), tempPath);
 817
 818             //Create the local folder if it doesn't exist (necessary for shared objects)
 819             var localFolder = Path.GetDirectoryName(localPath);
 820             if (!Directory.Exists(localFolder))
 821                 Directory.CreateDirectory(localFolder);
 822             //And move it to its actual location once downloading is finished
 823             if (File.Exists(localPath))
 824                 File.Replace(tempPath,localPath,null,true);
 825             else
 826                 File.Move(tempPath,localPath);
 827             //Notify listeners that a local file has changed
 828             StatusNotification.NotifyChangedFile(localPath);
 829
 830
 831         }
 832
 833         //Download a file asynchronously using blocks
 834         public async Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath, TreeHash serverHash)
 835         {
 836             if (client == null)
 837                 throw new ArgumentNullException("client");
 838             if (cloudFile == null)
 839                 throw new ArgumentNullException("cloudFile");
 840             if (relativeUrl == null)
 841                 throw new ArgumentNullException("relativeUrl");
 842             if (String.IsNullOrWhiteSpace(filePath))
 843                 throw new ArgumentNullException("filePath");
 844             if (!Path.IsPathRooted(filePath))
 845                 throw new ArgumentException("The filePath must be rooted", "filePath");
 846             if (serverHash == null)
 847                 throw new ArgumentNullException("serverHash");
 848             Contract.EndContractBlock();
 849
 850            var fileAgent = GetFileAgent(accountInfo);
 851             var localPath = Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath);
 852
 853             //Calculate the relative file path for the new file
 854             var relativePath = relativeUrl.RelativeUriToFilePath();
 855             var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash);
 856
 857
 858
 859             //Calculate the file's treehash
 860             var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash);
 861
 862             //And compare it with the server's hash
 863             var upHashes = serverHash.GetHashesAsStrings();
 864             var localHashes = treeHash.HashDictionary;
 865             for (int i = 0; i < upHashes.Length; i++)
 866             {
 867                 //For every non-matching hash
 868                 var upHash = upHashes[i];
 869                 if (!localHashes.ContainsKey(upHash))
 870                 {
 871                     if (blockUpdater.UseOrphan(i, upHash))
 872                     {
 873                         Log.InfoFormat("[BLOCK GET] ORPHAN FOUND for {0} of {1} for {2}", i, upHashes.Length, localPath);
 874                         continue;
 875                     }
 876                     Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath);
 877                     var start = i*serverHash.BlockSize;
 878                     //To download the last block just pass a null for the end of the range
 879                     long? end = null;
 880                     if (i < upHashes.Length - 1 )
 881                         end= ((i + 1)*serverHash.BlockSize) ;
 882
 883                     //Download the missing block
 884                     var block = await client.GetBlock(cloudFile.Account, cloudFile.Container, relativeUrl, start, end);
 885
 886                     //and store it
 887                     blockUpdater.StoreBlock(i, block);
 888
 889
 890                     Log.InfoFormat("[BLOCK GET] FINISH {0} of {1} for {2}", i, upHashes.Length, localPath);
 891                 }
 892             }
 893
 894             //Want to avoid notifications if no changes were made
 895             var hasChanges = blockUpdater.HasBlocks;
 896             blockUpdater.Commit();
 897
 898             if (hasChanges)
 899                 //Notify listeners that a local file has changed
 900                 StatusNotification.NotifyChangedFile(localPath);
 901
 902             Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath);
 903         }
 904
 905
 906         private async Task UploadCloudFile(CloudAction action)
 907         {
 908             if (action == null)
 909                 throw new ArgumentNullException("action");
 910             Contract.EndContractBlock();
 911
 912             try
 913             {
 914                 var accountInfo = action.AccountInfo;
 915
 916                 var fileInfo = action.LocalFile;
 917
 918                 if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
 919                     return;
 920
 921                 var relativePath = fileInfo.AsRelativeTo(accountInfo.AccountPath);
 922                 if (relativePath.StartsWith(FolderConstants.OthersFolder))
 923                 {
 924                     var parts = relativePath.Split('\\');
 925                     var accountName = parts[1];
 926                     var oldName = accountInfo.UserName;
 927                     var absoluteUri = accountInfo.StorageUri.AbsoluteUri;
 928                     var nameIndex = absoluteUri.IndexOf(oldName);
 929                     var root = absoluteUri.Substring(0, nameIndex);
 930
 931                     accountInfo = new AccountInfo
 932                     {
 933                         UserName = accountName,
 934                         AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 935                         StorageUri = new Uri(root + accountName),
 936                         BlockHash = accountInfo.BlockHash,
 937                         BlockSize = accountInfo.BlockSize,
 938                         Token = accountInfo.Token
 939                     };
 940                 }
 941
 942
 943                 var fullFileName = fileInfo.GetProperCapitalization();
 944                 using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))
 945                 {
 946                     //Abort if the file is already being uploaded or downloaded
 947                     if (gate.Failed)
 948                         return;
 949
 950                     var cloudFile = action.CloudFile;
 951                     var account = cloudFile.Account ?? accountInfo.UserName;
 952
 953                     var client = new CloudFilesClient(accountInfo);
 954                     //Even if GetObjectInfo times out, we can proceed with the upload
 955                     var info = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
 956
 957                     //If this is a read-only file, do not upload changes
 958                     if (info.AllowedTo == "read")
 959                         return;
 960
 961                     //TODO: Check how a directory hash is calculated -> All dirs seem to have the same hash
 962                     if (fileInfo is DirectoryInfo)
 963                     {
 964                         //If the directory doesn't exist the Hash property will be empty
 965                         if (String.IsNullOrWhiteSpace(info.Hash))
 966                             //Go on and create the directory
 967                             await client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName, String.Empty, "application/directory");
 968                     }
 969                     else
 970                     {
 971
 972                         var cloudHash = info.Hash.ToLower();
 973
 974                         var hash = action.LocalHash.Value;
 975                         var topHash = action.TopHash.Value;
 976
 977                         //If the file hashes match, abort the upload
 978                         if (hash == cloudHash || topHash == cloudHash)
 979                         {
 980                             //but store any metadata changes
 981                             StatusKeeper.StoreInfo(fullFileName, info);
 982                             Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
 983                             return;
 984                         }
 985
 986
 987                         //Mark the file as modified while we upload it
 988                         StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
 989                         //And then upload it
 990
 991                         //Upload even small files using the Hashmap. The server may already contain
 992                         //the relevant block
 993
 994                         //First, calculate the tree hash
 995                         var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize,
 996                                                                               accountInfo.BlockHash);
 997
 998                         await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash);
 999                     }
1000                     //If everything succeeds, change the file and overlay status to normal
1001                     StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal);
1002                 }
1003                 //Notify the Shell to update the overlays
1004                 NativeMethods.RaiseChangeNotification(fullFileName);
1005                 StatusNotification.NotifyChangedFile(fullFileName);
1006             }
1007             catch (AggregateException ex)
1008             {
1009                 var exc = ex.InnerException as WebException;
1010                 if (exc == null)
1011                     throw ex.InnerException;
1012                 if (HandleUploadWebException(action, exc))
1013                     return;
1014                 throw;
1015             }
1016             catch (WebException ex)
1017             {
1018                 if (HandleUploadWebException(action, ex))
1019                     return;
1020                 throw;
1021             }
1022             catch (Exception ex)
1023             {
1024                 Log.Error("Unexpected error while uploading file", ex);
1025                 throw;
1026             }
1027
1028         }
1029
1030         private bool IsDeletedFile(CloudAction action)
1031         {
1032             var key = GetFileKey(action.CloudFile);
1033             DateTime entryDate;
1034             if (_deletedFiles.TryGetValue(key, out entryDate))
1035             {
1036                 //If the delete entry was created after this action, abort the action
1037                 if (entryDate > action.Created)
1038                     return true;
1039                 //Otherwise, remove the stale entry
1040                 _deletedFiles.TryRemove(key, out entryDate);
1041             }
1042             return false;
1043         }
1044
1045         private bool HandleUploadWebException(CloudAction action, WebException exc)
1046         {
1047             var response = exc.Response as HttpWebResponse;
1048             if (response == null)
1049                 throw exc;
1050             if (response.StatusCode == HttpStatusCode.Unauthorized)
1051             {
1052                 Log.Error("Not allowed to upload file", exc);
1053                 var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);
1054                 StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal);
1055                 StatusNotification.NotifyChange(message, TraceLevel.Warning);
1056                 return true;
1057             }
1058             return false;
1059         }
1060
1061         public async Task UploadWithHashMap(AccountInfo accountInfo,ObjectInfo cloudFile,FileInfo fileInfo,string url,TreeHash treeHash)
1062         {
1063             if (accountInfo == null)
1064                 throw new ArgumentNullException("accountInfo");
1065             if (cloudFile==null)
1066                 throw new ArgumentNullException("cloudFile");
1067             if (fileInfo == null)
1068                 throw new ArgumentNullException("fileInfo");
1069             if (String.IsNullOrWhiteSpace(url))
1070                 throw new ArgumentNullException(url);
1071             if (treeHash==null)
1072                 throw new ArgumentNullException("treeHash");
1073             if (String.IsNullOrWhiteSpace(cloudFile.Container) )
1074                 throw new ArgumentException("Invalid container","cloudFile");
1075             Contract.EndContractBlock();
1076
1077             var fullFileName = fileInfo.GetProperCapitalization();
1078
1079             var account = cloudFile.Account ?? accountInfo.UserName;
1080             var container = cloudFile.Container ;
1081
1082             var client = new CloudFilesClient(accountInfo);
1083             //Send the hashmap to the server
1084             var missingHashes =  await client.PutHashMap(account, container, url, treeHash);
1085             //If the server returns no missing hashes, we are done
1086             while (missingHashes.Count > 0)
1087             {
1088
1089                 var buffer = new byte[accountInfo.BlockSize];
1090                 foreach (var missingHash in missingHashes)
1091                 {
1092                     //Find the proper block
1093                     var blockIndex = treeHash.HashDictionary[missingHash];
1094                     var offset = blockIndex*accountInfo.BlockSize;
1095
1096                     var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);
1097
1098                     try
1099                     {
1100                         //And upload the block
1101                         await client.PostBlock(account, container, buffer, 0, read);
1102                         Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);
1103                     }
1104                     catch (Exception exc)
1105                     {
1106                         Log.ErrorFormat("[ERROR] uploading block {0} of {1}\n{2}", blockIndex, fullFileName, exc);
1107                     }
1108
1109                 }
1110
1111                 //Repeat until there are no more missing hashes
1112                 missingHashes = await client.PutHashMap(account, container, url, treeHash);
1113             }
1114         }
1115
1116
1117         public void AddAccount(AccountInfo accountInfo)
1118         {
1119             if (!_accounts.Contains(accountInfo))
1120                 _accounts.Add(accountInfo);
1121         }
1122     }
1123
1124
1125
1126
1127 }