code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/NetworkAgent.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel.Composition;
   4 using System.Diagnostics;
   5 using System.Diagnostics.Contracts;
   6 using System.IO;
   7 using System.Linq;
   8 using System.Net;
   9 using System.Text;
  10 using System.Threading;
  11 using System.Threading.Tasks;
  12 using Pithos.Interfaces;
  13 using Pithos.Network;
  14 using log4net;
  15
  16 namespace Pithos.Core.Agents
  17 {
  18     [Export]
  19     public class NetworkAgent
  20     {
  21         private Agent<CloudAction> _agent;
  22
  23         [Import]
  24         public IStatusKeeper StatusKeeper { get; set; }
  25
  26         public IStatusNotification StatusNotification { get; set; }
  27 /*
  28         [Import]
  29         public FileAgent FileAgent {get;set;}
  30 */
  31
  32        /* public int BlockSize { get; set; }
  33         public string BlockHash { get; set; }*/
  34
  35         private static readonly ILog Log = LogManager.GetLogger("NetworkAgent");
  36
  37         private List<AccountInfo> _accounts=new List<AccountInfo>();
  38
  39         public void Start(/*int blockSize, string blockHash*/)
  40         {
  41 /*
  42             if (blockSize<0)
  43                 throw new ArgumentOutOfRangeException("blockSize");
  44             if (String.IsNullOrWhiteSpace(blockHash))
  45                 throw new ArgumentOutOfRangeException("blockHash");
  46             Contract.EndContractBlock();
  47 */
  48
  49 /*
  50             BlockSize = blockSize;
  51             BlockHash = blockHash;
  52 */
  53
  54
  55             _agent = Agent<CloudAction>.Start(inbox =>
  56             {
  57                 Action loop = null;
  58                 loop = () =>
  59                 {
  60                     var message = inbox.Receive();
  61                     var process=message.Then(Process,inbox.CancellationToken);
  62                     inbox.LoopAsync(process, loop);
  63                 };
  64                 loop();
  65             });
  66         }
  67
  68         private Task<object> Process(CloudAction action)
  69         {
  70             if (action == null)
  71                 throw new ArgumentNullException("action");
  72             if (action.AccountInfo==null)
  73                 throw new ArgumentException("The action.AccountInfo is empty","action");
  74             Contract.EndContractBlock();
  75
  76             var accountInfo = action.AccountInfo;
  77
  78             using (log4net.ThreadContext.Stacks["NETWORK"].Push("PROCESS"))
  79             {
  80                 Log.InfoFormat("[ACTION] Start Processing {0}:{1}->{2}", action.Action, action.LocalFile,
  81                                action.CloudFile.Name);
  82
  83                 var localFile = action.LocalFile;
  84                 var cloudFile = action.CloudFile;
  85                 var downloadPath = (cloudFile == null)
  86                                        ? String.Empty
  87                                        : Path.Combine(accountInfo.AccountPath, cloudFile.RelativeUrlToFilePath(accountInfo.UserName));
  88
  89                 try
  90                 {
  91                     var account = action.CloudFile.Account ?? accountInfo.UserName;
  92                     var container = action.CloudFile.Container ?? FolderConstants.PithosContainer;
  93
  94                     switch (action.Action)
  95                     {
  96                         case CloudActionType.UploadUnconditional:
  97                             UploadCloudFile(accountInfo,account, container, localFile, action.LocalHash.Value, action.TopHash.Value);
  98                             break;
  99                         case CloudActionType.DownloadUnconditional:
 100
 101                             DownloadCloudFile(accountInfo, account, container, cloudFile,
 102                                               downloadPath);
 103                             break;
 104                         case CloudActionType.DeleteCloud:
 105                             DeleteCloudFile(accountInfo, account, container, cloudFile.Name);
 106                             break;
 107                         case CloudActionType.RenameCloud:
 108                             var moveAction = (CloudMoveAction)action;
 109                             RenameCloudFile(accountInfo, account, container, moveAction.OldFileName, moveAction.NewPath,
 110                                             moveAction.NewFileName);
 111                             break;
 112                         case CloudActionType.MustSynch:
 113
 114                             if (!File.Exists(downloadPath))
 115                             {
 116                                 DownloadCloudFile(accountInfo, account, container, cloudFile, downloadPath);
 117                             }
 118                             else
 119                             {
 120                                 SyncFiles(accountInfo, action);
 121                             }
 122                             break;
 123                     }
 124                     Log.InfoFormat("[ACTION] End Processing {0}:{1}->{2}", action.Action, action.LocalFile,
 125                                            action.CloudFile.Name);
 126                 }
 127                 catch (OperationCanceledException)
 128                 {
 129                     throw;
 130                 }
 131                 catch (FileNotFoundException exc)
 132                 {
 133                     Log.ErrorFormat("{0} : {1} -> {2}  failed because the file was not found.\n Rescheduling a delete",
 134                         action.Action, action.LocalFile, action.CloudFile, exc);
 135                     Post(new CloudDeleteAction(accountInfo,action.CloudFile,action.FileState));
 136                 }
 137                 catch (Exception exc)
 138                 {
 139                     Log.ErrorFormat("[REQUEUE] {0} : {1} -> {2} due to exception\r\n{3}",
 140                                      action.Action, action.LocalFile, action.CloudFile, exc);
 141
 142                     _agent.Post(action);
 143                 }
 144                 return CompletedTask<object>.Default;
 145             }
 146         }
 147
 148         private void SyncFiles(AccountInfo accountInfo,CloudAction action)
 149         {
 150             if (accountInfo == null)
 151                 throw new ArgumentNullException("accountInfo");
 152             if (action==null)
 153                 throw new ArgumentNullException("action");
 154             if (action.LocalFile==null)
 155                 throw new ArgumentException("The action's local file is not specified","action");
 156             if (!Path.IsPathRooted(action.LocalFile.FullName))
 157                 throw new ArgumentException("The action's local file path must be absolute","action");
 158             if (action.CloudFile== null)
 159                 throw new ArgumentException("The action's cloud file is not specified", "action");
 160             Contract.EndContractBlock();
 161
 162             var localFile = action.LocalFile;
 163             var cloudFile = action.CloudFile;
 164             var downloadPath=action.LocalFile.FullName.ToLower();
 165
 166             var account = cloudFile.Account;
 167             //Use "pithos" by default if no container is specified
 168             var container = cloudFile.Container ?? FolderConstants.PithosContainer;
 169
 170             var cloudUri = new Uri(cloudFile.Name, UriKind.Relative);
 171             var cloudHash = cloudFile.Hash.ToLower();
 172             var localHash = action.LocalHash.Value.ToLower();
 173             var topHash = action.TopHash.Value.ToLower();
 174
 175             //Not enough to compare only the local hashes, also have to compare the tophashes
 176
 177             //If any of the hashes match, we are done
 178             if ((cloudHash == localHash || cloudHash == topHash))
 179             {
 180                 Log.InfoFormat("Skipping {0}, hashes match",downloadPath);
 181                 return;
 182             }
 183
 184             //The hashes DON'T match. We need to sync
 185             var lastLocalTime = localFile.LastWriteTime;
 186             var lastUpTime = cloudFile.Last_Modified;
 187
 188             //If the local file is newer upload it
 189             if (lastUpTime <= lastLocalTime)
 190             {
 191                 //It probably means it was changed while the app was down
 192                 UploadCloudFile(accountInfo,account, container, localFile, action.LocalHash.Value,
 193                                 action.TopHash.Value);
 194             }
 195             else
 196             {
 197                 //It the cloud file has a later date, it was modified by another user or computer.
 198                 //We need to check the local file's status
 199                 var status = StatusKeeper.GetFileStatus(downloadPath);
 200                 switch (status)
 201                 {
 202                     case FileStatus.Unchanged:
 203                         //If the local file's status is Unchanged, we can go on and download the newer cloud file
 204                         DownloadCloudFile(accountInfo,account, container,cloudFile,downloadPath);
 205                         break;
 206                     case FileStatus.Modified:
 207                         //If the local file is Modified, we may have a conflict. In this case we should mark the file as Conflict
 208                         //We can't ensure that a file modified online since the last time will appear as Modified, unless we
 209                         //index all files before we start listening.
 210                     case FileStatus.Created:
 211                         //If the local file is Created, it means that the local and cloud files aren't related,
 212                         // yet they have the same name.
 213
 214                         //In both cases we must mark the file as in conflict
 215                         ReportConflict(downloadPath);
 216                         break;
 217                     default:
 218                         //Other cases should never occur. Mark them as Conflict as well but log a warning
 219                         ReportConflict(downloadPath);
 220                         Log.WarnFormat("Unexcepted status {0} for file {1}->{2}", status,
 221                                        downloadPath, action.CloudFile.Name);
 222                         break;
 223                 }
 224             }
 225         }
 226
 227         private void ReportConflict(string downloadPath)
 228         {
 229             if (String.IsNullOrWhiteSpace(downloadPath))
 230                 throw new ArgumentNullException("downloadPath");
 231             Contract.EndContractBlock();
 232
 233             StatusKeeper.SetFileOverlayStatus(downloadPath, FileOverlayStatus.Conflict);
 234             var message = String.Format("Conflict detected for file {0}", downloadPath);
 235             Log.Warn(message);
 236             StatusNotification.NotifyChange(message, TraceLevel.Warning);
 237         }
 238
 239 /*
 240         private Task<object> Process(CloudMoveAction action)
 241         {
 242             if (action == null)
 243                 throw new ArgumentNullException("action");
 244             Contract.EndContractBlock();
 245
 246             Log.InfoFormat("[ACTION] Start Processing {0}:{1}->{2}", action.Action, action.LocalFile, action.CloudFile.Name);
 247
 248             try
 249             {
 250                 RenameCloudFile(action.OldFileName, action.NewPath, action.NewFileName);
 251                 Log.InfoFormat("[ACTION] End Processing {0}:{1}->{2}", action.Action, action.LocalFile, action.CloudFile.Name);
 252             }
 253             catch (OperationCanceledException)
 254             {
 255                 throw;
 256             }
 257             catch (Exception exc)
 258             {
 259                 Log.ErrorFormat("[REQUEUE] {0} : {1} -> {2} due to exception\r\n{3}",
 260                                 action.Action, action.OldFileName, action.NewFileName, exc);
 261
 262                 _agent.Post(action);
 263             }
 264             return CompletedTask<object>.Default;
 265         }
 266 */
 267
 268
 269         public void Post(CloudAction cloudAction)
 270         {
 271             if (cloudAction == null)
 272                 throw new ArgumentNullException("cloudAction");
 273             if (cloudAction.AccountInfo==null)
 274                 throw new ArgumentException("The CloudAction.AccountInfo is empty","cloudAction");
 275             Contract.EndContractBlock();
 276
 277             //If the action targets a local file, add a treehash calculation
 278             if (cloudAction.LocalFile != null)
 279             {
 280                 var accountInfo = cloudAction.AccountInfo;
 281                 if (cloudAction.LocalFile.Length>accountInfo.BlockSize)
 282                     cloudAction.TopHash = new Lazy<string>(() => Signature.CalculateTreeHashAsync(cloudAction.LocalFile,
 283                                     accountInfo.BlockSize, accountInfo.BlockHash).Result
 284                                      .TopHash.ToHashString());
 285                 else
 286                 {
 287                     cloudAction.TopHash=new Lazy<string>(()=> cloudAction.LocalHash.Value);
 288                 }
 289
 290             }
 291             _agent.Post(cloudAction);
 292         }
 293
 294         class ObjectInfoByNameComparer:IEqualityComparer<ObjectInfo>
 295         {
 296             public bool Equals(ObjectInfo x, ObjectInfo y)
 297             {
 298                 return x.Name.Equals(y.Name,StringComparison.InvariantCultureIgnoreCase);
 299             }
 300
 301             public int GetHashCode(ObjectInfo obj)
 302             {
 303                 return obj.Name.ToLower().GetHashCode();
 304             }
 305         }
 306
 307
 308
 309         //Remote files are polled periodically. Any changes are processed
 310         public Task ProcessRemoteFiles(DateTime? since=null)
 311         {
 312             return Task<Task>.Factory.StartNewDelayed(10000, () =>
 313             {
 314                 using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))
 315                 {
 316                     //Next time we will check for all changes since the current check minus 1 second
 317                     //This is done to ensure there are no discrepancies due to clock differences
 318                     DateTime nextSince = DateTime.Now.AddSeconds(-1);
 319
 320                     var tasks=from accountInfo in _accounts
 321                               select ProcessAccountFiles(accountInfo, since);
 322                     var process=Task.Factory.Iterate(tasks);
 323
 324                     return process.ContinueWith(t =>
 325                     {
 326                         if (t.IsFaulted)
 327                         {
 328                             Log.Error("Error while processing accounts");
 329                             t.Exception.Handle(exc=>
 330                                                    {
 331                                                        Log.Error("Details:", exc);
 332                                                        return true;
 333                                                    });
 334                         }
 335                         ProcessRemoteFiles(nextSince);
 336                     });
 337                 }
 338             });
 339         }
 340
 341         public Task ProcessAccountFiles(AccountInfo accountInfo,DateTime? since=null)
 342         {
 343             if (accountInfo==null)
 344                 throw new ArgumentNullException("accountInfo");
 345             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))
 346                 throw new ArgumentException("The AccountInfo.AccountPath is empty","accountInfo");
 347             Contract.EndContractBlock();
 348
 349             using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))
 350             {
 351                 Log.Info("Scheduled");
 352                 var client=new CloudFilesClient(accountInfo);
 353
 354                 //Get the list of server objects changed since the last check
 355                 var listObjects = Task<IList<ObjectInfo>>.Factory.StartNew(() =>
 356                                 client.ListObjects(accountInfo.UserName, FolderConstants.PithosContainer, since));
 357                 //Get the list of deleted objects since the last check
 358                 var listTrash = Task<IList<ObjectInfo>>.Factory.StartNew(() =>
 359                                 client.ListObjects(accountInfo.UserName, FolderConstants.TrashContainer, since));
 360
 361                 var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(() =>
 362                                 client.ListSharedObjects());
 363
 364                 var listAll = Task.Factory.TrackedSequence(
 365                     () => listObjects,
 366                     () => listTrash,
 367                     () => listShared);
 368
 369
 370
 371                 var enqueueFiles = listAll.ContinueWith(task =>
 372                 {
 373                     if (task.IsFaulted)
 374                     {
 375                         //ListObjects failed at this point, need to reschedule
 376                         Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {0}", accountInfo.UserName,task.Exception);
 377                         return;
 378                     }
 379                     using (log4net.ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))
 380                     {
 381                         var remoteObjects = ((Task<IList<ObjectInfo>>) task.Result[0]).Result;
 382                         var trashObjects = ((Task<IList<ObjectInfo>>) task.Result[1]).Result;
 383                         var sharedObjects = ((Task<IList<ObjectInfo>>) task.Result[2]).Result;
 384
 385                         //Items with the same name, hash may be both in the container and the trash
 386                         //Don't delete items that exist in the container
 387                         var realTrash = from trash in trashObjects
 388                                         where !remoteObjects.Any(info => info.Hash == trash.Hash)
 389                                         select trash;
 390                         ProcessDeletedFiles(accountInfo,realTrash);
 391
 392
 393                         var remote = from info in remoteObjects.Union(sharedObjects)
 394                                      let name = info.Name
 395                                      where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&
 396                                            !name.StartsWith("fragments/", StringComparison.InvariantCultureIgnoreCase)
 397                                      select info;
 398
 399                         //Create a list of actions from the remote files
 400                         var allActions = ObjectsToActions(accountInfo,remote);
 401
 402                         //And remove those that are already being processed by the agent
 403                         var distinctActions = allActions
 404                             .Except(_agent.GetEnumerable(), new PithosMonitor.LocalFileComparer())
 405                             .ToList();
 406
 407                         //Queue all the actions
 408                         foreach (var message in distinctActions)
 409                         {
 410                             Post(message);
 411                         }
 412
 413                         //Report the number of new files
 414                         var remoteCount = distinctActions.Count(action=>
 415                             action.Action==CloudActionType.DownloadUnconditional);
 416                         if ( remoteCount > 0)
 417                             StatusNotification.NotifyChange(String.Format("Processing {0} new files", remoteCount));
 418
 419                         Log.Info("[LISTENER] End Processing");
 420                     }
 421                 });
 422
 423                 var log = enqueueFiles.ContinueWith(t =>
 424                 {
 425                     if (t.IsFaulted)
 426                     {
 427                         Log.Error("[LISTENER] Exception", t.Exception);
 428                     }
 429                     else
 430                     {
 431                         Log.Info("[LISTENER] Finished");
 432                     }
 433                 });
 434                 return log;
 435             }
 436         }
 437
 438         //Creates an appropriate action for each server file
 439         private IEnumerable<CloudAction> ObjectsToActions(AccountInfo accountInfo,IEnumerable<ObjectInfo> remote)
 440         {
 441             if (remote==null)
 442                 throw new ArgumentNullException();
 443             Contract.EndContractBlock();
 444             var fileAgent = GetFileAgent(accountInfo);
 445
 446             //In order to avoid multiple iterations over the files, we iterate only once
 447             //over the remote files
 448             foreach (var objectInfo in remote)
 449             {
 450                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);
 451                 //and remove any matching objects from the list, adding them to the commonObjects list
 452
 453                 if (fileAgent.Exists(relativePath))
 454                 {
 455                     var localFile = fileAgent.GetFileInfo(relativePath);
 456                     var state = FileState.FindByFilePath(localFile.FullName);
 457                     //Common files should be checked on a per-case basis to detect differences, which is newer
 458
 459                     yield return new CloudAction(accountInfo,CloudActionType.MustSynch,
 460                                                    localFile, objectInfo, state, accountInfo.BlockSize,
 461                                                    accountInfo.BlockHash);
 462                 }
 463                 else
 464                 {
 465                     //If there is no match we add them to the localFiles list
 466                     //but only if the file is not marked for deletion
 467                     var targetFile = Path.Combine(accountInfo.AccountPath, relativePath);
 468                     var fileStatus = StatusKeeper.GetFileStatus(targetFile);
 469                     if (fileStatus != FileStatus.Deleted)
 470                     {
 471                         //Remote files should be downloaded
 472                         yield return new CloudDownloadAction(accountInfo,objectInfo);
 473                     }
 474                 }
 475             }
 476         }
 477
 478         private static FileAgent GetFileAgent(AccountInfo accountInfo)
 479         {
 480             return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);
 481         }
 482
 483         private void ProcessDeletedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects)
 484         {
 485             var fileAgent = GetFileAgent(accountInfo);
 486             foreach (var trashObject in trashObjects)
 487             {
 488                 var relativePath = trashObject.RelativeUrlToFilePath(accountInfo.UserName);
 489                 //and remove any matching objects from the list, adding them to the commonObjects list
 490                 fileAgent.Delete(relativePath);
 491             }
 492         }
 493
 494
 495         private void RenameCloudFile(AccountInfo accountInfo,string account, string container,string oldFileName, string newPath, string newFileName)
 496         {
 497             if (accountInfo==null)
 498                 throw new ArgumentNullException("accountInfo");
 499             if (String.IsNullOrWhiteSpace(account))
 500                 throw new ArgumentNullException("account");
 501             if (String.IsNullOrWhiteSpace(container))
 502                 throw new ArgumentNullException("container");
 503             if (String.IsNullOrWhiteSpace(oldFileName))
 504                 throw new ArgumentNullException("oldFileName");
 505             if (String.IsNullOrWhiteSpace(oldFileName))
 506                 throw new ArgumentNullException("newPath");
 507             if (String.IsNullOrWhiteSpace(oldFileName))
 508                 throw new ArgumentNullException("newFileName");
 509             Contract.EndContractBlock();
 510             //The local file is already renamed
 511             this.StatusKeeper.SetFileOverlayStatus(newPath, FileOverlayStatus.Modified);
 512
 513             var client = new CloudFilesClient(accountInfo);
 514             client.MoveObject(account, container, oldFileName, container, newFileName);
 515
 516             this.StatusKeeper.SetFileStatus(newPath, FileStatus.Unchanged);
 517             this.StatusKeeper.SetFileOverlayStatus(newPath, FileOverlayStatus.Normal);
 518             NativeMethods.RaiseChangeNotification(newPath);
 519         }
 520
 521         private void DeleteCloudFile(AccountInfo accountInfo, string account, string container, string fileName)
 522         {
 523             if (accountInfo == null)
 524                 throw new ArgumentNullException("accountInfo");
 525             if (String.IsNullOrWhiteSpace(account))
 526                 throw new ArgumentNullException("account");
 527             if (String.IsNullOrWhiteSpace(container))
 528                 throw new ArgumentNullException("container");
 529             if (String.IsNullOrWhiteSpace(container))
 530                 throw new ArgumentNullException("container");
 531
 532             if (String.IsNullOrWhiteSpace(fileName))
 533                 throw new ArgumentNullException("fileName");
 534             if (Path.IsPathRooted(fileName))
 535                 throw new ArgumentException("The fileName should not be rooted","fileName");
 536             Contract.EndContractBlock();
 537
 538             var fileAgent = GetFileAgent(accountInfo);
 539
 540             using ( log4net.ThreadContext.Stacks["DeleteCloudFile"].Push("Delete"))
 541             {
 542                 var info = fileAgent.GetFileInfo(fileName);
 543                 var fullPath = info.FullName.ToLower();
 544                 this.StatusKeeper.SetFileOverlayStatus(fullPath, FileOverlayStatus.Modified);
 545
 546                 var client = new CloudFilesClient(accountInfo);
 547                 client.DeleteObject(account, container, fileName);
 548
 549                 this.StatusKeeper.ClearFileStatus(fullPath);
 550             }
 551         }
 552
 553         //Download a file.
 554         private void DownloadCloudFile(AccountInfo accountInfo, string account, string container,ObjectInfo cloudFile , string localPath)
 555         {
 556             if (accountInfo == null)
 557                 throw new ArgumentNullException("accountInfo");
 558             if (String.IsNullOrWhiteSpace(account))
 559                 throw new ArgumentNullException("account");
 560             if (String.IsNullOrWhiteSpace(container))
 561                 throw new ArgumentNullException("container");
 562             if (cloudFile == null)
 563                 throw new ArgumentNullException("cloudFile");
 564             if (String.IsNullOrWhiteSpace(localPath))
 565                 throw new ArgumentNullException("localPath");
 566             if (!Path.IsPathRooted(localPath))
 567                 throw new ArgumentException("The localPath must be rooted", "localPath");
 568             Contract.EndContractBlock();
 569
 570             Debug.Assert(cloudFile.Account==account);
 571             Debug.Assert(cloudFile.Container == container);
 572
 573             var download=Task.Factory.Iterate(DownloadIterator(accountInfo,account,container, cloudFile, localPath));
 574             download.Wait();
 575         }
 576
 577         private IEnumerable<Task> DownloadIterator(AccountInfo accountInfo, string account, string container, ObjectInfo cloudFile, string localPath)
 578         {
 579             if (accountInfo == null)
 580                 throw new ArgumentNullException("accountInfo");
 581             if (String.IsNullOrWhiteSpace(account))
 582                 throw new ArgumentNullException("account");
 583             if (String.IsNullOrWhiteSpace(container))
 584                 throw new ArgumentNullException("container");
 585             if (cloudFile==null)
 586                 throw new ArgumentNullException("cloudFile");
 587             if (String.IsNullOrWhiteSpace(localPath))
 588                 throw new ArgumentNullException("localPath");
 589             if (!Path.IsPathRooted(localPath))
 590                 throw new ArgumentException("The localPath must be rooted", "localPath");
 591             Contract.EndContractBlock();
 592
 593             Uri relativeUrl = new Uri(cloudFile.Name, UriKind.Relative);
 594
 595             var url = relativeUrl.ToString();
 596             if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase))
 597                 yield break;
 598
 599             //Are we already downloading or uploading the file?
 600             using (var gate=NetworkGate.Acquire(localPath, NetworkOperation.Downloading))
 601             {
 602                 if (gate.Failed)
 603                     yield break;
 604                 //The file's hashmap will be stored in the same location with the extension .hashmap
 605                 //var hashPath = Path.Combine(FileAgent.FragmentsPath, relativePath + ".hashmap");
 606
 607                 var client = new CloudFilesClient(accountInfo);
 608                 //Retrieve the hashmap from the server
 609                 var getHashMap = client.GetHashMap(account, container, url);
 610                 yield return getHashMap;
 611
 612                 var serverHash=getHashMap.Result;
 613                 //If it's a small file
 614                 var downloadTask=(serverHash.Hashes.Count == 1 )
 615                     //Download it in one go
 616                     ? DownloadEntireFile(accountInfo,client, account, container, relativeUrl, localPath,serverHash)
 617                     //Otherwise download it block by block
 618                     : DownloadWithBlocks(accountInfo,client, account, container, relativeUrl, localPath, serverHash);
 619
 620                 yield return downloadTask;
 621
 622                 if (cloudFile.AllowedTo == "read")
 623                 {
 624                     var attributes=File.GetAttributes(localPath);
 625                     File.SetAttributes(localPath,attributes|FileAttributes.ReadOnly);
 626                 }
 627                 //Retrieve the object's metadata
 628                 var info=client.GetObjectInfo(account, container, url);
 629                 Debug.Assert(cloudFile==info);
 630                 //And store it
 631                 StatusKeeper.StoreInfo(localPath, info);
 632
 633                 //Notify listeners that a local file has changed
 634                 StatusNotification.NotifyChangedFile(localPath);
 635
 636             }
 637         }
 638
 639         //Download a small file with a single GET operation
 640         private Task DownloadEntireFile(AccountInfo accountInfo, CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath,TreeHash serverHash)
 641         {
 642             if (client == null)
 643                 throw new ArgumentNullException("client");
 644             if (String.IsNullOrWhiteSpace(account))
 645                 throw new ArgumentNullException("account");
 646             if (String.IsNullOrWhiteSpace(container))
 647                 throw new ArgumentNullException("container");
 648             if (relativeUrl == null)
 649                 throw new ArgumentNullException("relativeUrl");
 650             if (String.IsNullOrWhiteSpace(localPath))
 651                 throw new ArgumentNullException("localPath");
 652             if (!Path.IsPathRooted(localPath))
 653                 throw new ArgumentException("The localPath must be rooted", "localPath");
 654             Contract.EndContractBlock();
 655
 656             //If the file already exists
 657             if (File.Exists(localPath))
 658             {
 659                 //First check with MD5 as this is a small file
 660                 var localMD5 = Signature.CalculateMD5(localPath);
 661                 var cloudHash=serverHash.TopHash.ToHashString();
 662                 if (localMD5==cloudHash)
 663                     return CompletedTask.Default;
 664                 //Then check with a treehash
 665                 var localTreeHash = Signature.CalculateTreeHash(localPath, serverHash.BlockSize, serverHash.BlockHash);
 666                 var localHash = localTreeHash.TopHash.ToHashString();
 667                 if (localHash==cloudHash)
 668                     return CompletedTask.Default;
 669             }
 670
 671             var fileAgent = GetFileAgent(accountInfo);
 672             //Calculate the relative file path for the new file
 673             var relativePath = relativeUrl.RelativeUriToFilePath();
 674             //The file will be stored in a temporary location while downloading with an extension .download
 675             var tempPath = Path.Combine(fileAgent.FragmentsPath, relativePath + ".download");
 676             //Make sure the target folder exists. DownloadFileTask will not create the folder
 677             var tempFolder = Path.GetDirectoryName(tempPath);
 678             if (!Directory.Exists(tempFolder))
 679                 Directory.CreateDirectory(tempFolder);
 680
 681             //Download the object to the temporary location
 682             var getObject = client.GetObject(account, container, relativeUrl.ToString(), tempPath).ContinueWith(t =>
 683             {
 684                 t.PropagateExceptions();
 685                 //Create the local folder if it doesn't exist (necessary for shared objects)
 686                 var localFolder = Path.GetDirectoryName(localPath);
 687                 if (!Directory.Exists(localFolder))
 688                     Directory.CreateDirectory(localFolder);
 689                 //And move it to its actual location once downloading is finished
 690                 if (File.Exists(localPath))
 691                     File.Replace(tempPath,localPath,null,true);
 692                 else
 693                     File.Move(tempPath,localPath);
 694             });
 695             return getObject;
 696         }
 697
 698         //Download a file asynchronously using blocks
 699         public Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash)
 700         {
 701             if (client == null)
 702                 throw new ArgumentNullException("client");
 703             if (String.IsNullOrWhiteSpace(account))
 704                 throw new ArgumentNullException("account");
 705             if (String.IsNullOrWhiteSpace(container))
 706                 throw new ArgumentNullException("container");
 707             if (relativeUrl == null)
 708                 throw new ArgumentNullException("relativeUrl");
 709             if (String.IsNullOrWhiteSpace(localPath))
 710                 throw new ArgumentNullException("localPath");
 711             if (!Path.IsPathRooted(localPath))
 712                 throw new ArgumentException("The localPath must be rooted", "localPath");
 713             if (serverHash == null)
 714                 throw new ArgumentNullException("serverHash");
 715             Contract.EndContractBlock();
 716
 717             return Task.Factory.Iterate(BlockDownloadIterator(accountInfo,client,account,container, relativeUrl, localPath, serverHash));
 718         }
 719
 720         private IEnumerable<Task> BlockDownloadIterator(AccountInfo accountInfo,CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash)
 721         {
 722             if (client == null)
 723                 throw new ArgumentNullException("client");
 724             if (String.IsNullOrWhiteSpace(account))
 725                 throw new ArgumentNullException("account");
 726             if (String.IsNullOrWhiteSpace(container))
 727                 throw new ArgumentNullException("container");
 728             if (relativeUrl == null)
 729                 throw new ArgumentNullException("relativeUrl");
 730             if (String.IsNullOrWhiteSpace(localPath))
 731                 throw new ArgumentNullException("localPath");
 732             if (!Path.IsPathRooted(localPath))
 733                 throw new ArgumentException("The localPath must be rooted", "localPath");
 734             if(serverHash==null)
 735                 throw new ArgumentNullException("serverHash");
 736             Contract.EndContractBlock();
 737
 738             var fileAgent = GetFileAgent(accountInfo);
 739
 740             //Calculate the relative file path for the new file
 741             var relativePath = relativeUrl.RelativeUriToFilePath();
 742             var blockUpdater = new BlockUpdater(fileAgent.FragmentsPath, localPath, relativePath, serverHash);
 743
 744
 745
 746             //Calculate the file's treehash
 747             var calcHash = Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize,serverHash.BlockHash);
 748             yield return calcHash;
 749             var treeHash = calcHash.Result;
 750
 751             //And compare it with the server's hash
 752             var upHashes = serverHash.GetHashesAsStrings();
 753             var localHashes = treeHash.HashDictionary;
 754             for (int i = 0; i < upHashes.Length; i++)
 755             {
 756                 //For every non-matching hash
 757                 var upHash = upHashes[i];
 758                 if (!localHashes.ContainsKey(upHash))
 759                 {
 760                     if (blockUpdater.UseOrphan(i, upHash))
 761                     {
 762                         Log.InfoFormat("[BLOCK GET] ORPHAN FOUND for {0} of {1} for {2}", i, upHashes.Length, localPath);
 763                         continue;
 764                     }
 765                     Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath);
 766                     var start = i*serverHash.BlockSize;
 767                     //To download the last block just pass a null for the end of the range
 768                     long? end = null;
 769                     if (i < upHashes.Length - 1 )
 770                         end= ((i + 1)*serverHash.BlockSize) ;
 771
 772                     //Download the missing block
 773                     var getBlock = client.GetBlock(account, container, relativeUrl, start, end);
 774                     yield return getBlock;
 775                     var block = getBlock.Result;
 776
 777                     //and store it
 778                     yield return blockUpdater.StoreBlock(i, block);
 779
 780
 781                     Log.InfoFormat("[BLOCK GET] FINISH {0} of {1} for {2}", i, upHashes.Length, localPath);
 782                 }
 783             }
 784
 785             blockUpdater.Commit();
 786             Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath);
 787         }
 788
 789
 790         private void UploadCloudFile(AccountInfo accountInfo, string account, string container, FileInfo fileInfo, string hash, string topHash)
 791         {
 792             if (accountInfo == null)
 793                 throw new ArgumentNullException("accountInfo");
 794             if (String.IsNullOrWhiteSpace(account))
 795                 throw new ArgumentNullException("account");
 796             if (String.IsNullOrWhiteSpace(container))
 797                 throw new ArgumentNullException("container");
 798             if (fileInfo == null)
 799                 throw new ArgumentNullException("fileInfo");
 800             if (String.IsNullOrWhiteSpace(hash))
 801                 throw new ArgumentNullException("hash");
 802             if (topHash == null)
 803                 throw new ArgumentNullException("topHash");
 804             Contract.EndContractBlock();
 805
 806             var upload = Task.Factory.Iterate(UploadIterator(accountInfo,account,container,fileInfo, hash.ToLower(), topHash.ToLower()));
 807             upload.Wait();
 808         }
 809
 810         private IEnumerable<Task> UploadIterator(AccountInfo accountInfo, string account, string container, FileInfo fileInfo, string hash, string topHash)
 811         {
 812             if (accountInfo == null)
 813                 throw new ArgumentNullException("accountInfo");
 814             if (String.IsNullOrWhiteSpace(account))
 815                 throw new ArgumentNullException("account");
 816             if (String.IsNullOrWhiteSpace(container))
 817                 throw new ArgumentNullException("container");
 818             if (fileInfo == null)
 819                 throw new ArgumentNullException("fileInfo");
 820             if (String.IsNullOrWhiteSpace(hash))
 821                 throw new ArgumentNullException("hash");
 822             if (topHash == null)
 823                 throw new ArgumentNullException("topHash");
 824             Contract.EndContractBlock();
 825
 826             if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
 827                 yield break;
 828
 829             var url = fileInfo.AsRelativeUrlTo(accountInfo.AccountPath);
 830
 831             var fullFileName = fileInfo.FullName;
 832             using(var gate=NetworkGate.Acquire(fullFileName,NetworkOperation.Uploading))
 833             {
 834                 //Abort if the file is already being uploaded or downloaded
 835                 if (gate.Failed)
 836                     yield break;
 837
 838                 var client = new CloudFilesClient(accountInfo);
 839                 //Even if GetObjectInfo times out, we can proceed with the upload
 840                 var info = client.GetObjectInfo(account, container, url);
 841                 var cloudHash = info.Hash.ToLower();
 842
 843                 //If the file hashes match, abort the upload
 844                 if (hash == cloudHash  || topHash ==cloudHash)
 845                 {
 846                     //but store any metadata changes
 847                     this.StatusKeeper.StoreInfo(fullFileName, info);
 848                     Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
 849                     yield break;
 850                 }
 851
 852                 if (info.AllowedTo=="read")
 853                     yield break;
 854
 855                 //Mark the file as modified while we upload it
 856                 StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
 857                 //And then upload it
 858
 859                 //If the file is larger than the block size, try a hashmap PUT
 860                 if (fileInfo.Length > accountInfo.BlockSize )
 861                 {
 862                     //To upload using a hashmap
 863                     //First, calculate the tree hash
 864                     var treeHash = Signature.CalculateTreeHashAsync(fileInfo.FullName, accountInfo.BlockSize,
 865                         accountInfo.BlockHash);
 866                     yield return treeHash;
 867
 868                     yield return Task.Factory.Iterate(UploadWithHashMap(accountInfo,account,container,fileInfo,url,treeHash));
 869
 870                 }
 871                 else
 872                 {
 873                     //Otherwise do a regular PUT
 874                     yield return client.PutObject(account, container, url, fullFileName, hash);
 875                 }
 876                 //If everything succeeds, change the file and overlay status to normal
 877                 this.StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal);
 878             }
 879             //Notify the Shell to update the overlays
 880             NativeMethods.RaiseChangeNotification(fullFileName);
 881             StatusNotification.NotifyChangedFile(fullFileName);
 882         }
 883
 884         public IEnumerable<Task> UploadWithHashMap(AccountInfo accountInfo,string account,string container,FileInfo fileInfo,string url,Task<TreeHash> treeHash)
 885         {
 886             if (accountInfo == null)
 887                 throw new ArgumentNullException("accountInfo");
 888             if (String.IsNullOrWhiteSpace(account))
 889                 throw new ArgumentNullException("account");
 890             if (String.IsNullOrWhiteSpace(container))
 891                 throw new ArgumentNullException("container");
 892             if (fileInfo == null)
 893                 throw new ArgumentNullException("fileInfo");
 894             if (String.IsNullOrWhiteSpace(url))
 895                 throw new ArgumentNullException(url);
 896             if (treeHash==null)
 897                 throw new ArgumentNullException("treeHash");
 898             Contract.EndContractBlock();
 899
 900             var fullFileName = fileInfo.FullName;
 901
 902             var client = new CloudFilesClient(accountInfo);
 903             //Send the hashmap to the server
 904             var hashPut = client.PutHashMap(account, container, url, treeHash.Result);
 905             yield return hashPut;
 906
 907             var missingHashes = hashPut.Result;
 908             //If the server returns no missing hashes, we are done
 909             while (missingHashes.Count > 0)
 910             {
 911
 912                 var buffer = new byte[accountInfo.BlockSize];
 913                 foreach (var missingHash in missingHashes)
 914                 {
 915                     //Find the proper block
 916                     var blockIndex = treeHash.Result.HashDictionary[missingHash];
 917                     var offset = blockIndex*accountInfo.BlockSize;
 918
 919                     var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);
 920
 921                     //And upload the block
 922                     var postBlock = client.PostBlock(account, container, buffer, 0, read);
 923
 924                     //We have to handle possible exceptions in a continuation because
 925                     //*yield return* can't appear inside a try block
 926                     yield return postBlock.ContinueWith(t =>
 927                         t.ReportExceptions(
 928                             exc => Log.ErrorFormat("[ERROR] uploading block {0} of {1}\n{2}", blockIndex, fullFileName, exc),
 929                             ()=>Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex,fullFileName)));
 930                 }
 931
 932                 //Repeat until there are no more missing hashes
 933                 hashPut = client.PutHashMap(account, container, url, treeHash.Result);
 934                 yield return hashPut;
 935                 missingHashes = hashPut.Result;
 936             }
 937         }
 938
 939
 940         public void AddAccount(AccountInfo accountInfo)
 941         {
 942             if (!_accounts.Contains(accountInfo))
 943                 _accounts.Add(accountInfo);
 944         }
 945     }
 946
 947
 948
 949
 950 }