code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/NetworkAgent.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel;
   4 using System.ComponentModel.Composition;
   5 using System.Diagnostics;
   6 using System.Diagnostics.Contracts;
   7 using System.IO;
   8 using System.Linq;
   9 using System.Net;
  10 using System.Text;
  11 using System.Threading;
  12 using System.Threading.Tasks;
  13 using Pithos.Interfaces;
  14 using Pithos.Network;
  15 using log4net;
  16
  17 namespace Pithos.Core.Agents
  18 {
  19     [Export]
  20     public class NetworkAgent
  21     {
  22         private Agent<CloudAction> _agent;
  23
  24         [Import]
  25         public IStatusKeeper StatusKeeper { get; set; }
  26
  27         public IStatusNotification StatusNotification { get; set; }
  28 /*
  29         [Import]
  30         public FileAgent FileAgent {get;set;}
  31 */
  32
  33        /* public int BlockSize { get; set; }
  34         public string BlockHash { get; set; }*/
  35
  36         private static readonly ILog Log = LogManager.GetLogger("NetworkAgent");
  37
  38         private List<AccountInfo> _accounts=new List<AccountInfo>();
  39
  40         public void Start(/*int blockSize, string blockHash*/)
  41         {
  42 /*
  43             if (blockSize<0)
  44                 throw new ArgumentOutOfRangeException("blockSize");
  45             if (String.IsNullOrWhiteSpace(blockHash))
  46                 throw new ArgumentOutOfRangeException("blockHash");
  47             Contract.EndContractBlock();
  48 */
  49
  50 /*
  51             BlockSize = blockSize;
  52             BlockHash = blockHash;
  53 */
  54
  55
  56             _agent = Agent<CloudAction>.Start(inbox =>
  57             {
  58                 Action loop = null;
  59                 loop = () =>
  60                 {
  61                     var message = inbox.Receive();
  62                     var process=message.Then(Process,inbox.CancellationToken);
  63                     inbox.LoopAsync(process, loop);
  64                 };
  65                 loop();
  66             });
  67         }
  68
  69         private async Task Process(CloudAction action)
  70         {
  71             if (action == null)
  72                 throw new ArgumentNullException("action");
  73             if (action.AccountInfo==null)
  74                 throw new ArgumentException("The action.AccountInfo is empty","action");
  75             Contract.EndContractBlock();
  76
  77             var accountInfo = action.AccountInfo;
  78
  79             using (log4net.ThreadContext.Stacks["NETWORK"].Push("PROCESS"))
  80             {
  81                 Log.InfoFormat("[ACTION] Start Processing {0}", action);
  82
  83                 var localFile = action.LocalFile;
  84                 var cloudFile = action.CloudFile;
  85                 var downloadPath = action.GetDownloadPath();
  86
  87                 try
  88                 {
  89
  90                     switch (action.Action)
  91                     {
  92                         case CloudActionType.UploadUnconditional:
  93                             await UploadCloudFile(action);
  94                             break;
  95                         case CloudActionType.DownloadUnconditional:
  96
  97                             await DownloadCloudFile(accountInfo,  cloudFile,downloadPath);
  98                             break;
  99                         case CloudActionType.DeleteCloud:
 100                             DeleteCloudFile(accountInfo, cloudFile, cloudFile.Name);
 101                             break;
 102                         case CloudActionType.RenameCloud:
 103                             var moveAction = (CloudMoveAction)action;
 104                             RenameCloudFile(accountInfo, moveAction);
 105                             break;
 106                         case CloudActionType.MustSynch:
 107
 108                             if (!File.Exists(downloadPath))
 109                             {
 110                                 await DownloadCloudFile(accountInfo, cloudFile, downloadPath);
 111                             }
 112                             else
 113                             {
 114                                 await SyncFiles(accountInfo, action);
 115                             }
 116                             break;
 117                     }
 118                     Log.InfoFormat("[ACTION] End Processing {0}:{1}->{2}", action.Action, action.LocalFile,
 119                                            action.CloudFile.Name);
 120                 }
 121                 catch (OperationCanceledException)
 122                 {
 123                     throw;
 124                 }
 125                 catch (FileNotFoundException exc)
 126                 {
 127                     Log.ErrorFormat("{0} : {1} -> {2}  failed because the file was not found.\n Rescheduling a delete",
 128                         action.Action, action.LocalFile, action.CloudFile, exc);
 129                     //Post a delete action for the missing file
 130                     Post(new CloudDeleteAction(action));
 131                 }
 132                 catch (Exception exc)
 133                 {
 134                     Log.ErrorFormat("[REQUEUE] {0} : {1} -> {2} due to exception\r\n{3}",
 135                                      action.Action, action.LocalFile, action.CloudFile, exc);
 136
 137                     _agent.Post(action);
 138                 }
 139             }
 140         }
 141
 142         private async Task SyncFiles(AccountInfo accountInfo,CloudAction action)
 143         {
 144             if (accountInfo == null)
 145                 throw new ArgumentNullException("accountInfo");
 146             if (action==null)
 147                 throw new ArgumentNullException("action");
 148             if (action.LocalFile==null)
 149                 throw new ArgumentException("The action's local file is not specified","action");
 150             if (!Path.IsPathRooted(action.LocalFile.FullName))
 151                 throw new ArgumentException("The action's local file path must be absolute","action");
 152             if (action.CloudFile== null)
 153                 throw new ArgumentException("The action's cloud file is not specified", "action");
 154             Contract.EndContractBlock();
 155
 156             var localFile = action.LocalFile;
 157             var cloudFile = action.CloudFile;
 158             var downloadPath=action.LocalFile.FullName.ToLower();
 159
 160             var cloudHash = cloudFile.Hash.ToLower();
 161             var localHash = action.LocalHash.Value.ToLower();
 162             var topHash = action.TopHash.Value.ToLower();
 163
 164             //Not enough to compare only the local hashes, also have to compare the tophashes
 165
 166             //If any of the hashes match, we are done
 167             if ((cloudHash == localHash || cloudHash == topHash))
 168             {
 169                 Log.InfoFormat("Skipping {0}, hashes match",downloadPath);
 170                 return;
 171             }
 172
 173             //The hashes DON'T match. We need to sync
 174             var lastLocalTime = localFile.LastWriteTime;
 175             var lastUpTime = cloudFile.Last_Modified;
 176
 177             //If the local file is newer upload it
 178             if (lastUpTime <= lastLocalTime)
 179             {
 180                 //It probably means it was changed while the app was down
 181                 UploadCloudFile(action);
 182             }
 183             else
 184             {
 185                 //It the cloud file has a later date, it was modified by another user or computer.
 186                 //We need to check the local file's status
 187                 var status = StatusKeeper.GetFileStatus(downloadPath);
 188                 switch (status)
 189                 {
 190                     case FileStatus.Unchanged:
 191                         //If the local file's status is Unchanged, we can go on and download the newer cloud file
 192                         await DownloadCloudFile(accountInfo,cloudFile,downloadPath);
 193                         break;
 194                     case FileStatus.Modified:
 195                         //If the local file is Modified, we may have a conflict. In this case we should mark the file as Conflict
 196                         //We can't ensure that a file modified online since the last time will appear as Modified, unless we
 197                         //index all files before we start listening.
 198                     case FileStatus.Created:
 199                         //If the local file is Created, it means that the local and cloud files aren't related,
 200                         // yet they have the same name.
 201
 202                         //In both cases we must mark the file as in conflict
 203                         ReportConflict(downloadPath);
 204                         break;
 205                     default:
 206                         //Other cases should never occur. Mark them as Conflict as well but log a warning
 207                         ReportConflict(downloadPath);
 208                         Log.WarnFormat("Unexcepted status {0} for file {1}->{2}", status,
 209                                        downloadPath, action.CloudFile.Name);
 210                         break;
 211                 }
 212             }
 213         }
 214
 215         private void ReportConflict(string downloadPath)
 216         {
 217             if (String.IsNullOrWhiteSpace(downloadPath))
 218                 throw new ArgumentNullException("downloadPath");
 219             Contract.EndContractBlock();
 220
 221             StatusKeeper.SetFileOverlayStatus(downloadPath, FileOverlayStatus.Conflict);
 222             var message = String.Format("Conflict detected for file {0}", downloadPath);
 223             Log.Warn(message);
 224             StatusNotification.NotifyChange(message, TraceLevel.Warning);
 225         }
 226
 227         public void Post(CloudAction cloudAction)
 228         {
 229             if (cloudAction == null)
 230                 throw new ArgumentNullException("cloudAction");
 231             if (cloudAction.AccountInfo==null)
 232                 throw new ArgumentException("The CloudAction.AccountInfo is empty","cloudAction");
 233             Contract.EndContractBlock();
 234
 235             //If the action targets a local file, add a treehash calculation
 236             if (cloudAction.LocalFile != null)
 237             {
 238                 var accountInfo = cloudAction.AccountInfo;
 239                 if (cloudAction.LocalFile.Length>accountInfo.BlockSize)
 240                     cloudAction.TopHash = new Lazy<string>(() => Signature.CalculateTreeHashAsync(cloudAction.LocalFile,
 241                                     accountInfo.BlockSize, accountInfo.BlockHash).Result
 242                                      .TopHash.ToHashString());
 243                 else
 244                 {
 245                     cloudAction.TopHash=new Lazy<string>(()=> cloudAction.LocalHash.Value);
 246                 }
 247
 248             }
 249             _agent.Post(cloudAction);
 250         }
 251
 252         class ObjectInfoByNameComparer:IEqualityComparer<ObjectInfo>
 253         {
 254             public bool Equals(ObjectInfo x, ObjectInfo y)
 255             {
 256                 return x.Name.Equals(y.Name,StringComparison.InvariantCultureIgnoreCase);
 257             }
 258
 259             public int GetHashCode(ObjectInfo obj)
 260             {
 261                 return obj.Name.ToLower().GetHashCode();
 262             }
 263         }
 264
 265
 266
 267         //Remote files are polled periodically. Any changes are processed
 268         public Task ProcessRemoteFiles(DateTime? since=null)
 269         {
 270             return Task<Task>.Factory.StartNewDelayed(10000, () =>
 271             {
 272                 using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))
 273                 {
 274                     //Next time we will check for all changes since the current check minus 1 second
 275                     //This is done to ensure there are no discrepancies due to clock differences
 276                     DateTime nextSince = DateTime.Now.AddSeconds(-1);
 277
 278                     var tasks=from accountInfo in _accounts
 279                               select ProcessAccountFiles(accountInfo, since);
 280                     var process=Task.Factory.Iterate(tasks);
 281
 282                     return process.ContinueWith(t =>
 283                     {
 284                         if (t.IsFaulted)
 285                         {
 286                             Log.Error("Error while processing accounts");
 287                             t.Exception.Handle(exc=>
 288                                                    {
 289                                                        Log.Error("Details:", exc);
 290                                                        return true;
 291                                                    });
 292                         }
 293                         ProcessRemoteFiles(nextSince);
 294                     });
 295                 }
 296             });
 297         }
 298
 299         public Task ProcessAccountFiles(AccountInfo accountInfo,DateTime? since=null)
 300         {
 301             if (accountInfo==null)
 302                 throw new ArgumentNullException("accountInfo");
 303             if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))
 304                 throw new ArgumentException("The AccountInfo.AccountPath is empty","accountInfo");
 305             Contract.EndContractBlock();
 306
 307             using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))
 308             {
 309                 Log.Info("Scheduled");
 310                 var client=new CloudFilesClient(accountInfo);
 311
 312                 var containers = client.ListContainers(accountInfo.UserName);
 313
 314                 CreateContainerFolders(accountInfo, containers);
 315
 316
 317                 //Get the list of server objects changed since the last check
 318                 var listObjects = from container in containers
 319                                   select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>
 320                                         client.ListObjects(accountInfo.UserName, container.Name, since),container.Name);
 321
 322                 var listAll = Task.Factory.WhenAll(listObjects.ToArray());
 323
 324
 325
 326                 //Get the list of deleted objects since the last check
 327 /*
 328                 var listTrash = Task<IList<ObjectInfo>>.Factory.StartNew(() =>
 329                                 client.ListObjects(accountInfo.UserName, FolderConstants.TrashContainer, since));
 330
 331                 var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(() =>
 332                                 client.ListSharedObjects(since));
 333
 334                 var listAll = Task.Factory.TrackedSequence(
 335                     () => listObjects,
 336                     () => listTrash,
 337                     () => listShared);
 338 */
 339
 340
 341
 342                 var enqueueFiles = listAll.ContinueWith(task =>
 343                 {
 344                     if (task.IsFaulted)
 345                     {
 346                         //ListObjects failed at this point, need to reschedule
 347                         Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {0}", accountInfo.UserName,task.Exception);
 348                         return;
 349                     }
 350                     using (log4net.ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))
 351                     {
 352                         var dict=task.Result.ToDictionary(t=> t.AsyncState);
 353
 354                         //Get all non-trash objects. Remember, the container name is stored in AsyncState
 355                         var remoteObjects = from objectList in task.Result
 356                                             where (string)objectList.AsyncState != "trash"
 357                                             from obj in objectList.Result
 358                                             select obj;
 359
 360                         var trashObjects = dict["trash"].Result;
 361                         //var sharedObjects = ((Task<IList<ObjectInfo>>) task.Result[2]).Result;
 362
 363                         //Items with the same name, hash may be both in the container and the trash
 364                         //Don't delete items that exist in the container
 365                         var realTrash = from trash in trashObjects
 366                                         where !remoteObjects.Any(info => info.Hash == trash.Hash)
 367                                         select trash;
 368                         ProcessDeletedFiles(accountInfo,realTrash);
 369
 370
 371                         var remote = from info in remoteObjects//.Union(sharedObjects)
 372                                      let name = info.Name
 373                                      where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&
 374                                            !name.StartsWith(FolderConstants.CacheFolder +"/", StringComparison.InvariantCultureIgnoreCase)
 375                                      select info;
 376
 377                         //Create a list of actions from the remote files
 378                         var allActions = ObjectsToActions(accountInfo,remote);
 379
 380                         //And remove those that are already being processed by the agent
 381                         var distinctActions = allActions
 382                             .Except(_agent.GetEnumerable(), new PithosMonitor.LocalFileComparer())
 383                             .ToList();
 384
 385                         //Queue all the actions
 386                         foreach (var message in distinctActions)
 387                         {
 388                             Post(message);
 389                         }
 390
 391                         //Report the number of new files
 392                         var remoteCount = distinctActions.Count(action=>
 393                             action.Action==CloudActionType.DownloadUnconditional);
 394 /*
 395                         if ( remoteCount > 0)
 396                             StatusNotification.NotifyChange(String.Format("Processing {0} new files", remoteCount));
 397 */
 398
 399                         Log.Info("[LISTENER] End Processing");
 400                     }
 401                 });
 402
 403                 var log = enqueueFiles.ContinueWith(t =>
 404                 {
 405                     if (t.IsFaulted)
 406                     {
 407                         Log.Error("[LISTENER] Exception", t.Exception);
 408                     }
 409                     else
 410                     {
 411                         Log.Info("[LISTENER] Finished");
 412                     }
 413                 });
 414                 return log;
 415             }
 416         }
 417
 418         private static void CreateContainerFolders(AccountInfo accountInfo, IList<ContainerInfo> containers)
 419         {
 420             var containerPaths = from container in containers
 421                                  let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)
 422                                  where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)
 423                                  select containerPath;
 424
 425             foreach (var path in containerPaths)
 426             {
 427                 Directory.CreateDirectory(path);
 428             }
 429         }
 430
 431         //Creates an appropriate action for each server file
 432         private IEnumerable<CloudAction> ObjectsToActions(AccountInfo accountInfo,IEnumerable<ObjectInfo> remote)
 433         {
 434             if (remote==null)
 435                 throw new ArgumentNullException();
 436             Contract.EndContractBlock();
 437             var fileAgent = GetFileAgent(accountInfo);
 438
 439             //In order to avoid multiple iterations over the files, we iterate only once
 440             //over the remote files
 441             foreach (var objectInfo in remote)
 442             {
 443                 var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);
 444                 //and remove any matching objects from the list, adding them to the commonObjects list
 445
 446                 if (fileAgent.Exists(relativePath))
 447                 {
 448                     var localFile = fileAgent.GetFileInfo(relativePath);
 449                     var state = FileState.FindByFilePath(localFile.FullName);
 450                     //Common files should be checked on a per-case basis to detect differences, which is newer
 451
 452                     yield return new CloudAction(accountInfo,CloudActionType.MustSynch,
 453                                                    localFile, objectInfo, state, accountInfo.BlockSize,
 454                                                    accountInfo.BlockHash);
 455                 }
 456                 else
 457                 {
 458                     //If there is no match we add them to the localFiles list
 459                     //but only if the file is not marked for deletion
 460                     var targetFile = Path.Combine(accountInfo.AccountPath, relativePath);
 461                     var fileStatus = StatusKeeper.GetFileStatus(targetFile);
 462                     if (fileStatus != FileStatus.Deleted)
 463                     {
 464                         //Remote files should be downloaded
 465                         yield return new CloudDownloadAction(accountInfo,objectInfo);
 466                     }
 467                 }
 468             }
 469         }
 470
 471         private static FileAgent GetFileAgent(AccountInfo accountInfo)
 472         {
 473             return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);
 474         }
 475
 476         private void ProcessDeletedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects)
 477         {
 478             var fileAgent = GetFileAgent(accountInfo);
 479             foreach (var trashObject in trashObjects)
 480             {
 481                 var barePath = trashObject.RelativeUrlToFilePath(accountInfo.UserName);
 482                 //HACK: Assume only the "pithos" container is used. Must find out what happens when
 483                 //deleting a file from a different container
 484                 var relativePath = Path.Combine("pithos", barePath);
 485                 fileAgent.Delete(relativePath);
 486             }
 487         }
 488
 489
 490         private void RenameCloudFile(AccountInfo accountInfo,CloudMoveAction action)
 491         {
 492             if (accountInfo==null)
 493                 throw new ArgumentNullException("accountInfo");
 494             if (action==null)
 495                 throw new ArgumentNullException("action");
 496             if (action.CloudFile==null)
 497                 throw new ArgumentException("CloudFile","action");
 498             if (action.LocalFile==null)
 499                 throw new ArgumentException("LocalFile","action");
 500             if (action.OldLocalFile==null)
 501                 throw new ArgumentException("OldLocalFile","action");
 502             if (action.OldCloudFile==null)
 503                 throw new ArgumentException("OldCloudFile","action");
 504             Contract.EndContractBlock();
 505
 506             var newFilePath = action.LocalFile.FullName;
 507             //The local file is already renamed
 508             this.StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Modified);
 509
 510
 511             var account = action.CloudFile.Account ?? accountInfo.UserName;
 512             var container = action.CloudFile.Container;
 513
 514             var client = new CloudFilesClient(accountInfo);
 515             client.MoveObject(account, container, action.OldCloudFile.Name, container, action.CloudFile.Name);
 516
 517             this.StatusKeeper.SetFileStatus(newFilePath, FileStatus.Unchanged);
 518             this.StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Normal);
 519             NativeMethods.RaiseChangeNotification(newFilePath);
 520         }
 521
 522         private void DeleteCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile,string fileName)
 523         {
 524             if (accountInfo == null)
 525                 throw new ArgumentNullException("accountInfo");
 526             if (cloudFile==null)
 527                 throw new ArgumentNullException("cloudFile");
 528
 529             if (String.IsNullOrWhiteSpace(fileName))
 530                 throw new ArgumentNullException("fileName");
 531             if (Path.IsPathRooted(fileName))
 532                 throw new ArgumentException("The fileName should not be rooted","fileName");
 533             if (String.IsNullOrWhiteSpace(cloudFile.Container))
 534                 throw new ArgumentException("Invalid container", "cloudFile");
 535             Contract.EndContractBlock();
 536
 537             var fileAgent = GetFileAgent(accountInfo);
 538
 539             using ( log4net.ThreadContext.Stacks["DeleteCloudFile"].Push("Delete"))
 540             {
 541                 var info = fileAgent.GetFileInfo(fileName);
 542                 var fullPath = info.FullName.ToLower();
 543                 this.StatusKeeper.SetFileOverlayStatus(fullPath, FileOverlayStatus.Modified);
 544
 545                 var account = cloudFile.Account ?? accountInfo.UserName;
 546                 var container = cloudFile.Container ;//?? FolderConstants.PithosContainer;
 547
 548                 var client = new CloudFilesClient(accountInfo);
 549                 client.DeleteObject(account, container, fileName);
 550
 551                 this.StatusKeeper.ClearFileStatus(fullPath);
 552             }
 553         }
 554
 555         //Download a file.
 556         private async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile , string localPath)
 557         {
 558             if (accountInfo == null)
 559                 throw new ArgumentNullException("accountInfo");
 560             if (cloudFile == null)
 561                 throw new ArgumentNullException("cloudFile");
 562             if (String.IsNullOrWhiteSpace(cloudFile.Account))
 563                 throw new ArgumentNullException("cloudFile");
 564             if (String.IsNullOrWhiteSpace(cloudFile.Container))
 565                 throw new ArgumentNullException("cloudFile");
 566             if (String.IsNullOrWhiteSpace(localPath))
 567                 throw new ArgumentNullException("localPath");
 568             if (!Path.IsPathRooted(localPath))
 569                 throw new ArgumentException("The localPath must be rooted", "localPath");
 570             Contract.EndContractBlock();
 571
 572             Uri relativeUrl = new Uri(cloudFile.Name, UriKind.Relative);
 573
 574             var url = relativeUrl.ToString();
 575             if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase))
 576                 return;
 577
 578             //Are we already downloading or uploading the file?
 579             using (var gate=NetworkGate.Acquire(localPath, NetworkOperation.Downloading))
 580             {
 581                 if (gate.Failed)
 582                     return;
 583                 //The file's hashmap will be stored in the same location with the extension .hashmap
 584                 //var hashPath = Path.Combine(FileAgent.CachePath, relativePath + ".hashmap");
 585
 586                 var client = new CloudFilesClient(accountInfo);
 587                 var account = cloudFile.Account;
 588                 var container = cloudFile.Container;
 589
 590                 //Retrieve the hashmap from the server
 591                 var serverHash = await client.GetHashMap(account, container, url);
 592                 //If it's a small file
 593                 if (serverHash.Hashes.Count == 1 )
 594                     //Download it in one go
 595                     await DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath, serverHash);
 596                     //Otherwise download it block by block
 597                 else
 598                     await DownloadWithBlocks(accountInfo,client, cloudFile, relativeUrl, localPath, serverHash);
 599
 600                 if (cloudFile.AllowedTo == "read")
 601                 {
 602                     var attributes=File.GetAttributes(localPath);
 603                     File.SetAttributes(localPath,attributes|FileAttributes.ReadOnly);
 604                 }
 605
 606                 //Now we can store the object's metadata without worrying about ghost status entries
 607                 StatusKeeper.StoreInfo(localPath, cloudFile);
 608
 609             }
 610         }
 611
 612         //Download a small file with a single GET operation
 613         private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string localPath,TreeHash serverHash)
 614         {
 615             if (client == null)
 616                 throw new ArgumentNullException("client");
 617             if (cloudFile==null)
 618                 throw new ArgumentNullException("cloudFile");
 619             if (relativeUrl == null)
 620                 throw new ArgumentNullException("relativeUrl");
 621             if (String.IsNullOrWhiteSpace(localPath))
 622                 throw new ArgumentNullException("localPath");
 623             if (!Path.IsPathRooted(localPath))
 624                 throw new ArgumentException("The localPath must be rooted", "localPath");
 625             Contract.EndContractBlock();
 626
 627             //If the file already exists
 628             if (File.Exists(localPath))
 629             {
 630                 //First check with MD5 as this is a small file
 631                 var localMD5 = Signature.CalculateMD5(localPath);
 632                 var cloudHash=serverHash.TopHash.ToHashString();
 633                 if (localMD5==cloudHash)
 634                     return;
 635                 //Then check with a treehash
 636                 var localTreeHash = Signature.CalculateTreeHash(localPath, serverHash.BlockSize, serverHash.BlockHash);
 637                 var localHash = localTreeHash.TopHash.ToHashString();
 638                 if (localHash==cloudHash)
 639                     return;
 640             }
 641
 642             var fileAgent = GetFileAgent(accountInfo);
 643             //Calculate the relative file path for the new file
 644             var relativePath = relativeUrl.RelativeUriToFilePath();
 645             //The file will be stored in a temporary location while downloading with an extension .download
 646             var tempPath = Path.Combine(fileAgent.CachePath, relativePath + ".download");
 647             //Make sure the target folder exists. DownloadFileTask will not create the folder
 648             var tempFolder = Path.GetDirectoryName(tempPath);
 649             if (!Directory.Exists(tempFolder))
 650                 Directory.CreateDirectory(tempFolder);
 651
 652             //Download the object to the temporary location
 653             await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl.ToString(), tempPath).ContinueWith(t =>
 654             {
 655                 t.PropagateExceptions();
 656                 //Create the local folder if it doesn't exist (necessary for shared objects)
 657                 var localFolder = Path.GetDirectoryName(localPath);
 658                 if (!Directory.Exists(localFolder))
 659                     Directory.CreateDirectory(localFolder);
 660                 //And move it to its actual location once downloading is finished
 661                 if (File.Exists(localPath))
 662                     File.Replace(tempPath,localPath,null,true);
 663                 else
 664                     File.Move(tempPath,localPath);
 665                 //Notify listeners that a local file has changed
 666                 StatusNotification.NotifyChangedFile(localPath);
 667
 668             });
 669         }
 670
 671         //Download a file asynchronously using blocks
 672         public async Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string localPath, TreeHash serverHash)
 673         {
 674             if (client == null)
 675                 throw new ArgumentNullException("client");
 676             if (cloudFile == null)
 677                 throw new ArgumentNullException("cloudFile");
 678             if (relativeUrl == null)
 679                 throw new ArgumentNullException("relativeUrl");
 680             if (String.IsNullOrWhiteSpace(localPath))
 681                 throw new ArgumentNullException("localPath");
 682             if (!Path.IsPathRooted(localPath))
 683                 throw new ArgumentException("The localPath must be rooted", "localPath");
 684             if (serverHash == null)
 685                 throw new ArgumentNullException("serverHash");
 686             Contract.EndContractBlock();
 687
 688            var fileAgent = GetFileAgent(accountInfo);
 689
 690             //Calculate the relative file path for the new file
 691             var relativePath = relativeUrl.RelativeUriToFilePath();
 692             var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash);
 693
 694
 695
 696             //Calculate the file's treehash
 697             var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash);
 698
 699             //And compare it with the server's hash
 700             var upHashes = serverHash.GetHashesAsStrings();
 701             var localHashes = treeHash.HashDictionary;
 702             for (int i = 0; i < upHashes.Length; i++)
 703             {
 704                 //For every non-matching hash
 705                 var upHash = upHashes[i];
 706                 if (!localHashes.ContainsKey(upHash))
 707                 {
 708                     if (blockUpdater.UseOrphan(i, upHash))
 709                     {
 710                         Log.InfoFormat("[BLOCK GET] ORPHAN FOUND for {0} of {1} for {2}", i, upHashes.Length, localPath);
 711                         continue;
 712                     }
 713                     Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath);
 714                     var start = i*serverHash.BlockSize;
 715                     //To download the last block just pass a null for the end of the range
 716                     long? end = null;
 717                     if (i < upHashes.Length - 1 )
 718                         end= ((i + 1)*serverHash.BlockSize) ;
 719
 720                     //Download the missing block
 721                     var block = await client.GetBlock(cloudFile.Account, cloudFile.Container, relativeUrl, start, end);
 722
 723                     //and store it
 724                     blockUpdater.StoreBlock(i, block);
 725
 726
 727                     Log.InfoFormat("[BLOCK GET] FINISH {0} of {1} for {2}", i, upHashes.Length, localPath);
 728                 }
 729             }
 730
 731             //Want to avoid notifications if no changes were made
 732             var hasChanges = blockUpdater.HasBlocks;
 733             blockUpdater.Commit();
 734
 735             if (hasChanges)
 736                 //Notify listeners that a local file has changed
 737                 StatusNotification.NotifyChangedFile(localPath);
 738
 739             Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath);
 740         }
 741
 742
 743         private async Task UploadCloudFile(CloudAction action)
 744         {
 745             if (action == null)
 746                 throw new ArgumentNullException("action");
 747             Contract.EndContractBlock();
 748
 749             try
 750             {
 751                 if (action == null)
 752                     throw new ArgumentNullException("action");
 753                 Contract.EndContractBlock();
 754
 755                 var accountInfo = action.AccountInfo;
 756
 757                 var fileInfo = action.LocalFile;
 758
 759                 if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
 760                     return;
 761
 762                 var relativePath = fileInfo.AsRelativeTo(accountInfo.AccountPath);
 763                 if (relativePath.StartsWith(FolderConstants.OthersFolder))
 764                 {
 765                     var parts = relativePath.Split('\\');
 766                     var accountName = parts[1];
 767                     var oldName = accountInfo.UserName;
 768                     var absoluteUri = accountInfo.StorageUri.AbsoluteUri;
 769                     var nameIndex = absoluteUri.IndexOf(oldName);
 770                     var root = absoluteUri.Substring(0, nameIndex);
 771
 772                     accountInfo = new AccountInfo
 773                     {
 774                         UserName = accountName,
 775                         AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 776                         StorageUri = new Uri(root + accountName),
 777                         BlockHash = accountInfo.BlockHash,
 778                         BlockSize = accountInfo.BlockSize,
 779                         Token = accountInfo.Token
 780                     };
 781                 }
 782
 783
 784                 var fullFileName = fileInfo.FullName;
 785                 using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))
 786                 {
 787                     //Abort if the file is already being uploaded or downloaded
 788                     if (gate.Failed)
 789                         return;
 790
 791                     var cloudFile = action.CloudFile;
 792                     var account = cloudFile.Account ?? accountInfo.UserName;
 793
 794                     var client = new CloudFilesClient(accountInfo);
 795                     //Even if GetObjectInfo times out, we can proceed with the upload
 796                     var info = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
 797                     var cloudHash = info.Hash.ToLower();
 798
 799                     var hash = action.LocalHash.Value;
 800                     var topHash = action.TopHash.Value;
 801
 802                     //If the file hashes match, abort the upload
 803                     if (hash == cloudHash || topHash == cloudHash)
 804                     {
 805                         //but store any metadata changes
 806                         this.StatusKeeper.StoreInfo(fullFileName, info);
 807                         Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
 808                         return;
 809                     }
 810
 811                     if (info.AllowedTo == "read")
 812                         return;
 813
 814                     //Mark the file as modified while we upload it
 815                     StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
 816                     //And then upload it
 817
 818                     //Upload even small files using the Hashmap. The server may already containt
 819                     //the relevant folder
 820
 821                     //First, calculate the tree hash
 822                     var treeHash = await Signature.CalculateTreeHashAsync(fileInfo.FullName, accountInfo.BlockSize,
 823                         accountInfo.BlockHash);
 824
 825                     await UploadWithHashMap(accountInfo, cloudFile, fileInfo, cloudFile.Name, treeHash);
 826
 827                     //If everything succeeds, change the file and overlay status to normal
 828                     this.StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal);
 829                 }
 830                 //Notify the Shell to update the overlays
 831                 NativeMethods.RaiseChangeNotification(fullFileName);
 832                 StatusNotification.NotifyChangedFile(fullFileName);
 833             }
 834             catch (AggregateException ex)
 835             {
 836                 var exc = ex.InnerException as WebException;
 837                 if (exc == null)
 838                     throw ex.InnerException;
 839                 if (HandleUploadWebException(action, exc))
 840                     return;
 841                 throw;
 842             }
 843             catch (WebException ex)
 844             {
 845                 if (HandleUploadWebException(action, ex))
 846                     return;
 847                 throw;
 848             }
 849             catch (Exception ex)
 850             {
 851                 Log.Error("Unexpected error while uploading file", ex);
 852                 throw;
 853             }
 854
 855         }
 856
 857         private bool HandleUploadWebException(CloudAction action, WebException exc)
 858         {
 859             var response = exc.Response as HttpWebResponse;
 860             if (response == null)
 861                 throw exc;
 862             if (response.StatusCode == HttpStatusCode.Unauthorized)
 863             {
 864                 Log.Error("Not allowed to upload file", exc);
 865                 var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);
 866                 StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal);
 867                 StatusNotification.NotifyChange(message, TraceLevel.Warning);
 868                 return true;
 869             }
 870             return false;
 871         }
 872
 873         public async Task UploadWithHashMap(AccountInfo accountInfo,ObjectInfo cloudFile,FileInfo fileInfo,string url,TreeHash treeHash)
 874         {
 875             if (accountInfo == null)
 876                 throw new ArgumentNullException("accountInfo");
 877             if (cloudFile==null)
 878                 throw new ArgumentNullException("cloudFile");
 879             if (fileInfo == null)
 880                 throw new ArgumentNullException("fileInfo");
 881             if (String.IsNullOrWhiteSpace(url))
 882                 throw new ArgumentNullException(url);
 883             if (treeHash==null)
 884                 throw new ArgumentNullException("treeHash");
 885             if (String.IsNullOrWhiteSpace(cloudFile.Container) )
 886                 throw new ArgumentException("Invalid container","cloudFile");
 887             Contract.EndContractBlock();
 888
 889             var fullFileName = fileInfo.FullName;
 890
 891             var account = cloudFile.Account ?? accountInfo.UserName;
 892             var container = cloudFile.Container ;
 893
 894             var client = new CloudFilesClient(accountInfo);
 895             //Send the hashmap to the server
 896             var missingHashes =  await client.PutHashMap(account, container, url, treeHash);
 897             //If the server returns no missing hashes, we are done
 898             while (missingHashes.Count > 0)
 899             {
 900
 901                 var buffer = new byte[accountInfo.BlockSize];
 902                 foreach (var missingHash in missingHashes)
 903                 {
 904                     //Find the proper block
 905                     var blockIndex = treeHash.HashDictionary[missingHash];
 906                     var offset = blockIndex*accountInfo.BlockSize;
 907
 908                     var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);
 909
 910                     try
 911                     {
 912                         //And upload the block
 913                         await client.PostBlock(account, container, buffer, 0, read);
 914                         Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);
 915                     }
 916                     catch (Exception exc)
 917                     {
 918                         Log.ErrorFormat("[ERROR] uploading block {0} of {1}\n{2}", blockIndex, fullFileName, exc);
 919                     }
 920
 921                 }
 922
 923                 //Repeat until there are no more missing hashes
 924                 missingHashes = await client.PutHashMap(account, container, url, treeHash);
 925             }
 926         }
 927
 928
 929         public void AddAccount(AccountInfo accountInfo)
 930         {
 931             if (!_accounts.Contains(accountInfo))
 932                 _accounts.Add(accountInfo);
 933         }
 934     }
 935
 936
 937
 938
 939 }