code.grnet.gr Git - pithos-ms-client/blob - trunk%2FPithos.Core%2FAgents%2FNetworkAgent.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="NetworkAgent.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42
  43 using System;
  44 using System.Collections.Generic;
  45 using System.ComponentModel.Composition;
  46 using System.Diagnostics;
  47 using System.Diagnostics.Contracts;
  48 using System.IO;
  49 using System.Net;
  50 using System.Reflection;
  51 using System.Threading;
  52 using System.Threading.Tasks;
  53 using Castle.ActiveRecord;
  54 using Pithos.Interfaces;
  55 using Pithos.Network;
  56 using log4net;
  57
  58 namespace Pithos.Core.Agents
  59 {
  60     [Export]
  61     public class NetworkAgent
  62     {
  63         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  64
  65         private Agent<CloudAction> _agent;
  66
  67         [System.ComponentModel.Composition.Import]
  68         private DeleteAgent _deleteAgent { get; set; }
  69
  70         [System.ComponentModel.Composition.Import]
  71         public IStatusKeeper StatusKeeper { get; set; }
  72
  73         private IStatusNotification _statusNotification;
  74         public IStatusNotification StatusNotification
  75         {
  76             get { return _statusNotification; }
  77             set
  78             {
  79                 _statusNotification = value;
  80                 _deleteAgent.StatusNotification = value;
  81             }
  82         }
  83
  84
  85         [System.ComponentModel.Composition.Import]
  86         public IPithosSettings Settings { get; set; }
  87
  88         //The Proceed signals the poll agent that it can proceed with polling.
  89         //Essentially it stops the poll agent to give priority to the network agent
  90         //Initially the event is signalled because we don't need to pause
  91         private readonly AsyncManualResetEvent _proceedEvent = new AsyncManualResetEvent(true);
  92
  93         public AsyncManualResetEvent ProceedEvent
  94         {
  95             get { return _proceedEvent; }
  96         }
  97
  98
  99         public void Start()
 100         {
 101             if (_agent != null)
 102                 return;
 103
 104             if (Log.IsDebugEnabled)
 105                 Log.Debug("Starting Network Agent");
 106
 107             _agent = Agent<CloudAction>.Start(inbox =>
 108             {
 109                 Action loop = null;
 110                 loop = () =>
 111                 {
 112                     _deleteAgent.ProceedEvent.Wait();
 113                     var message = inbox.Receive();
 114                     var process=message.Then(Process,inbox.CancellationToken);
 115                     inbox.LoopAsync(process, loop);
 116                 };
 117                 loop();
 118             });
 119
 120         }
 121
 122         private async Task Process(CloudAction action)
 123         {
 124             if (action == null)
 125                 throw new ArgumentNullException("action");
 126             if (action.AccountInfo==null)
 127                 throw new ArgumentException("The action.AccountInfo is empty","action");
 128             Contract.EndContractBlock();
 129
 130
 131
 132
 133             using (log4net.ThreadContext.Stacks["Operation"].Push(action.ToString()))
 134             {
 135
 136                 var cloudFile = action.CloudFile;
 137                 var downloadPath = action.GetDownloadPath();
 138
 139                 try
 140                 {
 141                     StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Processing");
 142                     _proceedEvent.Reset();
 143                     //UpdateStatus(PithosStatus.Syncing);
 144                     var accountInfo = action.AccountInfo;
 145
 146                     if (action.Action == CloudActionType.DeleteCloud)
 147                     {
 148                         //Redirect deletes to the delete agent
 149                         _deleteAgent.Post((CloudDeleteAction)action);
 150                     }
 151                     if (_deleteAgent.IsDeletedFile(action))
 152                     {
 153                         //Clear the status of already deleted files to avoid reprocessing
 154                         if (action.LocalFile != null)
 155                             StatusKeeper.ClearFileStatus(action.LocalFile.FullName);
 156                     }
 157                     else
 158                     {
 159                         switch (action.Action)
 160                         {
 161                             case CloudActionType.UploadUnconditional:
 162                                 //Abort if the file was deleted before we reached this point
 163                                 await UploadCloudFile(action);
 164                                 break;
 165                             case CloudActionType.DownloadUnconditional:
 166                                 await DownloadCloudFile(accountInfo, cloudFile, downloadPath);
 167                                 break;
 168                             case CloudActionType.RenameCloud:
 169                                 var moveAction = (CloudMoveAction)action;
 170                                 RenameCloudFile(accountInfo, moveAction);
 171                                 break;
 172                             case CloudActionType.RenameLocal:
 173                                 RenameLocalFile(accountInfo, action);
 174                                 break;
 175                             case CloudActionType.MustSynch:
 176                                 if (!File.Exists(downloadPath) && !Directory.Exists(downloadPath))
 177                                 {
 178                                     await DownloadCloudFile(accountInfo, cloudFile, downloadPath);
 179                                 }
 180                                 else
 181                                 {
 182                                     await SyncFiles(accountInfo, action);
 183                                 }
 184                                 break;
 185                         }
 186                     }
 187                     Log.InfoFormat("End Processing {0}:{1}->{2}", action.Action, action.LocalFile,
 188                                            action.CloudFile.Name);
 189                 }
 190                 catch (WebException exc)
 191                 {
 192                     Log.ErrorFormat("[WEB ERROR] {0} : {1} -> {2} due to exception\r\n{3}", action.Action, action.LocalFile, action.CloudFile, exc);
 193                 }
 194                 catch (OperationCanceledException)
 195                 {
 196                     throw;
 197                 }
 198                 catch (DirectoryNotFoundException)
 199                 {
 200                     Log.ErrorFormat("{0} : {1} -> {2}  failed because the directory was not found.\n Rescheduling a delete",
 201                         action.Action, action.LocalFile, action.CloudFile);
 202                     //Post a delete action for the missing file
 203                     Post(new CloudDeleteAction(action));
 204                 }
 205                 catch (FileNotFoundException)
 206                 {
 207                     Log.ErrorFormat("{0} : {1} -> {2}  failed because the file was not found.\n Rescheduling a delete",
 208                         action.Action, action.LocalFile, action.CloudFile);
 209                     //Post a delete action for the missing file
 210                     Post(new CloudDeleteAction(action));
 211                 }
 212                 catch (Exception exc)
 213                 {
 214                     Log.ErrorFormat("[REQUEUE] {0} : {1} -> {2} due to exception\r\n{3}",
 215                                      action.Action, action.LocalFile, action.CloudFile, exc);
 216
 217                     _agent.Post(action);
 218                 }
 219                 finally
 220                 {
 221                     if (_agent.IsEmpty)
 222                         _proceedEvent.Set();
 223                     UpdateStatus(PithosStatus.LocalComplete);
 224                 }
 225             }
 226         }
 227
 228
 229         private void UpdateStatus(PithosStatus status)
 230         {
 231             StatusNotification.SetPithosStatus(status);
 232             //StatusNotification.Notify(new Notification());
 233         }
 234
 235         private void RenameLocalFile(AccountInfo accountInfo, CloudAction action)
 236         {
 237             if (accountInfo == null)
 238                 throw new ArgumentNullException("accountInfo");
 239             if (action == null)
 240                 throw new ArgumentNullException("action");
 241             if (action.LocalFile == null)
 242                 throw new ArgumentException("The action's local file is not specified", "action");
 243             if (!Path.IsPathRooted(action.LocalFile.FullName))
 244                 throw new ArgumentException("The action's local file path must be absolute", "action");
 245             if (action.CloudFile == null)
 246                 throw new ArgumentException("The action's cloud file is not specified", "action");
 247             Contract.EndContractBlock();
 248             using (ThreadContext.Stacks["Operation"].Push("RenameLocalFile"))
 249             {
 250
 251                 //We assume that the local file already exists, otherwise the poll agent
 252                 //would have issued a download request
 253
 254                 var currentInfo = action.CloudFile;
 255                 var previousInfo = action.CloudFile.Previous;
 256                 var fileAgent = FileAgent.GetFileAgent(accountInfo);
 257
 258                 var previousRelativepath = previousInfo.RelativeUrlToFilePath(accountInfo.UserName);
 259                 var previousFile = fileAgent.GetFileSystemInfo(previousRelativepath);
 260
 261                 //In every case we need to move the local file first
 262                 MoveLocalFile(accountInfo, previousFile, fileAgent, currentInfo);
 263             }
 264         }
 265
 266         private void MoveLocalFile(AccountInfo accountInfo, FileSystemInfo previousFile, FileAgent fileAgent,
 267                                    ObjectInfo currentInfo)
 268         {
 269             var currentRelativepath = currentInfo.RelativeUrlToFilePath(accountInfo.UserName);
 270             var newPath = Path.Combine(fileAgent.RootPath, currentRelativepath);
 271
 272             var isFile= (previousFile is FileInfo);
 273             var previousFullPath = isFile?
 274                 FileInfoExtensions.GetProperFilePathCapitalization(previousFile.FullName):
 275                 FileInfoExtensions.GetProperDirectoryCapitalization(previousFile.FullName);
 276
 277             using (var gateOld = NetworkGate.Acquire(previousFullPath, NetworkOperation.Renaming))
 278             using (var gateNew = NetworkGate.Acquire(newPath,NetworkOperation.Renaming))
 279             using (new SessionScope(FlushAction.Auto))
 280             {
 281                 if (isFile)
 282                     (previousFile as FileInfo).MoveTo(newPath);
 283                 else
 284                 {
 285                     (previousFile as DirectoryInfo).MoveTo(newPath);
 286                 }
 287                 var state = StatusKeeper.GetStateByFilePath(previousFullPath);
 288                 state.FilePath = newPath;
 289                 state.SaveCopy();
 290                 StatusKeeper.SetFileState(previousFullPath,FileStatus.Deleted,FileOverlayStatus.Deleted);
 291             }
 292         }
 293
 294         private async Task SyncFiles(AccountInfo accountInfo,CloudAction action)
 295         {
 296             if (accountInfo == null)
 297                 throw new ArgumentNullException("accountInfo");
 298             if (action==null)
 299                 throw new ArgumentNullException("action");
 300             if (action.LocalFile==null)
 301                 throw new ArgumentException("The action's local file is not specified","action");
 302             if (!Path.IsPathRooted(action.LocalFile.FullName))
 303                 throw new ArgumentException("The action's local file path must be absolute","action");
 304             if (action.CloudFile== null)
 305                 throw new ArgumentException("The action's cloud file is not specified", "action");
 306             Contract.EndContractBlock();
 307             using (ThreadContext.Stacks["Operation"].Push("SyncFiles"))
 308             {
 309
 310                 var localFile = action.LocalFile;
 311                 var cloudFile = action.CloudFile;
 312                 var downloadPath = action.LocalFile.GetProperCapitalization();
 313
 314                 var cloudHash = cloudFile.Hash.ToLower();
 315                 var previousCloudHash = cloudFile.PreviousHash.ToLower();
 316                 var localHash = action.TreeHash.Value.TopHash.ToHashString();// LocalHash.Value.ToLower();
 317                 //var topHash = action.TopHash.Value.ToLower();
 318
 319                 //At this point we know that an object has changed on the server and that a local
 320                 //file already exists. We need to decide whether the file has only changed on
 321                 //the server or there is a conflicting change on the client.
 322                 //
 323
 324                 //If the hashes match, we are done
 325                 if (cloudHash == localHash)
 326                 {
 327                     Log.InfoFormat("Skipping {0}, hashes match", downloadPath);
 328                     return;
 329                 }
 330
 331                 //The hashes DON'T match. We need to sync
 332
 333                 // If the previous tophash matches the local tophash, the file was only changed on the server.
 334                 if (localHash == previousCloudHash)
 335                 {
 336                     await DownloadCloudFile(accountInfo, cloudFile, downloadPath);
 337                 }
 338                 else
 339                 {
 340                     //If the previous and local hash don't match, there was a local conflict
 341                     //that was not uploaded to the server. We have a conflict
 342                     ReportConflict(downloadPath);
 343                 }
 344             }
 345         }
 346
 347         private void ReportConflict(string downloadPath)
 348         {
 349             if (String.IsNullOrWhiteSpace(downloadPath))
 350                 throw new ArgumentNullException("downloadPath");
 351             Contract.EndContractBlock();
 352
 353             StatusKeeper.SetFileOverlayStatus(downloadPath, FileOverlayStatus.Conflict);
 354             UpdateStatus(PithosStatus.HasConflicts);
 355             var message = String.Format("Conflict detected for file {0}", downloadPath);
 356             Log.Warn(message);
 357             StatusNotification.NotifyChange(message, TraceLevel.Warning);
 358         }
 359
 360         public void Post(CloudAction cloudAction)
 361         {
 362             if (cloudAction == null)
 363                 throw new ArgumentNullException("cloudAction");
 364             if (cloudAction.AccountInfo==null)
 365                 throw new ArgumentException("The CloudAction.AccountInfo is empty","cloudAction");
 366             Contract.EndContractBlock();
 367
 368             _deleteAgent.ProceedEvent.Wait();
 369 /*
 370
 371             //If the action targets a local file, add a treehash calculation
 372             if (!(cloudAction is CloudDeleteAction) && cloudAction.LocalFile as FileInfo != null)
 373             {
 374                 var accountInfo = cloudAction.AccountInfo;
 375                 var localFile = (FileInfo) cloudAction.LocalFile;
 376
 377                 if (localFile.Length > accountInfo.BlockSize)
 378                     cloudAction.TopHash =
 379                         new Lazy<string>(() => Signature.CalculateTreeHashAsync(localFile,
 380                                                                                 accountInfo.BlockSize,
 381                                                                                 accountInfo.BlockHash, Settings.HashingParallelism).Result
 382                                                     .TopHash.ToHashString());
 383                 else
 384                 {
 385                     cloudAction.TopHash = new Lazy<string>(() => cloudAction.LocalHash.Value);
 386                 }
 387
 388             }
 389             else
 390             {
 391                 //The hash for a directory is the empty string
 392                 cloudAction.TopHash = new Lazy<string>(() => String.Empty);
 393             }
 394 */
 395
 396             if (cloudAction is CloudDeleteAction)
 397                 _deleteAgent.Post((CloudDeleteAction)cloudAction);
 398             else
 399                 _agent.Post(cloudAction);
 400         }
 401
 402
 403         public IEnumerable<CloudAction> GetEnumerable()
 404         {
 405             return _agent.GetEnumerable();
 406         }
 407
 408         public Task GetDeleteAwaiter()
 409         {
 410             return _deleteAgent.ProceedEvent.WaitAsync();
 411         }
 412         public CancellationToken CancellationToken
 413         {
 414             get { return _agent.CancellationToken; }
 415         }
 416
 417         private static FileAgent GetFileAgent(AccountInfo accountInfo)
 418         {
 419             return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);
 420         }
 421
 422
 423
 424         private void RenameCloudFile(AccountInfo accountInfo,CloudMoveAction action)
 425         {
 426             if (accountInfo==null)
 427                 throw new ArgumentNullException("accountInfo");
 428             if (action==null)
 429                 throw new ArgumentNullException("action");
 430             if (action.CloudFile==null)
 431                 throw new ArgumentException("CloudFile","action");
 432             if (action.LocalFile==null)
 433                 throw new ArgumentException("LocalFile","action");
 434             if (action.OldLocalFile==null)
 435                 throw new ArgumentException("OldLocalFile","action");
 436             if (action.OldCloudFile==null)
 437                 throw new ArgumentException("OldCloudFile","action");
 438             Contract.EndContractBlock();
 439
 440             using (ThreadContext.Stacks["Operation"].Push("RenameCloudFile"))
 441             {
 442
 443                 var newFilePath = action.LocalFile.FullName;
 444
 445                 //How do we handle concurrent renames and deletes/uploads/downloads?
 446                 //* A conflicting upload means that a file was renamed before it had a chance to finish uploading
 447                 //  This should never happen as the network agent executes only one action at a time
 448                 //* A conflicting download means that the file was modified on the cloud. While we can go on and complete
 449                 //  the rename, there may be a problem if the file is downloaded in blocks, as subsequent block requests for the
 450                 //  same name will fail.
 451                 //  This should never happen as the network agent executes only one action at a time.
 452                 //* A conflicting delete can happen if the rename was followed by a delete action that didn't have the chance
 453                 //  to remove the rename from the queue.
 454                 //  We can probably ignore this case. It will result in an error which should be ignored
 455
 456
 457                 //The local file is already renamed
 458                 StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Modified);
 459
 460
 461                 var account = action.CloudFile.Account ?? accountInfo.UserName;
 462                 var container = action.CloudFile.Container;
 463
 464                 var client = new CloudFilesClient(accountInfo);
 465                 //TODO: What code is returned when the source file doesn't exist?
 466                 client.MoveObject(account, container, action.OldCloudFile.Name, container, action.CloudFile.Name);
 467
 468                 StatusKeeper.SetFileStatus(newFilePath, FileStatus.Unchanged);
 469                 StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Normal);
 470                 NativeMethods.RaiseChangeNotification(newFilePath);
 471             }
 472         }
 473
 474         //Download a file.
 475         private async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile , string filePath)
 476         {
 477             if (accountInfo == null)
 478                 throw new ArgumentNullException("accountInfo");
 479             if (cloudFile == null)
 480                 throw new ArgumentNullException("cloudFile");
 481             if (String.IsNullOrWhiteSpace(cloudFile.Account))
 482                 throw new ArgumentNullException("cloudFile");
 483             if (String.IsNullOrWhiteSpace(cloudFile.Container))
 484                 throw new ArgumentNullException("cloudFile");
 485             if (String.IsNullOrWhiteSpace(filePath))
 486                 throw new ArgumentNullException("filePath");
 487             if (!Path.IsPathRooted(filePath))
 488                 throw new ArgumentException("The filePath must be rooted", "filePath");
 489             Contract.EndContractBlock();
 490
 491             using (ThreadContext.Stacks["Operation"].Push("DownloadCloudFile"))
 492             {
 493
 494                 var localPath = Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath);
 495                 var relativeUrl = new Uri(cloudFile.Name, UriKind.Relative);
 496
 497                 var url = relativeUrl.ToString();
 498                 if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase))
 499                     return;
 500
 501
 502                 //Are we already downloading or uploading the file?
 503                 using (var gate = NetworkGate.Acquire(localPath, NetworkOperation.Downloading))
 504                 {
 505                     if (gate.Failed)
 506                         return;
 507
 508                     var client = new CloudFilesClient(accountInfo);
 509                     var account = cloudFile.Account;
 510                     var container = cloudFile.Container;
 511
 512                     if (cloudFile.IsDirectory)
 513                     {
 514                         if (!Directory.Exists(localPath))
 515                             try
 516                             {
 517                                 Directory.CreateDirectory(localPath);
 518                                 if (Log.IsDebugEnabled)
 519                                     Log.DebugFormat("Created Directory [{0}]",localPath);
 520                             }
 521                             catch (IOException)
 522                             {
 523                                 var localInfo = new FileInfo(localPath);
 524                                 if (localInfo.Exists && localInfo.Length == 0)
 525                                 {
 526                                     Log.WarnFormat("Malformed directory object detected for [{0}]",localPath);
 527                                     localInfo.Delete();
 528                                     Directory.CreateDirectory(localPath);
 529                                     if (Log.IsDebugEnabled)
 530                                         Log.DebugFormat("Created Directory [{0}]", localPath);
 531                                 }
 532                             }
 533                     }
 534                     else
 535                     {
 536                         var isChanged = IsObjectChanged(cloudFile, localPath);
 537                         if (isChanged)
 538                         {
 539                             //Retrieve the hashmap from the server
 540                             var serverHash = await client.GetHashMap(account, container, url);
 541                             //If it's a small file
 542                             if (serverHash.Hashes.Count == 1)
 543                                 //Download it in one go
 544                                 await
 545                                     DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath,
 546                                                             serverHash);
 547                             //Otherwise download it block by block
 548                             else
 549                                 await
 550                                     DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath,
 551                                                        serverHash);
 552
 553                             if (cloudFile.AllowedTo == "read")
 554                             {
 555                                 var attributes = File.GetAttributes(localPath);
 556                                 File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly);
 557                             }
 558                         }
 559                     }
 560
 561                     //Now we can store the object's metadata without worrying about ghost status entries
 562                     StatusKeeper.StoreInfo(localPath, cloudFile);
 563
 564                 }
 565             }
 566         }
 567
 568         private bool IsObjectChanged(ObjectInfo cloudFile, string localPath)
 569         {
 570             //If the target is a directory, there are no changes to download
 571             if (Directory.Exists(localPath))
 572                 return false;
 573             //If the file doesn't exist, we have a chagne
 574             if (!File.Exists(localPath))
 575                 return true;
 576             //If there is no stored state, we have a change
 577             var localState = StatusKeeper.GetStateByFilePath(localPath);
 578             if (localState == null)
 579                 return true;
 580
 581             var info = new FileInfo(localPath);
 582             var shortHash = info.ComputeShortHash();
 583             //If the file is different from the stored state, we have a change
 584             if (localState.ShortHash != shortHash)
 585                 return true;
 586             //If the top hashes differ, we have a change
 587             return (localState.Checksum != cloudFile.Hash);
 588         }
 589
 590         //Download a small file with a single GET operation
 591         private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath,TreeHash serverHash)
 592         {
 593             if (client == null)
 594                 throw new ArgumentNullException("client");
 595             if (cloudFile==null)
 596                 throw new ArgumentNullException("cloudFile");
 597             if (relativeUrl == null)
 598                 throw new ArgumentNullException("relativeUrl");
 599             if (String.IsNullOrWhiteSpace(filePath))
 600                 throw new ArgumentNullException("filePath");
 601             if (!Path.IsPathRooted(filePath))
 602                 throw new ArgumentException("The localPath must be rooted", "filePath");
 603             if (cloudFile.IsDirectory)
 604                 throw new ArgumentException("cloudFile is a directory, not a file","cloudFile");
 605             Contract.EndContractBlock();
 606
 607             var localPath = Pithos.Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath);
 608             StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Downloading {0}",Path.GetFileName(localPath)));
 609             StatusNotification.Notify(new CloudNotification { Data = cloudFile });
 610
 611             var fileAgent = GetFileAgent(accountInfo);
 612             //Calculate the relative file path for the new file
 613             var relativePath = relativeUrl.RelativeUriToFilePath();
 614             //The file will be stored in a temporary location while downloading with an extension .download
 615             var tempPath = Path.Combine(fileAgent.CachePath, relativePath + ".download");
 616             //Make sure the target folder exists. DownloadFileTask will not create the folder
 617             var tempFolder = Path.GetDirectoryName(tempPath);
 618             if (!Directory.Exists(tempFolder))
 619                 Directory.CreateDirectory(tempFolder);
 620
 621             //Download the object to the temporary location
 622             await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl.ToString(), tempPath);
 623
 624             //Create the local folder if it doesn't exist (necessary for shared objects)
 625             var localFolder = Path.GetDirectoryName(localPath);
 626             if (!Directory.Exists(localFolder))
 627                 try
 628                 {
 629                     Directory.CreateDirectory(localFolder);
 630                 }
 631                 catch (IOException)
 632                 {
 633                     //A file may already exist that has the same name as the new folder.
 634                     //This may be an artifact of the way Pithos handles directories
 635                     var fileInfo = new FileInfo(localFolder);
 636                     if (fileInfo.Exists && fileInfo.Length == 0)
 637                     {
 638                         Log.WarnFormat("Malformed directory object detected for [{0}]", localFolder);
 639                         fileInfo.Delete();
 640                         Directory.CreateDirectory(localFolder);
 641                     }
 642                     else
 643                         throw;
 644                 }
 645             //And move it to its actual location once downloading is finished
 646             if (File.Exists(localPath))
 647                 File.Replace(tempPath,localPath,null,true);
 648             else
 649                 File.Move(tempPath,localPath);
 650             //Notify listeners that a local file has changed
 651             StatusNotification.NotifyChangedFile(localPath);
 652
 653
 654         }
 655
 656         //Download a file asynchronously using blocks
 657         public async Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath, TreeHash serverHash)
 658         {
 659             if (client == null)
 660                 throw new ArgumentNullException("client");
 661             if (cloudFile == null)
 662                 throw new ArgumentNullException("cloudFile");
 663             if (relativeUrl == null)
 664                 throw new ArgumentNullException("relativeUrl");
 665             if (String.IsNullOrWhiteSpace(filePath))
 666                 throw new ArgumentNullException("filePath");
 667             if (!Path.IsPathRooted(filePath))
 668                 throw new ArgumentException("The filePath must be rooted", "filePath");
 669             if (serverHash == null)
 670                 throw new ArgumentNullException("serverHash");
 671             if (cloudFile.IsDirectory)
 672                 throw new ArgumentException("cloudFile is a directory, not a file", "cloudFile");
 673             Contract.EndContractBlock();
 674
 675            var fileAgent = GetFileAgent(accountInfo);
 676             var localPath = Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath);
 677
 678             //Calculate the relative file path for the new file
 679             var relativePath = relativeUrl.RelativeUriToFilePath();
 680             var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash);
 681
 682
 683             StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Calculating hashmap for {0} before download",Path.GetFileName(localPath)));
 684             //Calculate the file's treehash
 685             var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash, 2);
 686
 687             //And compare it with the server's hash
 688             var upHashes = serverHash.GetHashesAsStrings();
 689             var localHashes = treeHash.HashDictionary;
 690             ReportDownloadProgress(Path.GetFileName(localPath),0,upHashes.Length,cloudFile.Bytes);
 691             for (int i = 0; i < upHashes.Length; i++)
 692             {
 693                 //For every non-matching hash
 694                 var upHash = upHashes[i];
 695                 if (!localHashes.ContainsKey(upHash))
 696                 {
 697                     StatusNotification.Notify(new CloudNotification { Data = cloudFile });
 698
 699                     if (blockUpdater.UseOrphan(i, upHash))
 700                     {
 701                         Log.InfoFormat("[BLOCK GET] ORPHAN FOUND for {0} of {1} for {2}", i, upHashes.Length, localPath);
 702                         continue;
 703                     }
 704                     Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath);
 705                     var start = i*serverHash.BlockSize;
 706                     //To download the last block just pass a null for the end of the range
 707                     long? end = null;
 708                     if (i < upHashes.Length - 1 )
 709                         end= ((i + 1)*serverHash.BlockSize) ;
 710
 711                     //Download the missing block
 712                     var block = await client.GetBlock(cloudFile.Account, cloudFile.Container, relativeUrl, start, end);
 713
 714                     //and store it
 715                     blockUpdater.StoreBlock(i, block);
 716
 717
 718                     Log.InfoFormat("[BLOCK GET] FINISH {0} of {1} for {2}", i, upHashes.Length, localPath);
 719                 }
 720                 ReportDownloadProgress(Path.GetFileName(localPath), i, upHashes.Length, cloudFile.Bytes);
 721             }
 722
 723             //Want to avoid notifications if no changes were made
 724             var hasChanges = blockUpdater.HasBlocks;
 725             blockUpdater.Commit();
 726
 727             if (hasChanges)
 728                 //Notify listeners that a local file has changed
 729                 StatusNotification.NotifyChangedFile(localPath);
 730
 731             Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath);
 732         }
 733
 734
 735         private async Task UploadCloudFile(CloudAction action)
 736         {
 737             if (action == null)
 738                 throw new ArgumentNullException("action");
 739             Contract.EndContractBlock();
 740             using(ThreadContext.Stacks["Operation"].Push("UploadCloudFile"))
 741             {
 742                 try
 743                 {
 744
 745                     var accountInfo = action.AccountInfo;
 746
 747                     var fileInfo = action.LocalFile;
 748
 749                     if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
 750                         return;
 751
 752                     //
 753                     if (action.FileState == null)
 754                         action.FileState = StatusKeeper.GetStateByFilePath(fileInfo.FullName);
 755                     if (action.FileState == null)
 756                     {
 757                         Log.WarnFormat("File [{0}] has no local state. It was probably created by a download action",fileInfo.FullName);
 758                         return;
 759                     }
 760                     //Do not upload files in conflict
 761                     if (action.FileState.FileStatus == FileStatus.Conflict )
 762                     {
 763                         Log.InfoFormat("Skipping file in conflict [{0}]",fileInfo.FullName);
 764                         return;
 765                     }
 766                     if (action.FileState.FileStatus == FileStatus.Forbidden)
 767                     {
 768                         Log.InfoFormat("Skipping forbidden file [{0}]",fileInfo.FullName);
 769                         return;
 770                     }
 771
 772                     var relativePath = fileInfo.AsRelativeTo(accountInfo.AccountPath);
 773                     if (relativePath.StartsWith(FolderConstants.OthersFolder))
 774                     {
 775                         var parts = relativePath.Split('\\');
 776                         var accountName = parts[1];
 777                         var oldName = accountInfo.UserName;
 778                         var absoluteUri = accountInfo.StorageUri.AbsoluteUri;
 779                         var nameIndex = absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
 780                         var root = absoluteUri.Substring(0, nameIndex);
 781
 782                         accountInfo = new AccountInfo
 783                                           {
 784                                               UserName = accountName,
 785                                               AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 786                                               StorageUri = new Uri(root + accountName),
 787                                               BlockHash = accountInfo.BlockHash,
 788                                               BlockSize = accountInfo.BlockSize,
 789                                               Token = accountInfo.Token
 790                                           };
 791                     }
 792
 793
 794                     var fullFileName = fileInfo.GetProperCapitalization();
 795                     using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))
 796                     {
 797                         //Abort if the file is already being uploaded or downloaded
 798                         if (gate.Failed)
 799                             return;
 800
 801                         var cloudFile = action.CloudFile;
 802                         var account = cloudFile.Account ?? accountInfo.UserName;
 803                         try
 804                         {
 805
 806                         var client = new CloudFilesClient(accountInfo);
 807                         //Even if GetObjectInfo times out, we can proceed with the upload
 808                         var cloudInfo = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
 809
 810                         //If this is a read-only file, do not upload changes
 811                         if (cloudInfo.AllowedTo == "read")
 812                             return;
 813
 814                         //TODO: Check how a directory hash is calculated -> All dirs seem to have the same hash
 815                             if (fileInfo is DirectoryInfo)
 816                             {
 817                                 //If the directory doesn't exist the Hash property will be empty
 818                                 if (String.IsNullOrWhiteSpace(cloudInfo.Hash))
 819                                     //Go on and create the directory
 820                                     await
 821                                         client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,
 822                                                          String.Empty, "application/directory");
 823                             }
 824                             else
 825                             {
 826
 827                                 var cloudHash = cloudInfo.Hash.ToLower();
 828
 829                                 StatusNotification.Notify(new StatusNotification(String.Format("Hashing {0} for Upload",fileInfo.Name)));
 830
 831                                 //TODO: This is the same as the calculation for the Local Hash!
 832                                 //First, calculate the tree hash
 833 /*
 834                                 var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize,
 835                                                                                       accountInfo.BlockHash, 2);
 836 */
 837
 838
 839                                 var treeHash = action.TreeHash.Value;
 840                                 var topHash = treeHash.TopHash.ToHashString();
 841
 842                                 //var topHash = action.TopHash.Value;
 843
 844                                 //If the file hashes match, abort the upload
 845                                 if (topHash == cloudHash /*|| topHash == cloudHash*/)
 846                                 {
 847                                     //but store any metadata changes
 848                                     StatusKeeper.StoreInfo(fullFileName, cloudInfo);
 849                                     Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
 850                                     return;
 851                                 }
 852
 853
 854                                 //Mark the file as modified while we upload it
 855                                 StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
 856                                 //And then upload it
 857
 858                                 //Upload even small files using the Hashmap. The server may already contain
 859                                 //the relevant block
 860
 861                                 //TODO: If the upload fails with a 403, abort it and mark conflict
 862
 863                                 await
 864                                     UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash);
 865                             }
 866                             //If everything succeeds, change the file and overlay status to normal
 867                             StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal);
 868                         }
 869                         catch (WebException exc)
 870                         {
 871                             var response=(exc.Response as HttpWebResponse);
 872                             if (response == null)
 873                                 throw;
 874                             if (response.StatusCode == HttpStatusCode.Forbidden)
 875                             {
 876                                 StatusKeeper.SetFileState(fileInfo.FullName,FileStatus.Forbidden, FileOverlayStatus.Conflict);
 877                             }
 878                         }
 879                     }
 880                     //Notify the Shell to update the overlays
 881                     NativeMethods.RaiseChangeNotification(fullFileName);
 882                     StatusNotification.NotifyChangedFile(fullFileName);
 883                 }
 884                 catch (AggregateException ex)
 885                 {
 886                     var exc = ex.InnerException as WebException;
 887                     if (exc == null)
 888                         throw ex.InnerException;
 889                     if (HandleUploadWebException(action, exc))
 890                         return;
 891                     throw;
 892                 }
 893                 catch (WebException ex)
 894                 {
 895                     if (HandleUploadWebException(action, ex))
 896                         return;
 897                     throw;
 898                 }
 899                 catch (Exception ex)
 900                 {
 901                     Log.Error("Unexpected error while uploading file", ex);
 902                     throw;
 903                 }
 904             }
 905         }
 906
 907
 908
 909         private bool HandleUploadWebException(CloudAction action, WebException exc)
 910         {
 911             var response = exc.Response as HttpWebResponse;
 912             if (response == null)
 913                 throw exc;
 914             if (response.StatusCode == HttpStatusCode.Unauthorized)
 915             {
 916                 Log.Error("Not allowed to upload file", exc);
 917                 var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);
 918                 StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal);
 919                 StatusNotification.NotifyChange(message, TraceLevel.Warning);
 920                 return true;
 921             }
 922             return false;
 923         }
 924
 925         public async Task UploadWithHashMap(AccountInfo accountInfo,ObjectInfo cloudFile,FileInfo fileInfo,string url,TreeHash treeHash)
 926         {
 927             if (accountInfo == null)
 928                 throw new ArgumentNullException("accountInfo");
 929             if (cloudFile==null)
 930                 throw new ArgumentNullException("cloudFile");
 931             if (fileInfo == null)
 932                 throw new ArgumentNullException("fileInfo");
 933             if (String.IsNullOrWhiteSpace(url))
 934                 throw new ArgumentNullException(url);
 935             if (treeHash==null)
 936                 throw new ArgumentNullException("treeHash");
 937             if (String.IsNullOrWhiteSpace(cloudFile.Container) )
 938                 throw new ArgumentException("Invalid container","cloudFile");
 939             Contract.EndContractBlock();
 940
 941             StatusNotification.Notify(new StatusNotification(String.Format("Uploading {0}", fileInfo.Name)));
 942
 943             var fullFileName = fileInfo.GetProperCapitalization();
 944
 945             var account = cloudFile.Account ?? accountInfo.UserName;
 946             var container = cloudFile.Container ;
 947
 948             var client = new CloudFilesClient(accountInfo);
 949             //Send the hashmap to the server
 950             var missingHashes =  await client.PutHashMap(account, container, url, treeHash);
 951             int block = 0;
 952             ReportUploadProgress(fileInfo.Name,block++, missingHashes.Count, fileInfo.Length);
 953             //If the server returns no missing hashes, we are done
 954             while (missingHashes.Count > 0)
 955             {
 956
 957                 var buffer = new byte[accountInfo.BlockSize];
 958                 foreach (var missingHash in missingHashes)
 959                 {
 960                     //Find the proper block
 961                     var blockIndex = treeHash.HashDictionary[missingHash];
 962                     long offset = blockIndex*accountInfo.BlockSize;
 963
 964                     var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);
 965
 966                     try
 967                     {
 968                         //And upload the block
 969                         await client.PostBlock(account, container, buffer, 0, read);
 970                         Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);
 971                     }
 972                     catch (Exception exc)
 973                     {
 974                         Log.Error(String.Format("Uploading block {0} of {1}", blockIndex, fullFileName), exc);
 975                     }
 976                     ReportUploadProgress(fileInfo.Name,block++, missingHashes.Count, fileInfo.Length);
 977                 }
 978
 979                 //Repeat until there are no more missing hashes
 980                 missingHashes = await client.PutHashMap(account, container, url, treeHash);
 981             }
 982
 983             ReportUploadProgress(fileInfo.Name, missingHashes.Count, missingHashes.Count, fileInfo.Length);
 984         }
 985
 986         private void ReportUploadProgress(string fileName,int block, int totalBlocks, long fileSize)
 987         {
 988             StatusNotification.Notify(totalBlocks == 0
 989                                           ? new ProgressNotification(fileName, "Uploading", 1, 1, fileSize)
 990                                           : new ProgressNotification(fileName, "Uploading", block, totalBlocks, fileSize));
 991         }
 992
 993         private void ReportDownloadProgress(string fileName,int block, int totalBlocks, long fileSize)
 994         {
 995             StatusNotification.Notify(totalBlocks == 0
 996                                           ? new ProgressNotification(fileName, "Downloading", 1, 1, fileSize)
 997                                           : new ProgressNotification(fileName, "Downloading", block, totalBlocks, fileSize));
 998         }
 999     }
1000
1001
1002
1003
1004 }