code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/Downloader.cs

   1 using System;
   2 using System.ComponentModel.Composition;
   3 using System.Diagnostics.Contracts;
   4 using System.IO;
   5 using System.Reflection;
   6 using System.Threading.Tasks;
   7 using Pithos.Interfaces;
   8 using Pithos.Network;
   9 using log4net;
  10
  11 namespace Pithos.Core.Agents
  12 {
  13     [Export(typeof(Downloader))]
  14     public class Downloader
  15     {
  16         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  17
  18         [Import]
  19         private IStatusKeeper StatusKeeper { get; set; }
  20
  21
  22         public IStatusNotification StatusNotification { get; set; }
  23
  24         //Download a file.
  25         public async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile, string filePath)
  26         {
  27             if (accountInfo == null)
  28                 throw new ArgumentNullException("accountInfo");
  29             if (cloudFile == null)
  30                 throw new ArgumentNullException("cloudFile");
  31             if (String.IsNullOrWhiteSpace(cloudFile.Account))
  32                 throw new ArgumentNullException("cloudFile");
  33             if (String.IsNullOrWhiteSpace(cloudFile.Container))
  34                 throw new ArgumentNullException("cloudFile");
  35             if (String.IsNullOrWhiteSpace(filePath))
  36                 throw new ArgumentNullException("filePath");
  37             if (!Path.IsPathRooted(filePath))
  38                 throw new ArgumentException("The filePath must be rooted", "filePath");
  39             Contract.EndContractBlock();
  40
  41             using (ThreadContext.Stacks["Operation"].Push("DownloadCloudFile"))
  42             {
  43
  44                 var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath);
  45                 var relativeUrl = new Uri(cloudFile.Name, UriKind.Relative);
  46
  47                 var url = relativeUrl.ToString();
  48                 if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase))
  49                     return;
  50
  51
  52                 //Are we already downloading or uploading the file?
  53                 using (var gate = NetworkGate.Acquire(localPath, NetworkOperation.Downloading))
  54                 {
  55                     if (gate.Failed)
  56                         return;
  57
  58                     var client = new CloudFilesClient(accountInfo);
  59                     var account = cloudFile.Account;
  60                     var container = cloudFile.Container;
  61
  62                     if (cloudFile.IsDirectory)
  63                     {
  64                         if (!Directory.Exists(localPath))
  65                             try
  66                             {
  67                                 Directory.CreateDirectory(localPath);
  68                                 if (Log.IsDebugEnabled)
  69                                     Log.DebugFormat("Created Directory [{0}]", localPath);
  70                             }
  71                             catch (IOException)
  72                             {
  73                                 var localInfo = new FileInfo(localPath);
  74                                 if (localInfo.Exists && localInfo.Length == 0)
  75                                 {
  76                                     Log.WarnFormat("Malformed directory object detected for [{0}]", localPath);
  77                                     localInfo.Delete();
  78                                     Directory.CreateDirectory(localPath);
  79                                     if (Log.IsDebugEnabled)
  80                                         Log.DebugFormat("Created Directory [{0}]", localPath);
  81                                 }
  82                             }
  83                     }
  84                     else
  85                     {
  86                         var isChanged = IsObjectChanged(cloudFile, localPath);
  87                         if (isChanged)
  88                         {
  89                             //Retrieve the hashmap from the server
  90                             var serverHash = await client.GetHashMap(account, container, url);
  91                             //If it's a small file
  92                             if (serverHash.Hashes.Count == 1)
  93                                 //Download it in one go
  94                                 await
  95                                     DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath);
  96                             //Otherwise download it block by block
  97                             else
  98                                 await
  99                                     DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath,
 100                                                        serverHash);
 101
 102                             if (!cloudFile.IsWritable(accountInfo.UserName))
 103                             {
 104                                 var attributes = File.GetAttributes(localPath);
 105                                 File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly);
 106                             }
 107                         }
 108                     }
 109
 110                     //Now we can store the object's metadata without worrying about ghost status entries
 111                     StatusKeeper.StoreInfo(localPath, cloudFile);
 112
 113                 }
 114             }
 115         }
 116
 117         //Download a file asynchronously using blocks
 118         public async Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath, TreeHash serverHash)
 119         {
 120             if (client == null)
 121                 throw new ArgumentNullException("client");
 122             if (cloudFile == null)
 123                 throw new ArgumentNullException("cloudFile");
 124             if (relativeUrl == null)
 125                 throw new ArgumentNullException("relativeUrl");
 126             if (String.IsNullOrWhiteSpace(filePath))
 127                 throw new ArgumentNullException("filePath");
 128             if (!Path.IsPathRooted(filePath))
 129                 throw new ArgumentException("The filePath must be rooted", "filePath");
 130             if (serverHash == null)
 131                 throw new ArgumentNullException("serverHash");
 132             if (cloudFile.IsDirectory)
 133                 throw new ArgumentException("cloudFile is a directory, not a file", "cloudFile");
 134             Contract.EndContractBlock();
 135
 136             var fileAgent = GetFileAgent(accountInfo);
 137             var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath);
 138
 139             //Calculate the relative file path for the new file
 140             var relativePath = relativeUrl.RelativeUriToFilePath();
 141             var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash);
 142
 143
 144             StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing, String.Format("Calculating hashmap for {0} before download", Path.GetFileName(localPath)));
 145             //Calculate the file's treehash
 146             var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash, 2);
 147
 148             //And compare it with the server's hash
 149             var upHashes = serverHash.GetHashesAsStrings();
 150             var localHashes = treeHash.HashDictionary;
 151             ReportDownloadProgress(Path.GetFileName(localPath), 0, upHashes.Length, cloudFile.Bytes);
 152             for (int i = 0; i < upHashes.Length; i++)
 153             {
 154                 //For every non-matching hash
 155                 var upHash = upHashes[i];
 156                 if (!localHashes.ContainsKey(upHash))
 157                 {
 158                     StatusNotification.Notify(new CloudNotification { Data = cloudFile });
 159
 160                     if (blockUpdater.UseOrphan(i, upHash))
 161                     {
 162                         Log.InfoFormat("[BLOCK GET] ORPHAN FOUND for {0} of {1} for {2}", i, upHashes.Length, localPath);
 163                         continue;
 164                     }
 165                     Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath);
 166                     var start = i * serverHash.BlockSize;
 167                     //To download the last block just pass a null for the end of the range
 168                     long? end = null;
 169                     if (i < upHashes.Length - 1)
 170                         end = ((i + 1) * serverHash.BlockSize);
 171
 172                     //Download the missing block
 173                     var block = await client.GetBlock(cloudFile.Account, cloudFile.Container, relativeUrl, start, end);
 174
 175                     //and store it
 176                     blockUpdater.StoreBlock(i, block);
 177
 178
 179                     Log.InfoFormat("[BLOCK GET] FINISH {0} of {1} for {2}", i, upHashes.Length, localPath);
 180                 }
 181                 ReportDownloadProgress(Path.GetFileName(localPath), i, upHashes.Length, cloudFile.Bytes);
 182             }
 183
 184             //Want to avoid notifications if no changes were made
 185             var hasChanges = blockUpdater.HasBlocks;
 186             blockUpdater.Commit();
 187
 188             if (hasChanges)
 189                 //Notify listeners that a local file has changed
 190                 StatusNotification.NotifyChangedFile(localPath);
 191
 192             Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath);
 193         }
 194
 195         //Download a small file with a single GET operation
 196         private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath)
 197         {
 198             if (client == null)
 199                 throw new ArgumentNullException("client");
 200             if (cloudFile == null)
 201                 throw new ArgumentNullException("cloudFile");
 202             if (relativeUrl == null)
 203                 throw new ArgumentNullException("relativeUrl");
 204             if (String.IsNullOrWhiteSpace(filePath))
 205                 throw new ArgumentNullException("filePath");
 206             if (!Path.IsPathRooted(filePath))
 207                 throw new ArgumentException("The localPath must be rooted", "filePath");
 208             if (cloudFile.IsDirectory)
 209                 throw new ArgumentException("cloudFile is a directory, not a file", "cloudFile");
 210             Contract.EndContractBlock();
 211
 212             var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath);
 213             StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing, String.Format("Downloading {0}", Path.GetFileName(localPath)));
 214             StatusNotification.Notify(new CloudNotification { Data = cloudFile });
 215
 216             var fileAgent = GetFileAgent(accountInfo);
 217             //Calculate the relative file path for the new file
 218             var relativePath = relativeUrl.RelativeUriToFilePath();
 219             //The file will be stored in a temporary location while downloading with an extension .download
 220             var tempPath = Path.Combine(fileAgent.CachePath, relativePath + ".download");
 221             //Make sure the target folder exists. DownloadFileTask will not create the folder
 222             var tempFolder = Path.GetDirectoryName(tempPath);
 223             if (!Directory.Exists(tempFolder))
 224                 Directory.CreateDirectory(tempFolder);
 225
 226             //Download the object to the temporary location
 227             await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl.ToString(), tempPath);
 228
 229             //Create the local folder if it doesn't exist (necessary for shared objects)
 230             var localFolder = Path.GetDirectoryName(localPath);
 231             if (!Directory.Exists(localFolder))
 232                 try
 233                 {
 234                     Directory.CreateDirectory(localFolder);
 235                 }
 236                 catch (IOException)
 237                 {
 238                     //A file may already exist that has the same name as the new folder.
 239                     //This may be an artifact of the way Pithos handles directories
 240                     var fileInfo = new FileInfo(localFolder);
 241                     if (fileInfo.Exists && fileInfo.Length == 0)
 242                     {
 243                         Log.WarnFormat("Malformed directory object detected for [{0}]", localFolder);
 244                         fileInfo.Delete();
 245                         Directory.CreateDirectory(localFolder);
 246                     }
 247                     else
 248                         throw;
 249                 }
 250             //And move it to its actual location once downloading is finished
 251             if (File.Exists(localPath))
 252                 File.Replace(tempPath, localPath, null, true);
 253             else
 254                 File.Move(tempPath, localPath);
 255             //Notify listeners that a local file has changed
 256             StatusNotification.NotifyChangedFile(localPath);
 257
 258
 259         }
 260
 261
 262         private void ReportDownloadProgress(string fileName, int block, int totalBlocks, long fileSize)
 263         {
 264             StatusNotification.Notify(totalBlocks == 0
 265                                           ? new ProgressNotification(fileName, "Downloading", 1, 1, fileSize)
 266                                           : new ProgressNotification(fileName, "Downloading", block, totalBlocks, fileSize));
 267         }
 268
 269         private bool IsObjectChanged(ObjectInfo cloudFile, string localPath)
 270         {
 271             //If the target is a directory, there are no changes to download
 272             if (Directory.Exists(localPath))
 273                 return false;
 274             //If the file doesn't exist, we have a chagne
 275             if (!File.Exists(localPath))
 276                 return true;
 277             //If there is no stored state, we have a change
 278             var localState = StatusKeeper.GetStateByFilePath(localPath);
 279             if (localState == null)
 280                 return true;
 281
 282             var info = new FileInfo(localPath);
 283             var shortHash = info.ComputeShortHash();
 284             //If the file is different from the stored state, we have a change
 285             if (localState.ShortHash != shortHash)
 286                 return true;
 287             //If the top hashes differ, we have a change
 288             return (localState.Checksum != cloudFile.Hash);
 289         }
 290
 291         private static FileAgent GetFileAgent(AccountInfo accountInfo)
 292         {
 293             return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);
 294         }
 295
 296
 297     }
 298 }