code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/Uploader.cs

   1 using System;
   2 using System.ComponentModel.Composition;
   3 using System.Diagnostics;
   4 using System.Diagnostics.Contracts;
   5 using System.IO;
   6 using System.Net;
   7 using System.Reflection;
   8 using System.Threading.Tasks;
   9 using Pithos.Interfaces;
  10 using Pithos.Network;
  11 using log4net;
  12
  13 namespace Pithos.Core.Agents
  14 {
  15     [Export(typeof(Uploader))]
  16     public class Uploader
  17     {
  18         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  19
  20         [Import]
  21         private IStatusKeeper StatusKeeper { get; set; }
  22
  23
  24         public IStatusNotification StatusNotification { get; set; }
  25
  26         public async Task UploadCloudFile(CloudAction action)
  27         {
  28             if (action == null)
  29                 throw new ArgumentNullException("action");
  30             Contract.EndContractBlock();
  31
  32             using (ThreadContext.Stacks["Operation"].Push("UploadCloudFile"))
  33             {
  34                 try
  35                 {
  36                     var fileInfo = action.LocalFile;
  37
  38                     if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
  39                         return;
  40
  41                     //Try to load the action's local state, if it is empty
  42                     if (action.FileState == null)
  43                         action.FileState = StatusKeeper.GetStateByFilePath(fileInfo.FullName);
  44                     if (action.FileState == null)
  45                     {
  46                         Log.WarnFormat("File [{0}] has no local state. It was probably created by a download action", fileInfo.FullName);
  47                         return;
  48                     }
  49
  50                     var latestState = action.FileState;
  51
  52                     //Do not upload files in conflict
  53                     if (latestState.FileStatus == FileStatus.Conflict)
  54                     {
  55                         Log.InfoFormat("Skipping file in conflict [{0}]", fileInfo.FullName);
  56                         return;
  57                     }
  58                     //Do not upload files when we have no permission
  59                     if (latestState.FileStatus == FileStatus.Forbidden)
  60                     {
  61                         Log.InfoFormat("Skipping forbidden file [{0}]", fileInfo.FullName);
  62                         return;
  63                     }
  64
  65                     //Are we targeting our own account or a sharer account?
  66                     var relativePath = fileInfo.AsRelativeTo(action.AccountInfo.AccountPath);
  67                     var accountInfo = relativePath.StartsWith(FolderConstants.OthersFolder)
  68                                                   ? GetSharerAccount(relativePath, action.AccountInfo)
  69                                                   : action.AccountInfo;
  70
  71
  72
  73                     var fullFileName = fileInfo.GetProperCapitalization();
  74                     using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))
  75                     {
  76                         //Abort if the file is already being uploaded or downloaded
  77                         if (gate.Failed)
  78                             return;
  79
  80                         var cloudFile = action.CloudFile;
  81                         var account = cloudFile.Account ?? accountInfo.UserName;
  82                         try
  83                         {
  84
  85                             var client = new CloudFilesClient(accountInfo);
  86                             //Even if GetObjectInfo times out, we can proceed with the upload
  87                             var cloudInfo = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
  88
  89                             //If this is a read-only file, do not upload changes
  90                             if (!cloudInfo.IsWritable(action.AccountInfo.UserName))
  91                                 return;
  92
  93                             if (fileInfo is DirectoryInfo)
  94                             {
  95                                 //If the directory doesn't exist the Hash property will be empty
  96                                 if (String.IsNullOrWhiteSpace(cloudInfo.Hash))
  97                                     //Go on and create the directory
  98                                     await client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,
  99                                                          String.Empty, "application/directory");
 100                             }
 101                             else
 102                             {
 103
 104                                 var cloudHash = cloudInfo.Hash.ToLower();
 105
 106                                 string topHash;
 107                                 TreeHash treeHash;
 108                                 using(StatusNotification.GetNotifier("Hashing {0} for Upload", "Finished hashing {0}",fileInfo.Name))
 109                                 {
 110                                     treeHash = action.TreeHash.Value;
 111                                     topHash = treeHash.TopHash.ToHashString();
 112                                 }
 113
 114                                 //If the file hashes match, abort the upload
 115                                 if (cloudInfo != ObjectInfo.Empty && topHash == cloudHash)
 116                                 {
 117                                     //but store any metadata changes
 118                                     StatusKeeper.StoreInfo(fullFileName, cloudInfo);
 119                                     Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
 120                                     return;
 121                                 }
 122
 123
 124                                 //Mark the file as modified while we upload it
 125                                 StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
 126                                 //And then upload it
 127
 128                                 //Upload even small files using the Hashmap. The server may already contain
 129                                 //the relevant block
 130
 131                                 await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash);
 132                             }
 133                             //If everything succeeds, change the file and overlay status to normal
 134                             StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");
 135                         }
 136                         catch (WebException exc)
 137                         {
 138                             var response = (exc.Response as HttpWebResponse);
 139                             if (response == null)
 140                                 throw;
 141                             if (response.StatusCode == HttpStatusCode.Forbidden)
 142                             {
 143                                 StatusKeeper.SetFileState(fileInfo.FullName, FileStatus.Forbidden, FileOverlayStatus.Conflict, "Forbidden");
 144
 145                             }
 146                             else
 147                                 //In any other case, propagate the error
 148                                 throw;
 149                         }
 150                     }
 151                     //Notify the Shell to update the overlays
 152                     NativeMethods.RaiseChangeNotification(fullFileName);
 153                     StatusNotification.NotifyChangedFile(fullFileName);
 154                 }
 155                 catch (AggregateException ex)
 156                 {
 157                     var exc = ex.InnerException as WebException;
 158                     if (exc == null)
 159                         throw ex.InnerException;
 160                     if (HandleUploadWebException(action, exc))
 161                         return;
 162                     throw;
 163                 }
 164                 catch (WebException ex)
 165                 {
 166                     if (HandleUploadWebException(action, ex))
 167                         return;
 168                     throw;
 169                 }
 170                 catch (Exception ex)
 171                 {
 172                     Log.Error("Unexpected error while uploading file", ex);
 173                     throw;
 174                 }
 175             }
 176         }
 177
 178         private static AccountInfo GetSharerAccount(string relativePath, AccountInfo accountInfo)
 179         {
 180             var parts = relativePath.Split('\\');
 181             var accountName = parts[1];
 182             var oldName = accountInfo.UserName;
 183             var absoluteUri = accountInfo.StorageUri.AbsoluteUri;
 184             var nameIndex = absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
 185             var root = absoluteUri.Substring(0, nameIndex);
 186
 187             accountInfo = new AccountInfo
 188                               {
 189                                   UserName = accountName,
 190                                   AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 191                                   StorageUri = new Uri(root + accountName),
 192                                   BlockHash = accountInfo.BlockHash,
 193                                   BlockSize = accountInfo.BlockSize,
 194                                   Token = accountInfo.Token
 195                               };
 196             return accountInfo;
 197         }
 198
 199
 200         public async Task UploadWithHashMap(AccountInfo accountInfo, ObjectInfo cloudFile, FileInfo fileInfo, string url, TreeHash treeHash)
 201         {
 202             if (accountInfo == null)
 203                 throw new ArgumentNullException("accountInfo");
 204             if (cloudFile == null)
 205                 throw new ArgumentNullException("cloudFile");
 206             if (fileInfo == null)
 207                 throw new ArgumentNullException("fileInfo");
 208             if (String.IsNullOrWhiteSpace(url))
 209                 throw new ArgumentNullException(url);
 210             if (treeHash == null)
 211                 throw new ArgumentNullException("treeHash");
 212             if (String.IsNullOrWhiteSpace(cloudFile.Container))
 213                 throw new ArgumentException("Invalid container", "cloudFile");
 214             Contract.EndContractBlock();
 215
 216             using (StatusNotification.GetNotifier("Uploading {0}", "Finished Uploading {0}", fileInfo.Name))
 217             {
 218
 219                 var fullFileName = fileInfo.GetProperCapitalization();
 220
 221                 var account = cloudFile.Account ?? accountInfo.UserName;
 222                 var container = cloudFile.Container;
 223
 224                 var client = new CloudFilesClient(accountInfo);
 225                 //Send the hashmap to the server
 226                 var missingHashes = await client.PutHashMap(account, container, url, treeHash);
 227                 int block = 0;
 228                 ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);
 229                 //If the server returns no missing hashes, we are done
 230                 while (missingHashes.Count > 0)
 231                 {
 232
 233                     var buffer = new byte[accountInfo.BlockSize];
 234                     foreach (var missingHash in missingHashes)
 235                     {
 236                         //Find the proper block
 237                         var blockIndex = treeHash.HashDictionary[missingHash];
 238                         long offset = blockIndex*accountInfo.BlockSize;
 239
 240                         var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);
 241
 242                         try
 243                         {
 244                             //And upload the block
 245                             await client.PostBlock(account, container, buffer, 0, read);
 246                             Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);
 247                         }
 248                         catch (Exception exc)
 249                         {
 250                             Log.Error(String.Format("Uploading block {0} of {1}", blockIndex, fullFileName), exc);
 251                         }
 252                         ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);
 253                     }
 254
 255                     //Repeat until there are no more missing hashes
 256                     missingHashes = await client.PutHashMap(account, container, url, treeHash);
 257                 }
 258
 259                 ReportUploadProgress(fileInfo.Name, missingHashes.Count, missingHashes.Count, fileInfo.Length);
 260             }
 261         }
 262
 263         private void ReportUploadProgress(string fileName, int block, int totalBlocks, long fileSize)
 264         {
 265             StatusNotification.Notify(totalBlocks == 0
 266                                           ? new ProgressNotification(fileName, "Uploading", 1, 1, fileSize)
 267                                           : new ProgressNotification(fileName, "Uploading", block, totalBlocks, fileSize));
 268         }
 269
 270
 271         private bool HandleUploadWebException(CloudAction action, WebException exc)
 272         {
 273             var response = exc.Response as HttpWebResponse;
 274             if (response == null)
 275                 throw exc;
 276             if (response.StatusCode == HttpStatusCode.Unauthorized)
 277             {
 278                 Log.Error("Not allowed to upload file", exc);
 279                 var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);
 280                 StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");
 281                 StatusNotification.NotifyChange(message, TraceLevel.Warning);
 282                 return true;
 283             }
 284             return false;
 285         }
 286     }
 287 }