code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/Uploader.cs

   1 using System;
   2 using System.ComponentModel.Composition;
   3 using System.Diagnostics;
   4 using System.Diagnostics.Contracts;
   5 using System.IO;
   6 using System.Net;
   7 using System.Reflection;
   8 using System.Threading.Tasks;
   9 using Pithos.Interfaces;
  10 using Pithos.Network;
  11 using log4net;
  12
  13 namespace Pithos.Core.Agents
  14 {
  15     [Export(typeof(Uploader))]
  16     public class Uploader
  17     {
  18         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  19
  20         [Import]
  21         private IStatusKeeper StatusKeeper { get; set; }
  22
  23
  24         public IStatusNotification StatusNotification { get; set; }
  25
  26         public async Task UploadCloudFile(CloudAction action)
  27         {
  28             if (action == null)
  29                 throw new ArgumentNullException("action");
  30             Contract.EndContractBlock();
  31
  32             using (ThreadContext.Stacks["Operation"].Push("UploadCloudFile"))
  33             {
  34                 try
  35                 {
  36                     var fileInfo = action.LocalFile;
  37
  38                     if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
  39                         return;
  40
  41                     //Try to load the action's local state, if it is empty
  42                     if (action.FileState == null)
  43                         action.FileState = StatusKeeper.GetStateByFilePath(fileInfo.FullName);
  44                     if (action.FileState == null)
  45                     {
  46                         Log.WarnFormat("File [{0}] has no local state. It was probably created by a download action", fileInfo.FullName);
  47                         return;
  48                     }
  49
  50                     var latestState = action.FileState;
  51
  52                     //Do not upload files in conflict
  53                     if (latestState.FileStatus == FileStatus.Conflict)
  54                     {
  55                         Log.InfoFormat("Skipping file in conflict [{0}]", fileInfo.FullName);
  56                         return;
  57                     }
  58                     //Do not upload files when we have no permission
  59                     if (latestState.FileStatus == FileStatus.Forbidden)
  60                     {
  61                         Log.InfoFormat("Skipping forbidden file [{0}]", fileInfo.FullName);
  62                         return;
  63                     }
  64
  65                     //Are we targeting our own account or a sharer account?
  66                     var relativePath = fileInfo.AsRelativeTo(action.AccountInfo.AccountPath);
  67                     var accountInfo = relativePath.StartsWith(FolderConstants.OthersFolder)
  68                                                   ? GetSharerAccount(relativePath, action.AccountInfo)
  69                                                   : action.AccountInfo;
  70
  71
  72
  73                     var fullFileName = fileInfo.GetProperCapitalization();
  74                     using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))
  75                     {
  76                         //Abort if the file is already being uploaded or downloaded
  77                         if (gate.Failed)
  78                             return;
  79
  80                         var cloudFile = action.CloudFile;
  81                         var account = cloudFile.Account ?? accountInfo.UserName;
  82                         try
  83                         {
  84
  85                             var client = new CloudFilesClient(accountInfo);
  86                             //Even if GetObjectInfo times out, we can proceed with the upload
  87                             var cloudInfo = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
  88
  89                             //If this is a read-only file, do not upload changes
  90                             if (!cloudInfo.IsWritable(action.AccountInfo.UserName))
  91                                 return;
  92
  93                                 //TODO: If the object does not exist, check that we can upload to the folder
  94
  95                             if (fileInfo is DirectoryInfo)
  96                             {
  97                                 //If the directory doesn't exist the Hash property will be empty
  98                                 if (String.IsNullOrWhiteSpace(cloudInfo.Hash))
  99                                     //Go on and create the directory
 100                                     await client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,
 101                                                          String.Empty, "application/directory");
 102                             }
 103                             else
 104                             {
 105
 106                                 var cloudHash = cloudInfo.Hash.ToLower();
 107
 108                                 string topHash;
 109                                 TreeHash treeHash;
 110                                 using(StatusNotification.GetNotifier("Hashing {0} for Upload", "Finished hashing {0}",fileInfo.Name))
 111                                 {
 112                                     treeHash = action.TreeHash.Value;
 113                                     topHash = treeHash.TopHash.ToHashString();
 114                                 }
 115
 116                                 //If the file hashes match, abort the upload
 117                                 if (cloudInfo != ObjectInfo.Empty && topHash == cloudHash)
 118                                 {
 119                                     //but store any metadata changes
 120                                     StatusKeeper.StoreInfo(fullFileName, cloudInfo);
 121                                     Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
 122                                     return;
 123                                 }
 124
 125
 126                                 //Mark the file as modified while we upload it
 127                                 StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
 128                                 //And then upload it
 129
 130                                 //Upload even small files using the Hashmap. The server may already contain
 131                                 //the relevant block
 132
 133                                 await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash);
 134                             }
 135                             //If everything succeeds, change the file and overlay status to normal
 136                             StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");
 137                         }
 138                         catch (WebException exc)
 139                         {
 140                             var response = (exc.Response as HttpWebResponse);
 141                             if (response == null)
 142                                 throw;
 143                             if (response.StatusCode == HttpStatusCode.Forbidden)
 144                             {
 145                                 StatusKeeper.SetFileState(fileInfo.FullName, FileStatus.Forbidden, FileOverlayStatus.Conflict, "Forbidden");
 146
 147                             }
 148                             else
 149                                 //In any other case, propagate the error
 150                                 throw;
 151                         }
 152                     }
 153                     //Notify the Shell to update the overlays
 154                     NativeMethods.RaiseChangeNotification(fullFileName);
 155                     StatusNotification.NotifyChangedFile(fullFileName);
 156                 }
 157                 catch (AggregateException ex)
 158                 {
 159                     var exc = ex.InnerException as WebException;
 160                     if (exc == null)
 161                         throw ex.InnerException;
 162                     if (HandleUploadWebException(action, exc))
 163                         return;
 164                     throw;
 165                 }
 166                 catch (WebException ex)
 167                 {
 168                     if (HandleUploadWebException(action, ex))
 169                         return;
 170                     throw;
 171                 }
 172                 catch (Exception ex)
 173                 {
 174                     Log.Error("Unexpected error while uploading file", ex);
 175                     throw;
 176                 }
 177             }
 178         }
 179
 180         private static AccountInfo GetSharerAccount(string relativePath, AccountInfo accountInfo)
 181         {
 182             var parts = relativePath.Split('\\');
 183             var accountName = parts[1];
 184             var oldName = accountInfo.UserName;
 185             var absoluteUri = accountInfo.StorageUri.AbsoluteUri;
 186             var nameIndex = absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
 187             var root = absoluteUri.Substring(0, nameIndex);
 188
 189             accountInfo = new AccountInfo
 190                               {
 191                                   UserName = accountName,
 192                                   AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 193                                   StorageUri = new Uri(root + accountName),
 194                                   BlockHash = accountInfo.BlockHash,
 195                                   BlockSize = accountInfo.BlockSize,
 196                                   Token = accountInfo.Token
 197                               };
 198             return accountInfo;
 199         }
 200
 201
 202         public async Task UploadWithHashMap(AccountInfo accountInfo, ObjectInfo cloudFile, FileInfo fileInfo, string url, TreeHash treeHash)
 203         {
 204             if (accountInfo == null)
 205                 throw new ArgumentNullException("accountInfo");
 206             if (cloudFile == null)
 207                 throw new ArgumentNullException("cloudFile");
 208             if (fileInfo == null)
 209                 throw new ArgumentNullException("fileInfo");
 210             if (String.IsNullOrWhiteSpace(url))
 211                 throw new ArgumentNullException(url);
 212             if (treeHash == null)
 213                 throw new ArgumentNullException("treeHash");
 214             if (String.IsNullOrWhiteSpace(cloudFile.Container))
 215                 throw new ArgumentException("Invalid container", "cloudFile");
 216             Contract.EndContractBlock();
 217
 218             using (StatusNotification.GetNotifier("Uploading {0}", "Finished Uploading {0}", fileInfo.Name))
 219             {
 220
 221                 var fullFileName = fileInfo.GetProperCapitalization();
 222
 223                 var account = cloudFile.Account ?? accountInfo.UserName;
 224                 var container = cloudFile.Container;
 225
 226                 var client = new CloudFilesClient(accountInfo);
 227                 //Send the hashmap to the server
 228                 var missingHashes = await client.PutHashMap(account, container, url, treeHash);
 229                 int block = 0;
 230                 ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);
 231                 //If the server returns no missing hashes, we are done
 232                 while (missingHashes.Count > 0)
 233                 {
 234
 235                     var buffer = new byte[accountInfo.BlockSize];
 236                     foreach (var missingHash in missingHashes)
 237                     {
 238                         //Find the proper block
 239                         var blockIndex = treeHash.HashDictionary[missingHash];
 240                         long offset = blockIndex*accountInfo.BlockSize;
 241
 242                         var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);
 243
 244                         try
 245                         {
 246                             //And upload the block
 247                             await client.PostBlock(account, container, buffer, 0, read);
 248                             Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);
 249                         }
 250                         catch (Exception exc)
 251                         {
 252                             Log.Error(String.Format("Uploading block {0} of {1}", blockIndex, fullFileName), exc);
 253                         }
 254                         ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);
 255                     }
 256
 257                     //Repeat until there are no more missing hashes
 258                     missingHashes = await client.PutHashMap(account, container, url, treeHash);
 259                 }
 260
 261                 ReportUploadProgress(fileInfo.Name, missingHashes.Count, missingHashes.Count, fileInfo.Length);
 262             }
 263         }
 264
 265         private void ReportUploadProgress(string fileName, int block, int totalBlocks, long fileSize)
 266         {
 267             StatusNotification.Notify(totalBlocks == 0
 268                                           ? new ProgressNotification(fileName, "Uploading", 1, 1, fileSize)
 269                                           : new ProgressNotification(fileName, "Uploading", block, totalBlocks, fileSize));
 270         }
 271
 272
 273         private bool HandleUploadWebException(CloudAction action, WebException exc)
 274         {
 275             var response = exc.Response as HttpWebResponse;
 276             if (response == null)
 277                 throw exc;
 278             if (response.StatusCode == HttpStatusCode.Unauthorized)
 279             {
 280                 Log.Error("Not allowed to upload file", exc);
 281                 var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);
 282                 StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");
 283                 StatusNotification.NotifyChange(message, TraceLevel.Warning);
 284                 return true;
 285             }
 286             return false;
 287         }
 288     }
 289 }