code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/Uploader.cs

   1 using System;\r
   2 using System.Collections.Generic;\r
   3 using System.ComponentModel.Composition;\r
   4 using System.Diagnostics;\r
   5 using System.Diagnostics.Contracts;\r
   6 using System.IO;\r
   7 using System.Linq;\r
   8 using System.Net;\r
   9 using System.Reflection;\r
  10 using System.Security.Cryptography;\r
  11 using System.Threading;\r
  12 using System.Threading.Tasks;\r
  13 using Pithos.Interfaces;\r
  14 using Pithos.Network;\r
  15 using log4net;\r
  16 \r
  17 namespace Pithos.Core.Agents\r
  18 {\r
  19     [Export(typeof(Uploader))]\r
  20     public class Uploader\r
  21     {\r
  22         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  23 \r
  24         [Import]\r
  25         private IStatusKeeper StatusKeeper { get; set; }\r
  26 \r
  27         \r
  28         public IStatusNotification StatusNotification { get; set; }\r
  29 \r
  30         \r
  31         //CancellationTokenSource _cts = new CancellationTokenSource();\r
  32         /*public void SignalStop()\r
  33         {\r
  34             _cts.Cancel();\r
  35         }*/\r
  36 \r
  37         public async Task UploadCloudFile(CloudAction action,CancellationToken cancellationToken)\r
  38         {\r
  39             if (action == null)\r
  40                 throw new ArgumentNullException("action");\r
  41             Contract.EndContractBlock();\r
  42 \r
  43             using (ThreadContext.Stacks["Operation"].Push("UploadCloudFile"))\r
  44             {\r
  45                 try\r
  46                 {\r
  47                     await UnpauseEvent.WaitAsync();\r
  48 \r
  49                     var fileInfo = action.LocalFile;\r
  50 \r
  51                     if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))\r
  52                         return;\r
  53 \r
  54                     if (!Selectives.IsSelected(action.AccountInfo, fileInfo))\r
  55                         return;\r
  56 \r
  57                     //Try to load the action's local state, if it is empty\r
  58                     if (action.FileState == null)\r
  59                         action.FileState = StatusKeeper.GetStateByFilePath(fileInfo.FullName);\r
  60                     if (action.FileState == null)\r
  61                     {\r
  62                         Log.WarnFormat("File [{0}] has no local state. It was probably created by a download action", fileInfo.FullName);\r
  63                         return;\r
  64                     }\r
  65 \r
  66                     var latestState = action.FileState;\r
  67 \r
  68                     //Do not upload files in conflict\r
  69                     if (latestState.FileStatus == FileStatus.Conflict)\r
  70                     {\r
  71                         Log.InfoFormat("Skipping file in conflict [{0}]", fileInfo.FullName);\r
  72                         return;\r
  73                     }\r
  74                     //Do not upload files when we have no permission\r
  75                     if (latestState.FileStatus == FileStatus.Forbidden)\r
  76                     {\r
  77                         Log.InfoFormat("Skipping forbidden file [{0}]", fileInfo.FullName);\r
  78                         return;\r
  79                     }\r
  80 \r
  81                     //Are we targeting our own account or a sharer account?\r
  82                     var relativePath = fileInfo.AsRelativeTo(action.AccountInfo.AccountPath);\r
  83                     var accountInfo = relativePath.StartsWith(FolderConstants.OthersFolder) \r
  84                                                   ? GetSharerAccount(relativePath, action.AccountInfo) \r
  85                                                   : action.AccountInfo;\r
  86 \r
  87 \r
  88 \r
  89                     var fullFileName = fileInfo.GetProperCapitalization();\r
  90                     using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))\r
  91                     {\r
  92                         //Abort if the file is already being uploaded or downloaded\r
  93                         if (gate.Failed)\r
  94                             return;\r
  95 \r
  96                         var cloudFile = action.CloudFile;\r
  97                         var account = cloudFile.Account ?? accountInfo.UserName;\r
  98                         try\r
  99                         {\r
 100 \r
 101                             var client = new CloudFilesClient(accountInfo);\r
 102 \r
 103                             //Even if GetObjectInfo times out, we can proceed with the upload            \r
 104                             var cloudInfo = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);\r
 105 \r
 106                             //If this a shared file\r
 107                             if (!cloudFile.Account.Equals(action.AccountInfo.UserName,StringComparison.InvariantCultureIgnoreCase))\r
 108                             {\r
 109                                 \r
 110 /*\r
 111                                 if (!cloudInfo.IsWritable(action.AccountInfo.UserName))\r
 112                                 {\r
 113                                     MakeFileReadOnly(fullFileName);\r
 114                                     StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");\r
 115                                     return;\r
 116                                 }\r
 117 */\r
 118 \r
 119                                 //If this is a read-only file, do not upload changes\r
 120                                 if ( !cloudInfo.IsWritable(action.AccountInfo.UserName) ||\r
 121                                     //If the file is new, but we can't upload it\r
 122                                     (!cloudInfo.Exists && !client.CanUpload(account, cloudFile)) )\r
 123                                 {\r
 124                                     MakeFileReadOnly(fullFileName);\r
 125                                     StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");\r
 126                                     return;\r
 127                                 }\r
 128 \r
 129                             }\r
 130 \r
 131                             await UnpauseEvent.WaitAsync();\r
 132 \r
 133                             if (fileInfo is DirectoryInfo)\r
 134                             {\r
 135                                 //If the directory doesn't exist the Hash property will be empty\r
 136                                 if (String.IsNullOrWhiteSpace(cloudInfo.Hash))\r
 137                                     //Go on and create the directory\r
 138                                     await client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,\r
 139                                                          String.Empty, "application/directory");\r
 140                             }\r
 141                             else\r
 142                             {\r
 143 \r
 144                                 var cloudHash = cloudInfo.Hash.ToLower();\r
 145 \r
 146                                 string topHash;\r
 147                                 TreeHash treeHash;\r
 148                                 using(StatusNotification.GetNotifier("Hashing {0} for Upload", "Finished hashing {0}",fileInfo.Name))\r
 149                                 {\r
 150                                     treeHash = action.TreeHash.Value;\r
 151                                     topHash = treeHash.TopHash.ToHashString();\r
 152                                 }\r
 153 \r
 154 \r
 155 \r
 156                                 //If the file hashes match, abort the upload\r
 157                                 if (cloudInfo != ObjectInfo.Empty && (topHash == cloudHash ))\r
 158                                 {\r
 159                                     //but store any metadata changes \r
 160                                     StatusKeeper.StoreInfo(fullFileName, cloudInfo);\r
 161                                     Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);\r
 162                                     return;\r
 163                                 }\r
 164 \r
 165 \r
 166                                 //Mark the file as modified while we upload it\r
 167                                 StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);\r
 168                                 //And then upload it\r
 169 \r
 170                                 //Upload even small files using the Hashmap. The server may already contain\r
 171                                 //the relevant block                                \r
 172 \r
 173                                 \r
 174 \r
 175                                 await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash,cancellationToken);\r
 176                             }\r
 177                             //If everything succeeds, change the file and overlay status to normal\r
 178                             StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");\r
 179                         }\r
 180                         catch (WebException exc)\r
 181                         {\r
 182                             var response = (exc.Response as HttpWebResponse);\r
 183                             if (response == null)\r
 184                                 throw;\r
 185                             if (response.StatusCode == HttpStatusCode.Forbidden)\r
 186                             {\r
 187                                 StatusKeeper.SetFileState(fileInfo.FullName, FileStatus.Forbidden, FileOverlayStatus.Conflict, "Forbidden");\r
 188                                 MakeFileReadOnly(fullFileName);\r
 189                             }\r
 190                             else\r
 191                                 //In any other case, propagate the error\r
 192                                 throw;\r
 193                         }\r
 194                     }\r
 195                     //Notify the Shell to update the overlays\r
 196                     NativeMethods.RaiseChangeNotification(fullFileName);\r
 197                     StatusNotification.NotifyChangedFile(fullFileName);\r
 198                 }\r
 199                 catch (AggregateException ex)\r
 200                 {\r
 201                     var exc = ex.InnerException as WebException;\r
 202                     if (exc == null)\r
 203                         throw ex.InnerException;\r
 204                     if (HandleUploadWebException(action, exc))\r
 205                         return;\r
 206                     throw;\r
 207                 }\r
 208                 catch (WebException ex)\r
 209                 {\r
 210                     if (HandleUploadWebException(action, ex))\r
 211                         return;\r
 212                     throw;\r
 213                 }\r
 214                 catch (Exception ex)\r
 215                 {\r
 216                     Log.Error("Unexpected error while uploading file", ex);\r
 217                     throw;\r
 218                 }\r
 219             }\r
 220         }\r
 221 \r
 222         private static void MakeFileReadOnly(string fullFileName)\r
 223         {\r
 224             var attributes = File.GetAttributes(fullFileName);\r
 225             //Do not make any modifications if not necessary\r
 226             if (attributes.HasFlag(FileAttributes.ReadOnly))\r
 227                 return;\r
 228             File.SetAttributes(fullFileName, attributes | FileAttributes.ReadOnly);            \r
 229         }\r
 230 \r
 231         private static AccountInfo GetSharerAccount(string relativePath, AccountInfo accountInfo)\r
 232         {\r
 233             var parts = relativePath.Split('\\');\r
 234             var accountName = parts[1];\r
 235             var oldName = accountInfo.UserName;\r
 236             var absoluteUri = accountInfo.StorageUri.AbsoluteUri;\r
 237             var nameIndex = absoluteUri.IndexOf(oldName, StringComparison.Ordinal);\r
 238             var root = absoluteUri.Substring(0, nameIndex);\r
 239 \r
 240             accountInfo = new AccountInfo\r
 241                               {\r
 242                                   UserName = accountName,\r
 243                                   AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),\r
 244                                   StorageUri = new Uri(root + accountName),\r
 245                                   BlockHash = accountInfo.BlockHash,\r
 246                                   BlockSize = accountInfo.BlockSize,\r
 247                                   Token = accountInfo.Token\r
 248                               };\r
 249             return accountInfo;\r
 250         }\r
 251 \r
 252 \r
 253         public async Task UploadWithHashMap(AccountInfo accountInfo, ObjectInfo cloudFile, FileInfo fileInfo, string url, TreeHash treeHash, CancellationToken token)\r
 254         {\r
 255             if (accountInfo == null)\r
 256                 throw new ArgumentNullException("accountInfo");\r
 257             if (cloudFile == null)\r
 258                 throw new ArgumentNullException("cloudFile");\r
 259             if (fileInfo == null)\r
 260                 throw new ArgumentNullException("fileInfo");\r
 261             if (String.IsNullOrWhiteSpace(url))\r
 262                 throw new ArgumentNullException(url);\r
 263             if (treeHash == null)\r
 264                 throw new ArgumentNullException("treeHash");\r
 265             if (String.IsNullOrWhiteSpace(cloudFile.Container))\r
 266                 throw new ArgumentException("Invalid container", "cloudFile");\r
 267             Contract.EndContractBlock();\r
 268 \r
 269            \r
 270             using (StatusNotification.GetNotifier("Uploading {0}", "Finished Uploading {0}", fileInfo.Name))\r
 271             {\r
 272                 if (await WaitOrAbort(accountInfo,cloudFile, token)) \r
 273                     return;\r
 274 \r
 275                 var fullFileName = fileInfo.GetProperCapitalization();\r
 276 \r
 277                 var account = cloudFile.Account ?? accountInfo.UserName;\r
 278                 var container = cloudFile.Container;\r
 279 \r
 280 \r
 281                 var client = new CloudFilesClient(accountInfo);\r
 282                 //Send the hashmap to the server            \r
 283                 var missingHashes = await client.PutHashMap(account, container, url, treeHash);\r
 284                 int block = 0;\r
 285                 ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);\r
 286                 //If the server returns no missing hashes, we are done\r
 287                 while (missingHashes.Count > 0)\r
 288                 {\r
 289 \r
 290                     if (await WaitOrAbort(accountInfo,cloudFile, token))\r
 291                         return;\r
 292 \r
 293 \r
 294                     var buffer = new byte[accountInfo.BlockSize];\r
 295                     foreach (var missingHash in missingHashes)\r
 296                     {\r
 297                         if (await WaitOrAbort(accountInfo,cloudFile, token))\r
 298                             return;\r
 299 \r
 300 \r
 301                         //Find the proper block\r
 302                         var blockIndex = treeHash.HashDictionary[missingHash];\r
 303                         long offset = blockIndex*accountInfo.BlockSize;\r
 304 \r
 305                         var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);\r
 306 \r
 307                         try\r
 308                         {\r
 309                             //And upload the block                \r
 310                             await client.PostBlock(account, container, buffer, 0, read);\r
 311                             Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);\r
 312                         }\r
 313                         catch (Exception exc)\r
 314                         {\r
 315                             Log.Error(String.Format("Uploading block {0} of {1}", blockIndex, fullFileName), exc);\r
 316                         }\r
 317                         ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);\r
 318                     }\r
 319 \r
 320                     token.ThrowIfCancellationRequested();\r
 321                     //Repeat until there are no more missing hashes                \r
 322                     missingHashes = await client.PutHashMap(account, container, url, treeHash);\r
 323                 }\r
 324 \r
 325                 ReportUploadProgress(fileInfo.Name, missingHashes.Count, missingHashes.Count, fileInfo.Length);\r
 326             }\r
 327         }\r
 328 \r
 329         private async Task<bool> WaitOrAbort(AccountInfo account,ObjectInfo cloudFile, CancellationToken token)\r
 330         {\r
 331             token.ThrowIfCancellationRequested();\r
 332             await UnpauseEvent.WaitAsync();\r
 333             var shouldAbort = !Selectives.IsSelected(account,cloudFile);\r
 334             if (shouldAbort)\r
 335                 Log.InfoFormat("Aborting [{0}]",cloudFile.Uri);\r
 336             return shouldAbort;\r
 337         }\r
 338 \r
 339         private void ReportUploadProgress(string fileName, int block, int totalBlocks, long fileSize)\r
 340         {\r
 341             StatusNotification.Notify(totalBlocks == 0\r
 342                                           ? new ProgressNotification(fileName, "Uploading", 1, 1, fileSize)\r
 343                                           : new ProgressNotification(fileName, "Uploading", block, totalBlocks, fileSize));\r
 344         }\r
 345 \r
 346 \r
 347         private bool HandleUploadWebException(CloudAction action, WebException exc)\r
 348         {\r
 349             var response = exc.Response as HttpWebResponse;\r
 350             if (response == null)\r
 351                 throw exc;\r
 352             if (response.StatusCode == HttpStatusCode.Unauthorized)\r
 353             {\r
 354                 Log.Error("Not allowed to upload file", exc);\r
 355                 var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);\r
 356                 StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");\r
 357                 StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
 358                 return true;\r
 359             }\r
 360             return false;\r
 361         }\r
 362 \r
 363         [Import]\r
 364         public Selectives Selectives { get; set; }\r
 365 \r
 366         public AsyncManualResetEvent UnpauseEvent { get; set; }\r
 367     }\r
 368 }\r