code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/Uploader.cs

   1 using System;\r
   2 using System.Collections.Generic;\r
   3 using System.ComponentModel.Composition;\r
   4 using System.Diagnostics;\r
   5 using System.Diagnostics.Contracts;\r
   6 using System.IO;\r
   7 using System.Linq;\r
   8 using System.Net;\r
   9 using System.Reflection;\r
  10 using System.Security.Cryptography;\r
  11 using System.Threading;\r
  12 using System.Threading.Tasks;\r
  13 using Pithos.Interfaces;\r
  14 using Pithos.Network;\r
  15 using log4net;\r
  16 \r
  17 namespace Pithos.Core.Agents\r
  18 {\r
  19     [Export(typeof(Uploader))]\r
  20     public class Uploader\r
  21     {\r
  22         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  23 \r
  24         [Import]\r
  25         private IStatusKeeper StatusKeeper { get; set; }\r
  26 \r
  27         public IStatusNotification StatusNotification { get; set; }\r
  28 \r
  29         \r
  30         //CancellationTokenSource _cts = new CancellationTokenSource();\r
  31         /*public void SignalStop()\r
  32         {\r
  33             _cts.Cancel();\r
  34         }*/\r
  35 \r
  36         public async Task UploadCloudFile(CloudUploadAction action,CancellationToken cancellationToken)\r
  37         {\r
  38             if (action == null)\r
  39                 throw new ArgumentNullException("action");\r
  40             Contract.EndContractBlock();\r
  41 \r
  42             using (ThreadContext.Stacks["Operation"].Push("UploadCloudFile"))\r
  43             {\r
  44                 try\r
  45                 {\r
  46                     await UnpauseEvent.WaitAsync();\r
  47 \r
  48                     var fileInfo = action.LocalFile;\r
  49 \r
  50                     if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))\r
  51                         return;\r
  52 \r
  53                     if (!Selectives.IsSelected(action.AccountInfo, fileInfo) && !action.IsCreation)\r
  54                         return;\r
  55 \r
  56                     //Try to load the action's local state, if it is empty\r
  57                     if (action.FileState == null)\r
  58                         action.FileState = StatusKeeper.GetStateByFilePath(fileInfo.FullName);\r
  59                     if (action.FileState != null)\r
  60                     {\r
  61                         /*\r
  62                                                 Log.WarnFormat("File [{0}] has no local state. It was probably created by a download action", fileInfo.FullName);\r
  63                                                 return;\r
  64                         */\r
  65 \r
  66 \r
  67                         var latestState = action.FileState;\r
  68 \r
  69                         //Do not upload files in conflict\r
  70                         if (latestState.FileStatus == FileStatus.Conflict)\r
  71                         {\r
  72                             Log.InfoFormat("Skipping file in conflict [{0}]", fileInfo.FullName);\r
  73                             return;\r
  74                         }\r
  75                         //Do not upload files when we have no permission\r
  76                         if (latestState.FileStatus == FileStatus.Forbidden)\r
  77                         {\r
  78                             Log.InfoFormat("Skipping forbidden file [{0}]", fileInfo.FullName);\r
  79                             return;\r
  80                         }\r
  81                     }\r
  82                     //Are we targeting our own account or a sharer account?\r
  83                     var relativePath = fileInfo.AsRelativeTo(action.AccountInfo.AccountPath);\r
  84                     var accountInfo = relativePath.StartsWith(FolderConstants.OthersFolder) \r
  85                                                   ? GetSharerAccount(relativePath, action.AccountInfo) \r
  86                                                   : action.AccountInfo;\r
  87 \r
  88 \r
  89 \r
  90                     var fullFileName = fileInfo.GetProperCapitalization();\r
  91                     using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))\r
  92                     {\r
  93                         //Abort if the file is already being uploaded or downloaded\r
  94                         if (gate.Failed)\r
  95                             return;\r
  96 \r
  97                         var cloudFile = action.CloudFile;\r
  98                         var account = cloudFile.Account ?? accountInfo.UserName;\r
  99                         try\r
 100                         {\r
 101 \r
 102                             var client = new CloudFilesClient(accountInfo);\r
 103 \r
 104                             //Even if GetObjectInfo times out, we can proceed with the upload            \r
 105                             var cloudInfo = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);\r
 106 \r
 107                             //If this a shared file\r
 108                             if (!cloudFile.Account.Equals(action.AccountInfo.UserName,StringComparison.InvariantCultureIgnoreCase))\r
 109                             {\r
 110                                 \r
 111 /*\r
 112                                 if (!cloudInfo.IsWritable(action.AccountInfo.UserName))\r
 113                                 {\r
 114                                     MakeFileReadOnly(fullFileName);\r
 115                                     StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");\r
 116                                     return;\r
 117                                 }\r
 118 */\r
 119 \r
 120                                 //If this is a read-only file, do not upload changes\r
 121                                 if ( !cloudInfo.IsWritable(action.AccountInfo.UserName) ||\r
 122                                     //If the file is new, but we can't upload it\r
 123                                     (!cloudInfo.Exists && !client.CanUpload(account, cloudFile)) )\r
 124                                 {\r
 125                                     MakeFileReadOnly(fullFileName);\r
 126                                     StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");\r
 127                                     return;\r
 128                                 }\r
 129 \r
 130                             }\r
 131 \r
 132                             await UnpauseEvent.WaitAsync();\r
 133 \r
 134                             if (fileInfo is DirectoryInfo)\r
 135                             {\r
 136                                 //If the directory doesn't exist the Hash property will be empty\r
 137                                 if (String.IsNullOrWhiteSpace(cloudInfo.X_Object_Hash))\r
 138                                     //Go on and create the directory\r
 139                                     await client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,\r
 140                                                          String.Empty, "application/directory");\r
 141                                 //If the upload is in response to a Folder create with Selective Sync enabled\r
 142                                 if (action.IsCreation)\r
 143                                 {\r
 144                                     //Add the folder to the Selected URls\r
 145                                     var selectiveUri = new Uri(client.RootAddressUri, cloudFile.Uri);\r
 146                                     Selectives.AddUri(accountInfo, selectiveUri);                                    \r
 147                                     Selectives.Save(accountInfo);\r
 148                                 }\r
 149                             }\r
 150                             else\r
 151                             {\r
 152 \r
 153                                 var cloudHash = cloudInfo.X_Object_Hash.ToLower();\r
 154 \r
 155                                 string topHash;\r
 156                                 TreeHash treeHash;\r
 157                                 using(StatusNotification.GetNotifier("Hashing {0} for Upload", "Finished hashing {0}",fileInfo.Name))\r
 158                                 {\r
 159                                     treeHash = action.TreeHash.Value;\r
 160                                     topHash = treeHash.TopHash.ToHashString();\r
 161                                 }\r
 162 \r
 163 \r
 164 \r
 165                                 //If the file hashes match, abort the upload\r
 166                                 if (cloudInfo != ObjectInfo.Empty && (topHash == cloudHash ))\r
 167                                 {\r
 168                                     //but store any metadata changes \r
 169                                     StatusKeeper.StoreInfo(fullFileName, cloudInfo);\r
 170                                     Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);\r
 171                                     return;\r
 172                                 }\r
 173 \r
 174 \r
 175                                 //Mark the file as modified while we upload it\r
 176                                 await StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);\r
 177                                 //And then upload it\r
 178 \r
 179                                 //Upload even small files using the Hashmap. The server may already contain\r
 180                                 //the relevant block                                \r
 181 \r
 182                                 \r
 183 \r
 184                                 await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash,cancellationToken);\r
 185                             }\r
 186                             var currentInfo=client.GetObjectInfo(cloudFile.Account, cloudFile.Container, cloudFile.Name);\r
 187                             //If there is no stored ObjectID in the file state, add it\r
 188                             if (action.FileState == null || action.FileState.ObjectID == null)\r
 189                             {\r
 190                                 StatusKeeper.StoreInfo(fullFileName,currentInfo);\r
 191                             }\r
 192                             else                             \r
 193                                 //If everything succeeds, change the file and overlay status to normal\r
 194                                 StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");\r
 195                         }\r
 196                         catch (WebException exc)\r
 197                         {\r
 198                             var response = (exc.Response as HttpWebResponse);\r
 199                             if (response == null)\r
 200                                 throw;\r
 201                             if (response.StatusCode == HttpStatusCode.Forbidden)\r
 202                             {\r
 203                                 StatusKeeper.SetFileState(fileInfo.FullName, FileStatus.Forbidden, FileOverlayStatus.Conflict, "Forbidden");\r
 204                                 MakeFileReadOnly(fullFileName);\r
 205                             }\r
 206                             else\r
 207                                 //In any other case, propagate the error\r
 208                                 throw;\r
 209                         }\r
 210                     }\r
 211                     //Notify the Shell to update the overlays\r
 212                     NativeMethods.RaiseChangeNotification(fullFileName);\r
 213                     StatusNotification.NotifyChangedFile(fullFileName);\r
 214                 }\r
 215                 catch (AggregateException ex)\r
 216                 {\r
 217                     var exc = ex.InnerException as WebException;\r
 218                     if (exc == null)\r
 219                         throw ex.InnerException;\r
 220                     if (HandleUploadWebException(action, exc))\r
 221                         return;\r
 222                     throw;\r
 223                 }\r
 224                 catch (WebException ex)\r
 225                 {\r
 226                     if (HandleUploadWebException(action, ex))\r
 227                         return;\r
 228                     throw;\r
 229                 }\r
 230                 catch (Exception ex)\r
 231                 {\r
 232                     Log.Error("Unexpected error while uploading file", ex);\r
 233                     throw;\r
 234                 }\r
 235             }\r
 236         }\r
 237 \r
 238         private static void MakeFileReadOnly(string fullFileName)\r
 239         {\r
 240             var attributes = File.GetAttributes(fullFileName);\r
 241             //Do not make any modifications if not necessary\r
 242             if (attributes.HasFlag(FileAttributes.ReadOnly))\r
 243                 return;\r
 244             File.SetAttributes(fullFileName, attributes | FileAttributes.ReadOnly);            \r
 245         }\r
 246 \r
 247         private static AccountInfo GetSharerAccount(string relativePath, AccountInfo accountInfo)\r
 248         {\r
 249             var parts = relativePath.Split('\\');\r
 250             var accountName = parts[1];\r
 251             var oldName = accountInfo.UserName;\r
 252             var absoluteUri = accountInfo.StorageUri.AbsoluteUri;\r
 253             var nameIndex = absoluteUri.IndexOf(oldName, StringComparison.Ordinal);\r
 254             var root = absoluteUri.Substring(0, nameIndex);\r
 255 \r
 256             accountInfo = new AccountInfo\r
 257                               {\r
 258                                   UserName = accountName,\r
 259                                   AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),\r
 260                                   StorageUri = new Uri(root + accountName),\r
 261                                   BlockHash = accountInfo.BlockHash,\r
 262                                   BlockSize = accountInfo.BlockSize,\r
 263                                   Token = accountInfo.Token\r
 264                               };\r
 265             return accountInfo;\r
 266         }\r
 267 \r
 268 \r
 269         public async Task UploadWithHashMap(AccountInfo accountInfo, ObjectInfo cloudFile, FileInfo fileInfo, string url, TreeHash treeHash, CancellationToken token)\r
 270         {\r
 271             if (accountInfo == null)\r
 272                 throw new ArgumentNullException("accountInfo");\r
 273             if (cloudFile == null)\r
 274                 throw new ArgumentNullException("cloudFile");\r
 275             if (fileInfo == null)\r
 276                 throw new ArgumentNullException("fileInfo");\r
 277             if (String.IsNullOrWhiteSpace(url))\r
 278                 throw new ArgumentNullException(url);\r
 279             if (treeHash == null)\r
 280                 throw new ArgumentNullException("treeHash");\r
 281             if (String.IsNullOrWhiteSpace(cloudFile.Container))\r
 282                 throw new ArgumentException("Invalid container", "cloudFile");\r
 283             Contract.EndContractBlock();\r
 284 \r
 285            \r
 286             using (StatusNotification.GetNotifier("Uploading {0}", "Finished Uploading {0}", fileInfo.Name))\r
 287             {\r
 288                 if (await WaitOrAbort(accountInfo,cloudFile, token)) \r
 289                     return;\r
 290 \r
 291                 var fullFileName = fileInfo.GetProperCapitalization();\r
 292 \r
 293                 var account = cloudFile.Account ?? accountInfo.UserName;\r
 294                 var container = cloudFile.Container;\r
 295 \r
 296 \r
 297                 var client = new CloudFilesClient(accountInfo);\r
 298                 //Send the hashmap to the server            \r
 299                 var missingHashes = await client.PutHashMap(account, container, url, treeHash);\r
 300                 int block = 0;\r
 301                 ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);\r
 302                 //If the server returns no missing hashes, we are done\r
 303                 while (missingHashes.Count > 0)\r
 304                 {\r
 305 \r
 306                     if (await WaitOrAbort(accountInfo,cloudFile, token))\r
 307                         return;\r
 308 \r
 309 \r
 310                     var buffer = new byte[accountInfo.BlockSize];\r
 311                     foreach (var missingHash in missingHashes)\r
 312                     {\r
 313                         if (await WaitOrAbort(accountInfo,cloudFile, token))\r
 314                             return;\r
 315 \r
 316 \r
 317                         //Find the proper block\r
 318                         var blockIndex = treeHash.HashDictionary[missingHash];\r
 319                         long offset = blockIndex*accountInfo.BlockSize;\r
 320 \r
 321                         var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);\r
 322 \r
 323                         try\r
 324                         {\r
 325                             //And upload the block                \r
 326                             await client.PostBlock(account, container, buffer, 0, read, token);\r
 327                             Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);\r
 328                         }\r
 329                         catch (TaskCanceledException exc)\r
 330                         {\r
 331                             throw new OperationCanceledException(token);\r
 332                         }\r
 333                         catch (Exception exc)\r
 334                         {\r
 335                             Log.Error(String.Format("Uploading block {0} of {1}", blockIndex, fullFileName), exc);\r
 336                         }\r
 337                         ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);\r
 338                     }\r
 339 \r
 340                     token.ThrowIfCancellationRequested();\r
 341                     //Repeat until there are no more missing hashes                \r
 342                     missingHashes = await client.PutHashMap(account, container, url, treeHash);\r
 343                 }\r
 344 \r
 345                 ReportUploadProgress(fileInfo.Name, missingHashes.Count, missingHashes.Count, fileInfo.Length);\r
 346             }\r
 347         }\r
 348 \r
 349         private async Task<bool> WaitOrAbort(AccountInfo account,ObjectInfo cloudFile, CancellationToken token)\r
 350         {\r
 351             token.ThrowIfCancellationRequested();\r
 352             await UnpauseEvent.WaitAsync();\r
 353             var shouldAbort = !Selectives.IsSelected(account,cloudFile);\r
 354             if (shouldAbort)\r
 355                 Log.InfoFormat("Aborting [{0}]",cloudFile.Uri);\r
 356             return shouldAbort;\r
 357         }\r
 358 \r
 359         private void ReportUploadProgress(string fileName, int block, int totalBlocks, long fileSize)\r
 360         {\r
 361             StatusNotification.Notify(totalBlocks == 0\r
 362                                           ? new ProgressNotification(fileName, "Uploading", 1, 1, fileSize)\r
 363                                           : new ProgressNotification(fileName, "Uploading", block, totalBlocks, fileSize));\r
 364         }\r
 365 \r
 366 \r
 367         private bool HandleUploadWebException(CloudAction action, WebException exc)\r
 368         {\r
 369             var response = exc.Response as HttpWebResponse;\r
 370             if (response == null)\r
 371                 throw exc;\r
 372             if (response.StatusCode == HttpStatusCode.Unauthorized)\r
 373             {\r
 374                 Log.Error("Not allowed to upload file", exc);\r
 375                 var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);\r
 376                 StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");\r
 377                 StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
 378                 return true;\r
 379             }\r
 380             return false;\r
 381         }\r
 382 \r
 383         [Import]\r
 384         public Selectives Selectives { get; set; }\r
 385 \r
 386         public AsyncManualResetEvent UnpauseEvent { get; set; }\r
 387     }\r
 388 }\r