code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/Uploader.cs

   1 using System;\r
   2 using System.Collections.Generic;\r
   3 using System.ComponentModel.Composition;\r
   4 using System.Diagnostics;\r
   5 using System.Diagnostics.Contracts;\r
   6 using System.IO;\r
   7 using System.Linq;\r
   8 using System.Net;\r
   9 using System.Reflection;\r
  10 using System.Security.Cryptography;\r
  11 using System.Threading;\r
  12 using System.Threading.Tasks;\r
  13 using Pithos.Interfaces;\r
  14 using Pithos.Network;\r
  15 using log4net;\r
  16 \r
  17 namespace Pithos.Core.Agents\r
  18 {\r
  19     [Export(typeof(Uploader))]\r
  20     public class Uploader\r
  21     {\r
  22         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  23 \r
  24         [Import]\r
  25         private IStatusKeeper StatusKeeper { get; set; }\r
  26 \r
  27         \r
  28         public IStatusNotification StatusNotification { get; set; }\r
  29 \r
  30         \r
  31         //CancellationTokenSource _cts = new CancellationTokenSource();\r
  32         /*public void SignalStop()\r
  33         {\r
  34             _cts.Cancel();\r
  35         }*/\r
  36 \r
  37         public async Task UploadCloudFile(CloudAction action,CancellationToken cancellationToken)\r
  38         {\r
  39             if (action == null)\r
  40                 throw new ArgumentNullException("action");\r
  41             Contract.EndContractBlock();\r
  42 \r
  43             using (ThreadContext.Stacks["Operation"].Push("UploadCloudFile"))\r
  44             {\r
  45                 try\r
  46                 {\r
  47                     await UnpauseEvent.WaitAsync();\r
  48 \r
  49                     var fileInfo = action.LocalFile;\r
  50 \r
  51                     if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))\r
  52                         return;\r
  53 \r
  54                     if (!Selectives.IsSelected(action.AccountInfo, fileInfo))\r
  55                         return;\r
  56 \r
  57                     //Try to load the action's local state, if it is empty\r
  58                     if (action.FileState == null)\r
  59                         action.FileState = StatusKeeper.GetStateByFilePath(fileInfo.FullName);\r
  60                     if (action.FileState == null)\r
  61                     {\r
  62                         Log.WarnFormat("File [{0}] has no local state. It was probably created by a download action", fileInfo.FullName);\r
  63                         return;\r
  64                     }\r
  65 \r
  66                     var latestState = action.FileState;\r
  67 \r
  68                     //Do not upload files in conflict\r
  69                     if (latestState.FileStatus == FileStatus.Conflict)\r
  70                     {\r
  71                         Log.InfoFormat("Skipping file in conflict [{0}]", fileInfo.FullName);\r
  72                         return;\r
  73                     }\r
  74                     //Do not upload files when we have no permission\r
  75                     if (latestState.FileStatus == FileStatus.Forbidden)\r
  76                     {\r
  77                         Log.InfoFormat("Skipping forbidden file [{0}]", fileInfo.FullName);\r
  78                         return;\r
  79                     }\r
  80 \r
  81                     //Are we targeting our own account or a sharer account?\r
  82                     var relativePath = fileInfo.AsRelativeTo(action.AccountInfo.AccountPath);\r
  83                     var accountInfo = relativePath.StartsWith(FolderConstants.OthersFolder) \r
  84                                                   ? GetSharerAccount(relativePath, action.AccountInfo) \r
  85                                                   : action.AccountInfo;\r
  86 \r
  87 \r
  88 \r
  89                     var fullFileName = fileInfo.GetProperCapitalization();\r
  90                     using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))\r
  91                     {\r
  92                         //Abort if the file is already being uploaded or downloaded\r
  93                         if (gate.Failed)\r
  94                             return;\r
  95 \r
  96                         var cloudFile = action.CloudFile;\r
  97                         var account = cloudFile.Account ?? accountInfo.UserName;\r
  98                         try\r
  99                         {\r
 100 \r
 101                             var client = new CloudFilesClient(accountInfo);\r
 102 \r
 103                             //Even if GetObjectInfo times out, we can proceed with the upload            \r
 104                             var cloudInfo = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);\r
 105 \r
 106                             //If this a shared file\r
 107                             if (!cloudFile.Account.Equals(action.AccountInfo.UserName,StringComparison.InvariantCultureIgnoreCase))\r
 108                             {\r
 109                                 //If this is a read-only file, do not upload changes\r
 110                                 if (!cloudInfo.IsWritable(action.AccountInfo.UserName))\r
 111                                 {\r
 112                                     MakeFileReadOnly(fullFileName);\r
 113                                     return;\r
 114                                 }\r
 115 \r
 116                                 //If the file is new, can we upload it?\r
 117                                 if ( !cloudInfo.Exists && !client.CanUpload(account, cloudFile))\r
 118                                 {\r
 119                                     MakeFileReadOnly(fullFileName);\r
 120                                     return;\r
 121                                 }\r
 122 \r
 123                             }\r
 124 \r
 125                             await UnpauseEvent.WaitAsync();\r
 126 \r
 127                             if (fileInfo is DirectoryInfo)\r
 128                             {\r
 129                                 //If the directory doesn't exist the Hash property will be empty\r
 130                                 if (String.IsNullOrWhiteSpace(cloudInfo.Hash))\r
 131                                     //Go on and create the directory\r
 132                                     await client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,\r
 133                                                          String.Empty, "application/directory");\r
 134                             }\r
 135                             else\r
 136                             {\r
 137 \r
 138                                 var cloudHash = cloudInfo.Hash.ToLower();\r
 139 \r
 140                                 string topHash;\r
 141                                 TreeHash treeHash;\r
 142                                 using(StatusNotification.GetNotifier("Hashing {0} for Upload", "Finished hashing {0}",fileInfo.Name))\r
 143                                 {\r
 144                                     treeHash = action.TreeHash.Value;\r
 145                                     topHash = treeHash.TopHash.ToHashString();\r
 146                                 }\r
 147 \r
 148 \r
 149 \r
 150                                 //If the file hashes match, abort the upload\r
 151                                 if (cloudInfo != ObjectInfo.Empty && (topHash == cloudHash ))\r
 152                                 {\r
 153                                     //but store any metadata changes \r
 154                                     StatusKeeper.StoreInfo(fullFileName, cloudInfo);\r
 155                                     Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);\r
 156                                     return;\r
 157                                 }\r
 158 \r
 159 \r
 160                                 //Mark the file as modified while we upload it\r
 161                                 StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);\r
 162                                 //And then upload it\r
 163 \r
 164                                 //Upload even small files using the Hashmap. The server may already contain\r
 165                                 //the relevant block                                \r
 166 \r
 167                                 \r
 168 \r
 169                                 await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash,cancellationToken);\r
 170                             }\r
 171                             //If everything succeeds, change the file and overlay status to normal\r
 172                             StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");\r
 173                         }\r
 174                         catch (WebException exc)\r
 175                         {\r
 176                             var response = (exc.Response as HttpWebResponse);\r
 177                             if (response == null)\r
 178                                 throw;\r
 179                             if (response.StatusCode == HttpStatusCode.Forbidden)\r
 180                             {\r
 181                                 StatusKeeper.SetFileState(fileInfo.FullName, FileStatus.Forbidden, FileOverlayStatus.Conflict, "Forbidden");\r
 182                                 MakeFileReadOnly(fullFileName);\r
 183                             }\r
 184                             else\r
 185                                 //In any other case, propagate the error\r
 186                                 throw;\r
 187                         }\r
 188                     }\r
 189                     //Notify the Shell to update the overlays\r
 190                     NativeMethods.RaiseChangeNotification(fullFileName);\r
 191                     StatusNotification.NotifyChangedFile(fullFileName);\r
 192                 }\r
 193                 catch (AggregateException ex)\r
 194                 {\r
 195                     var exc = ex.InnerException as WebException;\r
 196                     if (exc == null)\r
 197                         throw ex.InnerException;\r
 198                     if (HandleUploadWebException(action, exc))\r
 199                         return;\r
 200                     throw;\r
 201                 }\r
 202                 catch (WebException ex)\r
 203                 {\r
 204                     if (HandleUploadWebException(action, ex))\r
 205                         return;\r
 206                     throw;\r
 207                 }\r
 208                 catch (Exception ex)\r
 209                 {\r
 210                     Log.Error("Unexpected error while uploading file", ex);\r
 211                     throw;\r
 212                 }\r
 213             }\r
 214         }\r
 215 \r
 216         private static void MakeFileReadOnly(string fullFileName)\r
 217         {\r
 218             var attributes = File.GetAttributes(fullFileName);\r
 219             //Do not make any modifications if not necessary\r
 220             if (attributes.HasFlag(FileAttributes.ReadOnly))\r
 221                 return;\r
 222             File.SetAttributes(fullFileName, attributes | FileAttributes.ReadOnly);\r
 223         }\r
 224 \r
 225         private static AccountInfo GetSharerAccount(string relativePath, AccountInfo accountInfo)\r
 226         {\r
 227             var parts = relativePath.Split('\\');\r
 228             var accountName = parts[1];\r
 229             var oldName = accountInfo.UserName;\r
 230             var absoluteUri = accountInfo.StorageUri.AbsoluteUri;\r
 231             var nameIndex = absoluteUri.IndexOf(oldName, StringComparison.Ordinal);\r
 232             var root = absoluteUri.Substring(0, nameIndex);\r
 233 \r
 234             accountInfo = new AccountInfo\r
 235                               {\r
 236                                   UserName = accountName,\r
 237                                   AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),\r
 238                                   StorageUri = new Uri(root + accountName),\r
 239                                   BlockHash = accountInfo.BlockHash,\r
 240                                   BlockSize = accountInfo.BlockSize,\r
 241                                   Token = accountInfo.Token\r
 242                               };\r
 243             return accountInfo;\r
 244         }\r
 245 \r
 246 \r
 247         public async Task UploadWithHashMap(AccountInfo accountInfo, ObjectInfo cloudFile, FileInfo fileInfo, string url, TreeHash treeHash, CancellationToken token)\r
 248         {\r
 249             if (accountInfo == null)\r
 250                 throw new ArgumentNullException("accountInfo");\r
 251             if (cloudFile == null)\r
 252                 throw new ArgumentNullException("cloudFile");\r
 253             if (fileInfo == null)\r
 254                 throw new ArgumentNullException("fileInfo");\r
 255             if (String.IsNullOrWhiteSpace(url))\r
 256                 throw new ArgumentNullException(url);\r
 257             if (treeHash == null)\r
 258                 throw new ArgumentNullException("treeHash");\r
 259             if (String.IsNullOrWhiteSpace(cloudFile.Container))\r
 260                 throw new ArgumentException("Invalid container", "cloudFile");\r
 261             Contract.EndContractBlock();\r
 262 \r
 263            \r
 264             using (StatusNotification.GetNotifier("Uploading {0}", "Finished Uploading {0}", fileInfo.Name))\r
 265             {\r
 266                 if (await WaitOrAbort(accountInfo,cloudFile, token)) \r
 267                     return;\r
 268 \r
 269                 var fullFileName = fileInfo.GetProperCapitalization();\r
 270 \r
 271                 var account = cloudFile.Account ?? accountInfo.UserName;\r
 272                 var container = cloudFile.Container;\r
 273 \r
 274 \r
 275                 var client = new CloudFilesClient(accountInfo);\r
 276                 //Send the hashmap to the server            \r
 277                 var missingHashes = await client.PutHashMap(account, container, url, treeHash);\r
 278                 int block = 0;\r
 279                 ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);\r
 280                 //If the server returns no missing hashes, we are done\r
 281                 while (missingHashes.Count > 0)\r
 282                 {\r
 283 \r
 284                     if (await WaitOrAbort(accountInfo,cloudFile, token))\r
 285                         return;\r
 286 \r
 287 \r
 288                     var buffer = new byte[accountInfo.BlockSize];\r
 289                     foreach (var missingHash in missingHashes)\r
 290                     {\r
 291                         if (await WaitOrAbort(accountInfo,cloudFile, token))\r
 292                             return;\r
 293 \r
 294 \r
 295                         //Find the proper block\r
 296                         var blockIndex = treeHash.HashDictionary[missingHash];\r
 297                         long offset = blockIndex*accountInfo.BlockSize;\r
 298 \r
 299                         var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);\r
 300 \r
 301                         try\r
 302                         {\r
 303                             //And upload the block                \r
 304                             await client.PostBlock(account, container, buffer, 0, read);\r
 305                             Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);\r
 306                         }\r
 307                         catch (Exception exc)\r
 308                         {\r
 309                             Log.Error(String.Format("Uploading block {0} of {1}", blockIndex, fullFileName), exc);\r
 310                         }\r
 311                         ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);\r
 312                     }\r
 313 \r
 314                     token.ThrowIfCancellationRequested();\r
 315                     //Repeat until there are no more missing hashes                \r
 316                     missingHashes = await client.PutHashMap(account, container, url, treeHash);\r
 317                 }\r
 318 \r
 319                 ReportUploadProgress(fileInfo.Name, missingHashes.Count, missingHashes.Count, fileInfo.Length);\r
 320             }\r
 321         }\r
 322 \r
 323         private async Task<bool> WaitOrAbort(AccountInfo account,ObjectInfo cloudFile, CancellationToken token)\r
 324         {\r
 325             token.ThrowIfCancellationRequested();\r
 326             await UnpauseEvent.WaitAsync();\r
 327             var shouldAbort = !Selectives.IsSelected(account,cloudFile);\r
 328             if (shouldAbort)\r
 329                 Log.InfoFormat("Aborting [{0}]",cloudFile.Uri);\r
 330             return shouldAbort;\r
 331         }\r
 332 \r
 333         private void ReportUploadProgress(string fileName, int block, int totalBlocks, long fileSize)\r
 334         {\r
 335             StatusNotification.Notify(totalBlocks == 0\r
 336                                           ? new ProgressNotification(fileName, "Uploading", 1, 1, fileSize)\r
 337                                           : new ProgressNotification(fileName, "Uploading", block, totalBlocks, fileSize));\r
 338         }\r
 339 \r
 340 \r
 341         private bool HandleUploadWebException(CloudAction action, WebException exc)\r
 342         {\r
 343             var response = exc.Response as HttpWebResponse;\r
 344             if (response == null)\r
 345                 throw exc;\r
 346             if (response.StatusCode == HttpStatusCode.Unauthorized)\r
 347             {\r
 348                 Log.Error("Not allowed to upload file", exc);\r
 349                 var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);\r
 350                 StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");\r
 351                 StatusNotification.NotifyChange(message, TraceLevel.Warning);\r
 352                 return true;\r
 353             }\r
 354             return false;\r
 355         }\r
 356 \r
 357         [Import]\r
 358         public Selectives Selectives { get; set; }\r
 359 \r
 360         public AsyncManualResetEvent UnpauseEvent { get; set; }\r
 361     }\r
 362 }\r