code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/Uploader.cs

   1 using System;
   2 using System.ComponentModel.Composition;
   3 using System.Diagnostics;
   4 using System.Diagnostics.Contracts;
   5 using System.IO;
   6 using System.Net;
   7 using System.Reflection;
   8 using System.Threading.Tasks;
   9 using Pithos.Interfaces;
  10 using Pithos.Network;
  11 using log4net;
  12
  13 namespace Pithos.Core.Agents
  14 {
  15     [Export(typeof(Uploader))]
  16     public class Uploader
  17     {
  18         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  19
  20         [Import]
  21         private IStatusKeeper StatusKeeper { get; set; }
  22
  23
  24         public IStatusNotification StatusNotification { get; set; }
  25
  26         public async Task UploadCloudFile(CloudAction action)
  27         {
  28             if (action == null)
  29                 throw new ArgumentNullException("action");
  30             Contract.EndContractBlock();
  31
  32             using (ThreadContext.Stacks["Operation"].Push("UploadCloudFile"))
  33             {
  34                 try
  35                 {
  36                     var fileInfo = action.LocalFile;
  37
  38                     if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
  39                         return;
  40
  41                     //Try to load the action's local state, if it is empty
  42                     if (action.FileState == null)
  43                         action.FileState = StatusKeeper.GetStateByFilePath(fileInfo.FullName);
  44                     if (action.FileState == null)
  45                     {
  46                         Log.WarnFormat("File [{0}] has no local state. It was probably created by a download action", fileInfo.FullName);
  47                         return;
  48                     }
  49
  50                     //Do not upload files in conflict
  51                     if (action.FileState.FileStatus == FileStatus.Conflict)
  52                     {
  53                         Log.InfoFormat("Skipping file in conflict [{0}]", fileInfo.FullName);
  54                         return;
  55                     }
  56                     //Do not upload files when we have no permission
  57                     if (action.FileState.FileStatus == FileStatus.Forbidden)
  58                     {
  59                         Log.InfoFormat("Skipping forbidden file [{0}]", fileInfo.FullName);
  60                         return;
  61                     }
  62
  63                     //Are we targeting our own account or a sharer account?
  64                     var relativePath = fileInfo.AsRelativeTo(action.AccountInfo.AccountPath);
  65                     var accountInfo = relativePath.StartsWith(FolderConstants.OthersFolder)
  66                                                   ? GetSharerAccount(relativePath, action.AccountInfo)
  67                                                   : action.AccountInfo;
  68
  69
  70
  71                     var fullFileName = fileInfo.GetProperCapitalization();
  72                     using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))
  73                     {
  74                         //Abort if the file is already being uploaded or downloaded
  75                         if (gate.Failed)
  76                             return;
  77
  78                         var cloudFile = action.CloudFile;
  79                         var account = cloudFile.Account ?? accountInfo.UserName;
  80                         try
  81                         {
  82
  83                             var client = new CloudFilesClient(accountInfo);
  84                             //Even if GetObjectInfo times out, we can proceed with the upload
  85                             var cloudInfo = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
  86
  87                             //If this is a read-only file, do not upload changes
  88                             if (cloudInfo.AllowedTo == "read")
  89                                 return;
  90
  91                             if (fileInfo is DirectoryInfo)
  92                             {
  93                                 //If the directory doesn't exist the Hash property will be empty
  94                                 if (String.IsNullOrWhiteSpace(cloudInfo.Hash))
  95                                     //Go on and create the directory
  96                                     await client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,
  97                                                          String.Empty, "application/directory");
  98                             }
  99                             else
 100                             {
 101
 102                                 var cloudHash = cloudInfo.Hash.ToLower();
 103
 104                                 StatusNotification.Notify(new StatusNotification(String.Format("Hashing {0} for Upload", fileInfo.Name)));
 105
 106                                 var treeHash = action.TreeHash.Value;
 107                                 var topHash = treeHash.TopHash.ToHashString();
 108
 109                                 //If the file hashes match, abort the upload
 110                                 if (cloudInfo != ObjectInfo.Empty && topHash == cloudHash)
 111                                 {
 112                                     //but store any metadata changes
 113                                     StatusKeeper.StoreInfo(fullFileName, cloudInfo);
 114                                     Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
 115                                     return;
 116                                 }
 117
 118
 119                                 //Mark the file as modified while we upload it
 120                                 StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
 121                                 //And then upload it
 122
 123                                 //Upload even small files using the Hashmap. The server may already contain
 124                                 //the relevant block
 125
 126                                 await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash);
 127                             }
 128                             //If everything succeeds, change the file and overlay status to normal
 129                             StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal);
 130                         }
 131                         catch (WebException exc)
 132                         {
 133                             var response = (exc.Response as HttpWebResponse);
 134                             if (response == null)
 135                                 throw;
 136                             if (response.StatusCode == HttpStatusCode.Forbidden)
 137                             {
 138                                 StatusKeeper.SetFileState(fileInfo.FullName, FileStatus.Forbidden, FileOverlayStatus.Conflict);
 139                             }
 140
 141                             //In any other case, propagate the error
 142                             throw;
 143                         }
 144                     }
 145                     //Notify the Shell to update the overlays
 146                     NativeMethods.RaiseChangeNotification(fullFileName);
 147                     StatusNotification.NotifyChangedFile(fullFileName);
 148                 }
 149                 catch (AggregateException ex)
 150                 {
 151                     var exc = ex.InnerException as WebException;
 152                     if (exc == null)
 153                         throw ex.InnerException;
 154                     if (HandleUploadWebException(action, exc))
 155                         return;
 156                     throw;
 157                 }
 158                 catch (WebException ex)
 159                 {
 160                     if (HandleUploadWebException(action, ex))
 161                         return;
 162                     throw;
 163                 }
 164                 catch (Exception ex)
 165                 {
 166                     Log.Error("Unexpected error while uploading file", ex);
 167                     throw;
 168                 }
 169             }
 170         }
 171
 172         private static AccountInfo GetSharerAccount(string relativePath, AccountInfo accountInfo)
 173         {
 174             var parts = relativePath.Split('\\');
 175             var accountName = parts[1];
 176             var oldName = accountInfo.UserName;
 177             var absoluteUri = accountInfo.StorageUri.AbsoluteUri;
 178             var nameIndex = absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
 179             var root = absoluteUri.Substring(0, nameIndex);
 180
 181             accountInfo = new AccountInfo
 182                               {
 183                                   UserName = accountName,
 184                                   AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 185                                   StorageUri = new Uri(root + accountName),
 186                                   BlockHash = accountInfo.BlockHash,
 187                                   BlockSize = accountInfo.BlockSize,
 188                                   Token = accountInfo.Token
 189                               };
 190             return accountInfo;
 191         }
 192
 193
 194         public async Task UploadWithHashMap(AccountInfo accountInfo, ObjectInfo cloudFile, FileInfo fileInfo, string url, TreeHash treeHash)
 195         {
 196             if (accountInfo == null)
 197                 throw new ArgumentNullException("accountInfo");
 198             if (cloudFile == null)
 199                 throw new ArgumentNullException("cloudFile");
 200             if (fileInfo == null)
 201                 throw new ArgumentNullException("fileInfo");
 202             if (String.IsNullOrWhiteSpace(url))
 203                 throw new ArgumentNullException(url);
 204             if (treeHash == null)
 205                 throw new ArgumentNullException("treeHash");
 206             if (String.IsNullOrWhiteSpace(cloudFile.Container))
 207                 throw new ArgumentException("Invalid container", "cloudFile");
 208             Contract.EndContractBlock();
 209
 210             StatusNotification.Notify(new StatusNotification(String.Format("Uploading {0}", fileInfo.Name)));
 211
 212             var fullFileName = fileInfo.GetProperCapitalization();
 213
 214             var account = cloudFile.Account ?? accountInfo.UserName;
 215             var container = cloudFile.Container;
 216
 217             var client = new CloudFilesClient(accountInfo);
 218             //Send the hashmap to the server
 219             var missingHashes = await client.PutHashMap(account, container, url, treeHash);
 220             int block = 0;
 221             ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);
 222             //If the server returns no missing hashes, we are done
 223             while (missingHashes.Count > 0)
 224             {
 225
 226                 var buffer = new byte[accountInfo.BlockSize];
 227                 foreach (var missingHash in missingHashes)
 228                 {
 229                     //Find the proper block
 230                     var blockIndex = treeHash.HashDictionary[missingHash];
 231                     long offset = blockIndex * accountInfo.BlockSize;
 232
 233                     var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);
 234
 235                     try
 236                     {
 237                         //And upload the block
 238                         await client.PostBlock(account, container, buffer, 0, read);
 239                         Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);
 240                     }
 241                     catch (Exception exc)
 242                     {
 243                         Log.Error(String.Format("Uploading block {0} of {1}", blockIndex, fullFileName), exc);
 244                     }
 245                     ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);
 246                 }
 247
 248                 //Repeat until there are no more missing hashes
 249                 missingHashes = await client.PutHashMap(account, container, url, treeHash);
 250             }
 251
 252             ReportUploadProgress(fileInfo.Name, missingHashes.Count, missingHashes.Count, fileInfo.Length);
 253         }
 254
 255         private void ReportUploadProgress(string fileName, int block, int totalBlocks, long fileSize)
 256         {
 257             StatusNotification.Notify(totalBlocks == 0
 258                                           ? new ProgressNotification(fileName, "Uploading", 1, 1, fileSize)
 259                                           : new ProgressNotification(fileName, "Uploading", block, totalBlocks, fileSize));
 260         }
 261
 262
 263         private bool HandleUploadWebException(CloudAction action, WebException exc)
 264         {
 265             var response = exc.Response as HttpWebResponse;
 266             if (response == null)
 267                 throw exc;
 268             if (response.StatusCode == HttpStatusCode.Unauthorized)
 269             {
 270                 Log.Error("Not allowed to upload file", exc);
 271                 var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);
 272                 StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal);
 273                 StatusNotification.NotifyChange(message, TraceLevel.Warning);
 274                 return true;
 275             }
 276             return false;
 277         }
 278     }
 279 }