code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel.Composition;
   4 using System.Diagnostics;
   5 using System.Diagnostics.Contracts;
   6 using System.IO;
   7 using System.Linq;
   8 using System.Net;
   9 using System.Security.Cryptography;
  10 using System.Text;
  11 using System.Threading.Tasks;
  12 using Hammock;
  13 using Hammock.Caching;
  14 using Hammock.Retries;
  15 using Hammock.Serialization;
  16 using Hammock.Tasks;
  17 using Hammock.Web;
  18 using Newtonsoft.Json;
  19 using Pithos.Interfaces;
  20
  21 namespace Pithos.Network
  22 {
  23     [Export(typeof(ICloudClient))]
  24     public class CloudFilesClient:ICloudClient
  25     {
  26         string _rackSpaceAuthUrl = "https://auth.api.rackspacecloud.com";
  27         private string _pithosAuthUrl = "https://pithos.grnet.gr";
  28
  29         private RestClient _client;
  30         private readonly TimeSpan _shortTimeout = TimeSpan.FromSeconds(10);
  31         private readonly int _retries = 5;
  32         private RetryPolicy _retryPolicy;
  33         public string ApiKey { get; set; }
  34         public string UserName { get; set; }
  35         public Uri StorageUrl { get; set; }
  36         public string Token { get; set; }
  37         public Uri Proxy { get; set; }
  38
  39         public double DownloadPercentLimit { get; set; }
  40         public double UploadPercentLimit { get; set; }
  41
  42         public string AuthUrl
  43         {
  44             get { return UsePithos ? _pithosAuthUrl : _rackSpaceAuthUrl; }
  45         }
  46
  47         public string VersionPath
  48         {
  49             get { return UsePithos ? "v1" : "v1.0"; }
  50         }
  51
  52         public bool UsePithos { get; set; }
  53
  54         public void Authenticate(string userName,string apiKey)
  55         {
  56             Trace.TraceInformation("[AUTHENTICATE] Start for {0}", userName);
  57             if (String.IsNullOrWhiteSpace(userName))
  58                 throw new ArgumentNullException("userName", "The userName property can't be empty");
  59             if (String.IsNullOrWhiteSpace(apiKey))
  60                 throw new ArgumentNullException("apiKey", "The apiKey property can't be empty");
  61
  62             UserName = userName;
  63             ApiKey = apiKey;
  64
  65             var proxy = Proxy != null ? Proxy.ToString() : null;
  66             if (UsePithos)
  67             {
  68                 Token = "0000";
  69                 string storageUrl = String.Format("{0}/{1}/{2}", AuthUrl, VersionPath, UserName);
  70                 StorageUrl = new Uri(storageUrl);
  71             }
  72             else
  73             {
  74
  75                 string authUrl = String.Format("{0}/{1}", AuthUrl, VersionPath);
  76                 var authClient = new RestClient {Path = authUrl, Proxy = proxy};
  77
  78                 authClient.AddHeader("X-Auth-User", UserName);
  79                 authClient.AddHeader("X-Auth-Key", ApiKey);
  80
  81                 var response = authClient.Request();
  82
  83                 ThrowIfNotStatusOK(response, "Authentication failed");
  84
  85                 var keys = response.Headers.AllKeys.AsQueryable();
  86
  87                 string storageUrl = GetHeaderValue("X-Storage-Url", response, keys);
  88                 if (String.IsNullOrWhiteSpace(storageUrl))
  89                     throw new InvalidOperationException("Failed to obtain storage url");
  90                 StorageUrl = new Uri(storageUrl);
  91
  92                 var token = GetHeaderValue("X-Auth-Token", response, keys);
  93                 if (String.IsNullOrWhiteSpace(token))
  94                     throw new InvalidOperationException("Failed to obtain token url");
  95                 Token = token;
  96             }
  97
  98             _retryPolicy = new RetryPolicy { RetryCount = _retries };
  99             _retryPolicy.RetryConditions.Add(new TimeoutRetryCondition());
 100
 101             _client = new RestClient { Authority = StorageUrl.AbsoluteUri, Path = UserName, Proxy = proxy };
 102             _client.FileProgress += OnFileProgress;
 103
 104             _client.AddHeader("X-Auth-Token", Token);
 105             /*if (UsePithos)
 106             {
 107                 _client.AddHeader("X-Auth-User", UserName);
 108                 _client.AddHeader("X-Auth-Key",ApiKey);
 109             }*/
 110
 111             Trace.TraceInformation("[AUTHENTICATE] End for {0}", userName);
 112         }
 113
 114         private void OnFileProgress(object sender, FileProgressEventArgs e)
 115         {
 116             Trace.TraceInformation("[PROGRESS] {0} {1:p} {2} of {3}",e.FileName,(double)e.BytesWritten/e.TotalBytes, e.BytesWritten,e.TotalBytes);
 117         }
 118
 119         public IList<ContainerInfo> ListContainers()
 120         {
 121             //Workaround for Hammock quirk: Hammock always
 122             //appends a / unless a Path is specified.
 123
 124             //Create a request with a complete path
 125             var request = new RestRequest { Path = StorageUrl.ToString(), RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
 126             request.AddParameter("format","json");
 127             //Create a client clone
 128             var client = new RestClient{Proxy=Proxy.ToString()};
 129             foreach (var header in _client.GetAllHeaders())
 130             {
 131                 client.AddHeader(header.Name,header.Value);
 132             }
 133
 134             var response = client.Request(request);
 135
 136             if (response.StatusCode == HttpStatusCode.NoContent)
 137                 return new List<ContainerInfo>();
 138
 139             ThrowIfNotStatusOK(response, "List Containers failed");
 140
 141
 142             var infos=JsonConvert.DeserializeObject<IList<ContainerInfo>>(response.Content);
 143
 144             return infos;
 145         }
 146
 147         public IList<ObjectInfo> ListObjects(string container)
 148         {
 149             if (String.IsNullOrWhiteSpace(container))
 150                 throw new ArgumentNullException("container", "The container property can't be empty");
 151
 152             Trace.TraceInformation("[START] ListObjects");
 153
 154             var request = new RestRequest { Path = container, RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) };
 155             request.AddParameter("format", "json");
 156             var response = _client.Request(request);
 157
 158             var infos = InfosFromContent(response);
 159
 160             Trace.TraceInformation("[END] ListObjects");
 161             return infos;
 162         }
 163
 164
 165
 166         public IList<ObjectInfo> ListObjects(string container,string folder)
 167         {
 168             if (String.IsNullOrWhiteSpace(container))
 169                 throw new ArgumentNullException("container", "The container property can't be empty");
 170
 171             Trace.TraceInformation("[START] ListObjects");
 172
 173             var request = new RestRequest { Path = container,RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) };
 174             request.AddParameter("format", "json");
 175             request.AddParameter("path", folder);
 176             var response = _client.Request(request);
 177
 178             var infos = InfosFromContent(response);
 179
 180             Trace.TraceInformation("[END] ListObjects");
 181             return infos;
 182         }
 183
 184         private static IList<ObjectInfo> InfosFromContent(RestResponse response)
 185         {
 186             if (response.TimedOut)
 187                 return new List<ObjectInfo>();
 188
 189             if (response.StatusCode == 0)
 190                 return new List<ObjectInfo>();
 191
 192             if (response.StatusCode == HttpStatusCode.NoContent)
 193                 return new List<ObjectInfo>();
 194
 195
 196             var statusCode = (int)response.StatusCode;
 197             if (statusCode < 200 || statusCode >= 300)
 198             {
 199                 Trace.TraceWarning("ListObjects failed with code {0} - {1}", response.StatusCode, response.StatusDescription);
 200                 return new List<ObjectInfo>();
 201             }
 202
 203             var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(response.Content);
 204             return infos;
 205         }
 206
 207         public bool ContainerExists(string container)
 208         {
 209             if (String.IsNullOrWhiteSpace(container))
 210                 throw new ArgumentNullException("container", "The container property can't be empty");
 211
 212             var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
 213             var response = _client.Request(request);
 214
 215             switch(response.StatusCode)
 216             {
 217                 case HttpStatusCode.NoContent:
 218                     return true;
 219                 case HttpStatusCode.NotFound:
 220                     return false;
 221                 default:
 222                     throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode));
 223             }
 224         }
 225
 226         public bool ObjectExists(string container,string objectName)
 227         {
 228             if (String.IsNullOrWhiteSpace(container))
 229                 throw new ArgumentNullException("container", "The container property can't be empty");
 230             if (String.IsNullOrWhiteSpace(objectName))
 231                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 232
 233
 234             var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head,RetryPolicy = _retryPolicy, Timeout = _shortTimeout };
 235             var response = _client.Request(request);
 236
 237             switch (response.StatusCode)
 238             {
 239                 case HttpStatusCode.OK:
 240                 case HttpStatusCode.NoContent:
 241                     return true;
 242                 case HttpStatusCode.NotFound:
 243                     return false;
 244                 default:
 245                     throw new WebException(String.Format("ObjectExists failed with unexpected status code {0}", response.StatusCode));
 246             }
 247
 248         }
 249
 250         public ObjectInfo GetObjectInfo(string container, string objectName)
 251         {
 252             if (String.IsNullOrWhiteSpace(container))
 253                 throw new ArgumentNullException("container", "The container property can't be empty");
 254             if (String.IsNullOrWhiteSpace(objectName))
 255                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 256
 257
 258             var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
 259             var response = _client.Request(request);
 260
 261             if (response.TimedOut)
 262                 return ObjectInfo.Empty;
 263
 264             switch (response.StatusCode)
 265             {
 266                 case HttpStatusCode.OK:
 267                 case HttpStatusCode.NoContent:
 268                     var keys = response.Headers.AllKeys.AsQueryable();
 269                     var tags=(from key in keys
 270                              where key.StartsWith("X-Object-Meta-")
 271                              let name=key.Substring(14)
 272                              select new {Name=name,Value=response.Headers[name]})
 273                              .ToDictionary(t=>t.Name,t=>t.Value);
 274                     return new ObjectInfo
 275                                {
 276                                    Name = objectName,
 277                                    Bytes = long.Parse(GetHeaderValue("Content-Length", response, keys)),
 278                                    Hash = GetHeaderValue("ETag", response, keys),
 279                                    Content_Type = GetHeaderValue("Content-Type", response, keys),
 280                                    Tags=tags
 281                                };
 282                 case HttpStatusCode.NotFound:
 283                     return ObjectInfo.Empty;
 284                 default:
 285                     if (request.RetryState.RepeatCount > 0)
 286                     {
 287                         Trace.TraceWarning("[RETRY FAIL] GetObjectInfo for {0} failed after {1} retries",
 288                                                       objectName, request.RetryState.RepeatCount);
 289                         return ObjectInfo.Empty;
 290                     }
 291                     if (response.InnerException != null)
 292                         throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode), response.InnerException);
 293                     throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode));
 294             }
 295         }
 296
 297         public void CreateFolder(string container, string folder)
 298         {
 299             if (String.IsNullOrWhiteSpace(container))
 300                 throw new ArgumentNullException("container", "The container property can't be empty");
 301             if (String.IsNullOrWhiteSpace(folder))
 302                 throw new ArgumentNullException("folder", "The folder property can't be empty");
 303
 304             var folderUrl=String.Format("{0}/{1}",container,folder);
 305             var request = new RestRequest { Path = folderUrl, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
 306             request.AddHeader("Content-Type", @"application/directory");
 307             request.AddHeader("Content-Length", "0");
 308
 309             var response = _client.Request(request);
 310
 311             if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)
 312                 throw new WebException(String.Format("CreateFolder failed with unexpected status code {0}", response.StatusCode));
 313
 314         }
 315
 316         public ContainerInfo GetContainerInfo(string container)
 317         {
 318             if (String.IsNullOrWhiteSpace(container))
 319                 throw new ArgumentNullException("container", "The container property can't be empty");
 320
 321             var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
 322             var response = _client.Request(request);
 323
 324             switch(response.StatusCode)
 325             {
 326                 case HttpStatusCode.NoContent:
 327                     var keys = response.Headers.AllKeys.AsQueryable();
 328                     var containerInfo = new ContainerInfo
 329                                             {
 330                                                 Name = container,
 331                                                 Count =long.Parse(GetHeaderValue("X-Container-Object-Count", response, keys)),
 332                                                 Bytes =long.Parse(GetHeaderValue("X-Container-Bytes-Used", response, keys))
 333                                             };
 334                     return containerInfo;
 335                 case HttpStatusCode.NotFound:
 336                     return ContainerInfo.Empty;
 337                 default:
 338                     throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode));
 339             }
 340         }
 341
 342         public void CreateContainer(string container)
 343         {
 344             if (String.IsNullOrWhiteSpace(container))
 345                 throw new ArgumentNullException("container", "The container property can't be empty");
 346
 347             var request = new RestRequest { Path = container, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
 348
 349             var response = _client.Request(request);
 350
 351             if (response.StatusCode!=HttpStatusCode.Created && response.StatusCode!=HttpStatusCode.Accepted )
 352                     throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}", response.StatusCode));
 353         }
 354
 355         public void DeleteContainer(string container)
 356         {
 357             if (String.IsNullOrWhiteSpace(container))
 358                 throw new ArgumentNullException("container", "The container property can't be empty");
 359
 360             var request = new RestRequest { Path = container, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
 361             var response = _client.Request(request);
 362
 363             if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent)
 364                 return;
 365             else
 366                 throw new WebException(String.Format("DeleteContainer failed with unexpected status code {0}", response.StatusCode));
 367
 368         }
 369
 370         /// <summary>
 371         ///
 372         /// </summary>
 373         /// <param name="container"></param>
 374         /// <param name="objectName"></param>
 375         /// <returns></returns>
 376         /// <remarks>>This method should have no timeout or a very long one</remarks>
 377         public Stream GetObject(string container, string objectName)
 378         {
 379             if (String.IsNullOrWhiteSpace(container))
 380                 throw new ArgumentNullException("container", "The container property can't be empty");
 381             if (String.IsNullOrWhiteSpace(objectName))
 382                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 383
 384             var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Get };
 385 /*
 386             if (DownloadPercentLimit > 0)
 387                 request.TaskOptions = new TaskOptions<int> { RateLimitPercent = DownloadPercentLimit };
 388 */
 389
 390             var response = _client.Request(request);
 391
 392             if (response.StatusCode == HttpStatusCode.NotFound)
 393                 throw new FileNotFoundException();
 394             if (response.StatusCode == HttpStatusCode.OK)
 395             {
 396                 return response.ContentStream;
 397             }
 398             else
 399                 throw new WebException(String.Format("GetObject failed with unexpected status code {0}", response.StatusCode));
 400         }
 401
 402         /// <summary>
 403         ///
 404         /// </summary>
 405         /// <param name="container"></param>
 406         /// <param name="objectName"></param>
 407         /// <param name="fileName"></param>
 408         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
 409         /// <remarks>>This method should have no timeout or a very long one</remarks>
 410         public Task PutObject(string container, string objectName, string fileName, string hash = null)
 411         {
 412             if (String.IsNullOrWhiteSpace(container))
 413                 throw new ArgumentNullException("container", "The container property can't be empty");
 414             if (String.IsNullOrWhiteSpace(objectName))
 415                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 416             if (String.IsNullOrWhiteSpace(fileName))
 417                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
 418             if (!File.Exists(fileName))
 419                 throw new FileNotFoundException("The file does not exist",fileName);
 420
 421
 422             try
 423             {
 424                 var url = String.Join("/",new[]{_client.Authority,container,objectName});
 425                 var uri = new Uri(url);
 426
 427                 var client = new WebClient();
 428                 string etag = hash ?? CalculateHash(fileName);
 429
 430                 client.Headers.Add("Content-Type", "application/octet-stream");
 431                 client.Headers.Add("ETag", etag);
 432
 433                 if(!String.IsNullOrWhiteSpace(_client.Proxy))
 434                     client.Proxy = new WebProxy(_client.Proxy);
 435
 436                 CopyHeaders(_client, client);
 437
 438                 Trace.TraceInformation("[PUT] START {0}", objectName);
 439                 client.UploadProgressChanged += (sender, args) =>
 440                 {
 441                     Trace.TraceInformation("[PROGRESS] {0} {1}% {2} of {3}", fileName, args.ProgressPercentage, args.BytesSent, args.TotalBytesToSend);
 442                 };
 443
 444                 return client.UploadFileTask(uri, "PUT", fileName)
 445                     .ContinueWith(upload=>
 446                                       {
 447                                           client.Dispose();
 448
 449                                           if (upload.IsFaulted)
 450                                           {
 451                                               Trace.TraceError("[PUT] FAIL for {0} with \r{1}",objectName,upload.Exception);
 452                                           }
 453                                           else
 454                                             Trace.TraceInformation("[PUT] END {0}", objectName);
 455                                       });
 456             }
 457             catch (Exception exc)
 458             {
 459                 Trace.TraceError("[PUT] END {0} with {1}", objectName, exc);
 460                 throw;
 461             }
 462
 463         }
 464
 465
 466         /// <summary>
 467         /// Copies headers from a Hammock RestClient to a WebClient
 468         /// </summary>
 469         /// <param name="source">The RestClient from which the headers are copied</param>
 470         /// <param name="target">The WebClient to which the headers are copied</param>
 471         private static void CopyHeaders(RestClient source, WebClient target)
 472         {
 473             Contract.Requires(source!=null,"source can't be null");
 474             Contract.Requires(target != null, "target can't be null");
 475             if (source == null)
 476                 throw new ArgumentNullException("source", "source can't be null");
 477             if (source == null)
 478                 throw new ArgumentNullException("target", "target can't be null");
 479
 480             foreach (var header in source.GetAllHeaders())
 481             {
 482                 target.Headers.Add(header.Name, header.Value);
 483             }
 484         }
 485
 486         private static string CalculateHash(string fileName)
 487         {
 488             string hash;
 489             using (var hasher = MD5.Create())
 490             using(var stream=File.OpenRead(fileName))
 491             {
 492                 var hashBuilder=new StringBuilder();
 493                 foreach (byte b in hasher.ComputeHash(stream))
 494                     hashBuilder.Append(b.ToString("x2").ToLower());
 495                 hash = hashBuilder.ToString();
 496             }
 497             return hash;
 498         }
 499
 500         public void DeleteObject(string container, string objectName)
 501         {
 502             if (String.IsNullOrWhiteSpace(container))
 503                 throw new ArgumentNullException("container", "The container property can't be empty");
 504             if (String.IsNullOrWhiteSpace(objectName))
 505                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 506
 507             var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
 508             var response = _client.Request(request);
 509
 510             if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent)
 511                 return;
 512             else
 513                 throw new WebException(String.Format("DeleteObject failed with unexpected status code {0}", response.StatusCode));
 514
 515         }
 516
 517         public void MoveObject(string container, string oldObjectName, string newObjectName)
 518         {
 519             if (String.IsNullOrWhiteSpace(container))
 520                 throw new ArgumentNullException("container", "The container property can't be empty");
 521             if (String.IsNullOrWhiteSpace(oldObjectName))
 522                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
 523             if (String.IsNullOrWhiteSpace(newObjectName))
 524                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
 525
 526             var request = new RestRequest { Path = container + "/" + newObjectName, Method = WebMethod.Put };
 527             request.AddHeader("X-Copy-From",String.Format("/{0}/{1}",container,oldObjectName));
 528             request.AddPostContent(new byte[]{});
 529             var response = _client.Request(request);
 530
 531             if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.NoContent || response.StatusCode==HttpStatusCode.Created)
 532             {
 533                 this.DeleteObject(container,oldObjectName);
 534             }
 535             else
 536                 throw new WebException(String.Format("MoveObject failed with unexpected status code {0}", response.StatusCode));
 537         }
 538
 539         private string GetHeaderValue(string headerName, RestResponse response, IQueryable<string> keys)
 540         {
 541             if (keys.Any(key => key == headerName))
 542                 return response.Headers[headerName];
 543             else
 544                 throw new WebException(String.Format("The {0}  header is missing",headerName));
 545         }
 546
 547         private static void ThrowIfNotStatusOK(RestResponse response, string message)
 548         {
 549             int status = (int)response.StatusCode;
 550             if (status < 200 || status >= 300)
 551                 throw new WebException(String.Format("{0} with code {1}",message, status));
 552         }
 553     }
 554 }