code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel.Composition;
   4 using System.Diagnostics;
   5 using System.Diagnostics.Contracts;
   6 using System.IO;
   7 using System.Linq;
   8 using System.Net;
   9 using System.Security.Cryptography;
  10 using System.Text;
  11 using Hammock;
  12 using Hammock.Caching;
  13 using Hammock.Retries;
  14 using Hammock.Serialization;
  15 using Hammock.Tasks;
  16 using Hammock.Web;
  17 using Newtonsoft.Json;
  18 using Pithos.Interfaces;
  19
  20 namespace Pithos.Network
  21 {
  22     [Export(typeof(ICloudClient))]
  23     public class CloudFilesClient:ICloudClient
  24     {
  25         string _rackSpaceAuthUrl = "https://auth.api.rackspacecloud.com";
  26         private string _pithosAuthUrl = "http://pithos.dev.grnet.gr";
  27
  28         private RestClient _client;
  29         private readonly TimeSpan _shortTimeout = TimeSpan.FromSeconds(10);
  30         private readonly int _retries = 5;
  31         public string ApiKey { get; set; }
  32         public string UserName { get; set; }
  33         public Uri StorageUrl { get; set; }
  34         public string Token { get; set; }
  35         public Uri Proxy { get; set; }
  36
  37         public string AuthUrl
  38         {
  39             get { return UsePithos ? _pithosAuthUrl : _rackSpaceAuthUrl; }
  40         }
  41
  42         public string VersionPath
  43         {
  44             get { return UsePithos ? "v1" : "v1.0"; }
  45         }
  46
  47         public bool UsePithos { get; set; }
  48
  49         public void Authenticate(string userName,string apiKey)
  50         {
  51             if (String.IsNullOrWhiteSpace(userName))
  52                 throw new ArgumentNullException("userName","The userName property can't be empty");
  53             if (String.IsNullOrWhiteSpace(apiKey))
  54                 throw new ArgumentNullException("apiKey", "The apiKey property can't be empty");
  55
  56
  57             UserName = userName;
  58             ApiKey = apiKey;
  59
  60             string authUrl = UsePithos ? String.Format("{0}/{1}/{2}", AuthUrl, VersionPath,UserName)
  61                                     : String.Format("{0}/{1}", AuthUrl, VersionPath);
  62
  63             var proxy = Proxy != null ? Proxy.ToString():null;
  64
  65             var authClient = new RestClient{Path=authUrl,Proxy=proxy};
  66
  67             authClient.AddHeader("X-Auth-User", UserName);
  68             authClient.AddHeader("X-Auth-Key", ApiKey);
  69
  70             var response=authClient.Request();
  71
  72             ThrowIfNotStatusOK(response, "Authentication failed");
  73
  74             var keys = response.Headers.AllKeys.AsQueryable();
  75
  76             string storageUrl =UsePithos?
  77                 String.Format("{0}/{1}/{2}",AuthUrl,VersionPath,UserName)
  78                 :GetHeaderValue("X-Storage-Url", response, keys);
  79
  80             if (String.IsNullOrWhiteSpace(storageUrl))
  81                 throw new InvalidOperationException("Failed to obtain storage url");
  82             StorageUrl = new Uri(storageUrl);
  83
  84             if (!UsePithos)
  85             {
  86                 var token = GetHeaderValue("X-Auth-Token", response, keys);
  87                 if (String.IsNullOrWhiteSpace(token))
  88                     throw new InvalidOperationException("Failed to obtain token url");
  89                 Token = token;
  90             }
  91             else
  92                 Token = "0000";
  93
  94             var retryPolicy = new RetryPolicy { RetryCount = _retries };
  95             retryPolicy.RetryConditions.Add(new TimeoutRetryCondition());
  96
  97             _client = new RestClient { Authority = StorageUrl.AbsoluteUri, Path = UserName, Proxy = proxy, RetryPolicy = retryPolicy, };
  98
  99             _client.AddHeader("X-Auth-Token", Token);
 100             if (UsePithos)
 101             {
 102                 _client.AddHeader("X-Auth-User", UserName);
 103                 _client.AddHeader("X-Auth-Key",ApiKey);
 104             }
 105
 106
 107         }
 108
 109         public IList<ContainerInfo> ListContainers()
 110         {
 111             //Workaround for Hammock quirk: Hammock always
 112             //appends a / unless a Path is specified.
 113
 114             //Create a request with a complete path
 115             var request = new RestRequest { Path = StorageUrl.ToString(), Timeout = _shortTimeout };
 116             request.AddParameter("format","json");
 117             //Create a client clone
 118             var client = new RestClient{Proxy=Proxy.ToString()};
 119             foreach (var header in _client.GetAllHeaders())
 120             {
 121                 client.AddHeader(header.Name,header.Value);
 122             }
 123
 124             var response = client.Request(request);
 125
 126             if (response.StatusCode == HttpStatusCode.NoContent)
 127                 return new List<ContainerInfo>();
 128
 129             ThrowIfNotStatusOK(response, "List Containers failed");
 130
 131
 132             var infos=JsonConvert.DeserializeObject<IList<ContainerInfo>>(response.Content);
 133
 134             return infos;
 135         }
 136
 137         public IList<ObjectInfo> ListObjects(string container)
 138         {
 139             if (String.IsNullOrWhiteSpace(container))
 140                 throw new ArgumentNullException("container", "The container property can't be empty");
 141
 142             var request = new RestRequest { Path = container, Timeout = _shortTimeout };
 143             request.AddParameter("format", "json");
 144             var response = _client.Request(request);
 145
 146             var infos = InfosFromContent(response);
 147
 148             return infos;
 149         }
 150
 151
 152
 153         public IList<ObjectInfo> ListObjects(string container,string folder)
 154         {
 155             if (String.IsNullOrWhiteSpace(container))
 156                 throw new ArgumentNullException("container", "The container property can't be empty");
 157
 158             var request = new RestRequest { Path = container, Timeout = _shortTimeout };
 159             request.AddParameter("format", "json");
 160             request.AddParameter("path", folder);
 161             var response = _client.Request(request);
 162
 163             var infos = InfosFromContent(response);
 164
 165             return infos;
 166         }
 167
 168         private static IList<ObjectInfo> InfosFromContent(RestResponse response)
 169         {
 170             if (response.TimedOut)
 171                 return new List<ObjectInfo>();
 172
 173             if (response.StatusCode == 0)
 174                 return new List<ObjectInfo>();
 175
 176             if (response.StatusCode == HttpStatusCode.NoContent)
 177                 return new List<ObjectInfo>();
 178
 179
 180             var statusCode = (int)response.StatusCode;
 181             if (statusCode < 200 || statusCode >= 300)
 182             {
 183                 Trace.TraceWarning("ListObjects failed with code {1} - {2}", response.StatusCode, response.StatusDescription);
 184                 return new List<ObjectInfo>();
 185             }
 186
 187             var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(response.Content);
 188             return infos;
 189         }
 190
 191         public bool ContainerExists(string container)
 192         {
 193             if (String.IsNullOrWhiteSpace(container))
 194                 throw new ArgumentNullException("container", "The container property can't be empty");
 195
 196             var request = new RestRequest { Path = container, Method = WebMethod.Head, Timeout = _shortTimeout };
 197             var response = _client.Request(request);
 198
 199             switch(response.StatusCode)
 200             {
 201                 case HttpStatusCode.NoContent:
 202                     return true;
 203                 case HttpStatusCode.NotFound:
 204                     return false;
 205                 default:
 206                     throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode));
 207             }
 208         }
 209
 210         public bool ObjectExists(string container,string objectName)
 211         {
 212             if (String.IsNullOrWhiteSpace(container))
 213                 throw new ArgumentNullException("container", "The container property can't be empty");
 214             if (String.IsNullOrWhiteSpace(objectName))
 215                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 216
 217
 218             var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head, Timeout = _shortTimeout };
 219             var response = _client.Request(request);
 220
 221             switch (response.StatusCode)
 222             {
 223                 case HttpStatusCode.OK:
 224                 case HttpStatusCode.NoContent:
 225                     return true;
 226                 case HttpStatusCode.NotFound:
 227                     return false;
 228                 default:
 229                     throw new WebException(String.Format("ObjectExists failed with unexpected status code {0}", response.StatusCode));
 230             }
 231
 232         }
 233
 234         public ObjectInfo GetObjectInfo(string container, string objectName)
 235         {
 236             if (String.IsNullOrWhiteSpace(container))
 237                 throw new ArgumentNullException("container", "The container property can't be empty");
 238             if (String.IsNullOrWhiteSpace(objectName))
 239                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 240
 241
 242             var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head, Timeout = _shortTimeout };
 243             var response = _client.Request(request);
 244
 245             if (response.TimedOut)
 246                 return ObjectInfo.Empty;
 247
 248             switch (response.StatusCode)
 249             {
 250                 case HttpStatusCode.OK:
 251                 case HttpStatusCode.NoContent:
 252                     var keys = response.Headers.AllKeys.AsQueryable();
 253                     return new ObjectInfo
 254                                {
 255                                    Name=objectName,
 256                                    Bytes = long.Parse(GetHeaderValue("Content-Length", response, keys)),
 257                                    Hash = GetHeaderValue("ETag", response, keys),
 258                                    Content_Type = GetHeaderValue("Content-Type", response, keys)
 259                                };
 260                 case HttpStatusCode.NotFound:
 261                     return ObjectInfo.Empty;
 262                 default:
 263                         throw new WebException(String.Format("GetObjectInfo failed with unexpected status code {0}", response.StatusCode));
 264             }
 265         }
 266
 267         public void CreateFolder(string container, string folder)
 268         {
 269             if (String.IsNullOrWhiteSpace(container))
 270                 throw new ArgumentNullException("container", "The container property can't be empty");
 271             if (String.IsNullOrWhiteSpace(folder))
 272                 throw new ArgumentNullException("folder", "The folder property can't be empty");
 273
 274             var folderUrl=String.Format("{0}/{1}",container,folder);
 275             var request = new RestRequest { Path = folderUrl, Method = WebMethod.Put, Timeout = _shortTimeout };
 276             request.AddHeader("Content-Type", @"application/directory");
 277             request.AddHeader("Content-Length", "0");
 278
 279             var response = _client.Request(request);
 280
 281             if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)
 282                 throw new WebException(String.Format("CreateFolder failed with unexpected status code {0}", response.StatusCode));
 283
 284         }
 285
 286         public ContainerInfo GetContainerInfo(string container)
 287         {
 288             if (String.IsNullOrWhiteSpace(container))
 289                 throw new ArgumentNullException("container", "The container property can't be empty");
 290
 291             var request = new RestRequest { Path = container, Method = WebMethod.Head, Timeout = _shortTimeout };
 292             var response = _client.Request(request);
 293
 294             switch(response.StatusCode)
 295             {
 296                 case HttpStatusCode.NoContent:
 297                     var keys = response.Headers.AllKeys.AsQueryable();
 298                     var containerInfo = new ContainerInfo
 299                                             {
 300                                                 Name = container,
 301                                                 Count =long.Parse(GetHeaderValue("X-Container-Object-Count", response, keys)),
 302                                                 Bytes =long.Parse(GetHeaderValue("X-Container-Bytes-Used", response, keys))
 303                                             };
 304                     return containerInfo;
 305                 case HttpStatusCode.NotFound:
 306                     return ContainerInfo.Empty;
 307                 default:
 308                     throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode));
 309             }
 310         }
 311
 312         public void CreateContainer(string container)
 313         {
 314             if (String.IsNullOrWhiteSpace(container))
 315                 throw new ArgumentNullException("container", "The container property can't be empty");
 316
 317             var request = new RestRequest { Path = container, Method = WebMethod.Put, Timeout = _shortTimeout };
 318
 319             var response = _client.Request(request);
 320
 321             if (response.StatusCode!=HttpStatusCode.Created && response.StatusCode!=HttpStatusCode.Accepted )
 322                     throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}", response.StatusCode));
 323         }
 324
 325         public void DeleteContainer(string container)
 326         {
 327             if (String.IsNullOrWhiteSpace(container))
 328                 throw new ArgumentNullException("container", "The container property can't be empty");
 329
 330             var request = new RestRequest { Path = container, Method = WebMethod.Delete, Timeout = _shortTimeout };
 331             var response = _client.Request(request);
 332
 333             if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent)
 334                 return;
 335             else
 336                 throw new WebException(String.Format("DeleteContainer failed with unexpected status code {0}", response.StatusCode));
 337
 338         }
 339
 340         /// <summary>
 341         ///
 342         /// </summary>
 343         /// <param name="container"></param>
 344         /// <param name="objectName"></param>
 345         /// <returns></returns>
 346         /// <remarks>>This method should have no timeout or a very long one</remarks>
 347         public Stream GetObject(string container, string objectName)
 348         {
 349             if (String.IsNullOrWhiteSpace(container))
 350                 throw new ArgumentNullException("container", "The container property can't be empty");
 351             if (String.IsNullOrWhiteSpace(objectName))
 352                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 353
 354             var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Get };
 355             var response = _client.Request(request);
 356
 357             if (response.StatusCode == HttpStatusCode.NotFound)
 358                 throw new FileNotFoundException();
 359             if (response.StatusCode == HttpStatusCode.OK)
 360             {
 361                 return response.ContentStream;
 362             }
 363             else
 364                 throw new WebException(String.Format("GetObject failed with unexpected status code {0}", response.StatusCode));
 365         }
 366
 367         /// <summary>
 368         ///
 369         /// </summary>
 370         /// <param name="container"></param>
 371         /// <param name="objectName"></param>
 372         /// <param name="fileName"></param>
 373         /// <remarks>>This method should have no timeout or a very long one</remarks>
 374         public void PutObject(string container, string objectName, string fileName)
 375         {
 376             if (String.IsNullOrWhiteSpace(container))
 377                 throw new ArgumentNullException("container", "The container property can't be empty");
 378             if (String.IsNullOrWhiteSpace(objectName))
 379                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 380             if (String.IsNullOrWhiteSpace(fileName))
 381                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
 382             if (!File.Exists(fileName))
 383                 throw new FileNotFoundException("The file does not exist",fileName);
 384
 385
 386             string url = container + "/" + objectName;
 387
 388             var request = new RestRequest {Path=url,Method=WebMethod.Put};
 389             request.TaskOptions=new TaskOptions<int>{RateLimitPercent=0.5};
 390
 391             string hash = CalculateHash(fileName);
 392
 393             request.AddPostContent(File.ReadAllBytes(fileName));
 394             request.AddHeader("Content-Type","application/octet-stream");
 395             request.AddHeader("ETag",hash);
 396             var response=_client.Request(request);
 397             _client.TaskOptions = new TaskOptions<int> {RateLimitPercent = 0.5};
 398             if (response.StatusCode == HttpStatusCode.Created)
 399                 return;
 400             if (response.StatusCode == HttpStatusCode.LengthRequired)
 401                 throw new InvalidOperationException();
 402             else
 403                 throw new WebException(String.Format("GetObject failed with unexpected status code {0}", response.StatusCode));
 404         }
 405
 406         private static string CalculateHash(string fileName)
 407         {
 408             string hash;
 409             using (var hasher = MD5.Create())
 410             using(var stream=File.OpenRead(fileName))
 411             {
 412                 var hashBuilder=new StringBuilder();
 413                 foreach (byte b in hasher.ComputeHash(stream))
 414                     hashBuilder.Append(b.ToString("x2").ToLower());
 415                 hash = hashBuilder.ToString();
 416             }
 417             return hash;
 418         }
 419
 420         public void DeleteObject(string container, string objectName)
 421         {
 422             if (String.IsNullOrWhiteSpace(container))
 423                 throw new ArgumentNullException("container", "The container property can't be empty");
 424             if (String.IsNullOrWhiteSpace(objectName))
 425                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 426
 427             var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Delete, Timeout=_shortTimeout };
 428             var response = _client.Request(request);
 429
 430             if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent)
 431                 return;
 432             else
 433                 throw new WebException(String.Format("DeleteObject failed with unexpected status code {0}", response.StatusCode));
 434
 435         }
 436
 437         public void MoveObject(string container, string oldObjectName, string newObjectName)
 438         {
 439             if (String.IsNullOrWhiteSpace(container))
 440                 throw new ArgumentNullException("container", "The container property can't be empty");
 441             if (String.IsNullOrWhiteSpace(oldObjectName))
 442                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
 443             if (String.IsNullOrWhiteSpace(newObjectName))
 444                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
 445
 446             var request = new RestRequest { Path = container + "/" + newObjectName, Method = WebMethod.Put };
 447             request.AddHeader("X-Copy-From",String.Format("/{0}/{1}",container,oldObjectName));
 448             request.AddPostContent(new byte[]{});
 449             var response = _client.Request(request);
 450
 451             if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.NoContent || response.StatusCode==HttpStatusCode.Created)
 452             {
 453                 this.DeleteObject(container,oldObjectName);
 454             }
 455             else
 456                 throw new WebException(String.Format("MoveObject failed with unexpected status code {0}", response.StatusCode));
 457         }
 458
 459         private string GetHeaderValue(string headerName, RestResponse response, IQueryable<string> keys)
 460         {
 461             if (keys.Any(key => key == headerName))
 462                 return response.Headers[headerName];
 463             else
 464                 throw new WebException(String.Format("The {0}  header is missing",headerName));
 465         }
 466
 467         private static void ThrowIfNotStatusOK(RestResponse response, string message)
 468         {
 469             int status = (int)response.StatusCode;
 470             if (status < 200 || status >= 300)
 471                 throw new WebException(String.Format("{0} with code {1}",message, status));
 472         }
 473     }
 474 }