code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
   2 //
   3 // The class provides methods to upload/download files, delete files, manage containers
   4
   5
   6 using System;
   7 using System.Collections.Generic;
   8 using System.ComponentModel.Composition;
   9 using System.Diagnostics;
  10 using System.Diagnostics.Contracts;
  11 using System.Globalization;
  12 using System.IO;
  13 using System.Linq;
  14 using System.Net;
  15 using System.Security.Cryptography;
  16 using System.Text;
  17 using System.Threading.Algorithms;
  18 using System.Threading.Tasks;
  19 using Newtonsoft.Json;
  20 using Pithos.Interfaces;
  21 using WebHeaderCollection = System.Net.WebHeaderCollection;
  22
  23 namespace Pithos.Network
  24 {
  25     [Export(typeof(ICloudClient))]
  26     public class CloudFilesClient:ICloudClient
  27     {
  28         //CloudFilesClient uses *_baseClient* internally to communicate with the server
  29         //RestClient provides a REST-friendly interface over the standard WebClient.
  30         private RestClient _baseClient;
  31
  32         //Some operations can specify a Timeout. The default value of all timeouts is 10 seconds
  33         private readonly TimeSpan _shortTimeout = TimeSpan.FromSeconds(10);
  34
  35         //Some operations can be retried before failing. The default number of retries is 5
  36         private readonly int _retries = 5;
  37
  38         //During authentication the client provides a UserName
  39         public string UserName { get; set; }
  40
  41         //and and ApiKey to the server
  42         public string ApiKey { get; set; }
  43
  44         //And receives an authentication Token. This token must be provided in ALL other operations,
  45         //in the X-Auth-Token header
  46         public string Token { get; set; }
  47
  48         //The client also receives a StorageUrl after authentication. All subsequent operations must
  49         //use this url
  50         public Uri StorageUrl { get; set; }
  51
  52         public Uri Proxy { get; set; }
  53
  54         public double DownloadPercentLimit { get; set; }
  55         public double UploadPercentLimit { get; set; }
  56
  57         public string AuthenticationUrl { get; set; }
  58
  59
  60         public string VersionPath
  61         {
  62             get { return UsePithos ? "v1" : "v1.0"; }
  63         }
  64
  65         public bool UsePithos { get; set; }
  66
  67         private bool _authenticated = false;
  68
  69         //
  70         public void Authenticate(string userName,string apiKey)
  71         {
  72             Trace.TraceInformation("[AUTHENTICATE] Start for {0}", userName);
  73             if (String.IsNullOrWhiteSpace(userName))
  74                 throw new ArgumentNullException("userName", "The userName property can't be empty");
  75             if (String.IsNullOrWhiteSpace(apiKey))
  76                 throw new ArgumentNullException("apiKey", "The apiKey property can't be empty");
  77
  78             if (_authenticated)
  79                 return;
  80
  81             UserName = userName;
  82             ApiKey = apiKey;
  83
  84
  85             using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
  86             {
  87                 if (Proxy != null)
  88                     authClient.Proxy = new WebProxy(Proxy);
  89
  90                 authClient.Headers.Add("X-Auth-User", UserName);
  91                 authClient.Headers.Add("X-Auth-Key", ApiKey);
  92
  93                 authClient.DownloadStringWithRetry(VersionPath, 3);
  94
  95                 authClient.AssertStatusOK("Authentication failed");
  96
  97                 var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
  98                 if (String.IsNullOrWhiteSpace(storageUrl))
  99                     throw new InvalidOperationException("Failed to obtain storage url");
 100                 StorageUrl = new Uri(storageUrl);
 101
 102                 var token = authClient.GetHeaderValue("X-Auth-Token");
 103                 if (String.IsNullOrWhiteSpace(token))
 104                     throw new InvalidOperationException("Failed to obtain token url");
 105                 Token = token;
 106             }
 107
 108             _baseClient = new RestClient{
 109                 BaseAddress  = StorageUrl.AbsoluteUri,
 110                 Timeout=10000,
 111                 Retries=3};
 112             if (Proxy!=null)
 113                 _baseClient.Proxy = new WebProxy(Proxy);
 114
 115             _baseClient.Headers.Add("X-Auth-Token", Token);
 116
 117             Trace.TraceInformation("[AUTHENTICATE] End for {0}", userName);
 118         }
 119
 120
 121         public IList<ContainerInfo> ListContainers()
 122         {
 123             using (var client = new RestClient(_baseClient))
 124             {
 125                 client.Parameters.Clear();
 126                 client.Parameters.Add("format", "json");
 127                 var content = client.DownloadStringWithRetry("", 3);
 128                 client.AssertStatusOK("List Containers failed");
 129
 130                 if (client.StatusCode == HttpStatusCode.NoContent)
 131                     return new List<ContainerInfo>();
 132                 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
 133                 return infos;
 134             }
 135
 136         }
 137
 138         //Request listing of all objects in a container modified since a specific time.
 139         //If the *since* value is missing, return all objects
 140         public IList<ObjectInfo> ListObjects(string container, DateTime? since = null)
 141         {
 142             if (String.IsNullOrWhiteSpace(container))
 143                 throw new ArgumentNullException("container");
 144             Contract.EndContractBlock();
 145
 146             Trace.TraceInformation("[START] ListObjects");
 147
 148             using (var client = new RestClient(_baseClient))
 149             {
 150                 client.Parameters.Clear();
 151                 client.Parameters.Add("format", "json");
 152                 client.IfModifiedSince = since;
 153                 var content = client.DownloadStringWithRetry(container, 3);
 154
 155                 client.AssertStatusOK("ListObjects failed");
 156
 157                 //If the result is empty, return an empty list,
 158                 var infos=String.IsNullOrWhiteSpace(content)
 159                     ? new List<ObjectInfo>()
 160                     //Otherwise deserialize the object list into a list of ObjectInfos
 161                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 162
 163                 Trace.TraceInformation("[END] ListObjects");
 164                 return infos;
 165             }
 166         }
 167
 168
 169
 170         public IList<ObjectInfo> ListObjects(string container, string folder, DateTime? since = null)
 171         {
 172             if (String.IsNullOrWhiteSpace(container))
 173                 throw new ArgumentNullException("container");
 174             if (String.IsNullOrWhiteSpace(folder))
 175                 throw new ArgumentNullException("folder");
 176             Contract.EndContractBlock();
 177
 178             Trace.TraceInformation("[START] ListObjects");
 179
 180             using (var client = new RestClient(_baseClient))
 181             {
 182                 client.Parameters.Clear();
 183                 client.Parameters.Add("format", "json");
 184                 client.Parameters.Add("path", folder);
 185                 client.IfModifiedSince = since;
 186                 var content = client.DownloadStringWithRetry(container, 3);
 187                 client.AssertStatusOK("ListObjects failed");
 188
 189                 var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 190
 191                 Trace.TraceInformation("[END] ListObjects");
 192                 return infos;
 193             }
 194         }
 195
 196
 197         public bool ContainerExists(string container)
 198         {
 199             if (String.IsNullOrWhiteSpace(container))
 200                 throw new ArgumentNullException("container", "The container property can't be empty");
 201             using (var client = new RestClient(_baseClient))
 202             {
 203                 client.Parameters.Clear();
 204                 client.Head(container, 3);
 205
 206                 switch (client.StatusCode)
 207                 {
 208                     case HttpStatusCode.OK:
 209                     case HttpStatusCode.NoContent:
 210                         return true;
 211                     case HttpStatusCode.NotFound:
 212                         return false;
 213                     default:
 214                         throw CreateWebException("ContainerExists", client.StatusCode);
 215                 }
 216             }
 217         }
 218
 219         public bool ObjectExists(string container,string objectName)
 220         {
 221             if (String.IsNullOrWhiteSpace(container))
 222                 throw new ArgumentNullException("container", "The container property can't be empty");
 223             if (String.IsNullOrWhiteSpace(objectName))
 224                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 225             using (var client = new RestClient(_baseClient))
 226             {
 227                 client.Parameters.Clear();
 228                 client.Head(container + "/" + objectName, 3);
 229
 230                 switch (client.StatusCode)
 231                 {
 232                     case HttpStatusCode.OK:
 233                     case HttpStatusCode.NoContent:
 234                         return true;
 235                     case HttpStatusCode.NotFound:
 236                         return false;
 237                     default:
 238                         throw CreateWebException("ObjectExists", client.StatusCode);
 239                 }
 240             }
 241
 242         }
 243
 244         public ObjectInfo GetObjectInfo(string container, string objectName)
 245         {
 246             if (String.IsNullOrWhiteSpace(container))
 247                 throw new ArgumentNullException("container", "The container property can't be empty");
 248             if (String.IsNullOrWhiteSpace(objectName))
 249                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 250
 251             using (var client = new RestClient(_baseClient))
 252             {
 253                 try
 254                 {
 255                     client.Parameters.Clear();
 256
 257                     client.Head(container + "/" + objectName, 3);
 258
 259                     if (client.TimedOut)
 260                         return ObjectInfo.Empty;
 261
 262                     switch (client.StatusCode)
 263                     {
 264                         case HttpStatusCode.OK:
 265                         case HttpStatusCode.NoContent:
 266                             var keys = client.ResponseHeaders.AllKeys.AsQueryable();
 267                             var tags = (from key in keys
 268                                         where key.StartsWith("X-Object-Meta-")
 269                                         let name = key.Substring(14)
 270                                         select new {Name = name, Value = client.ResponseHeaders[name]})
 271                                 .ToDictionary(t => t.Name, t => t.Value);
 272                             var extensions = (from key in keys
 273                                               where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
 274                                               select new {Name = key, Value = client.ResponseHeaders[key]})
 275                                 .ToDictionary(t => t.Name, t => t.Value);
 276                             var info = new ObjectInfo
 277                                                  {
 278                                                      Name = objectName,
 279                                                      Hash = client.GetHeaderValue("ETag"),
 280                                                      Content_Type = client.GetHeaderValue("Content-Type"),
 281                                                      Tags = tags,
 282                                                      Last_Modified = client.LastModified,
 283                                                      Extensions = extensions
 284                                                  };
 285                             return info;
 286                         case HttpStatusCode.NotFound:
 287                             return ObjectInfo.Empty;
 288                         default:
 289                             throw new WebException(
 290                                 String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 291                                               objectName, client.StatusCode));
 292                     }
 293
 294                 }
 295                 catch(RetryException)
 296                 {
 297                     Trace.TraceWarning("[RETRY FAIL] GetObjectInfo for {0} failed.");
 298                     return ObjectInfo.Empty;
 299                 }
 300                 catch(WebException e)
 301                 {
 302                     Trace.TraceError(
 303                         String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 304                                       objectName, client.StatusCode), e);
 305                     throw;
 306                 }
 307             }
 308
 309         }
 310
 311         public void CreateFolder(string container, string folder)
 312         {
 313             if (String.IsNullOrWhiteSpace(container))
 314                 throw new ArgumentNullException("container", "The container property can't be empty");
 315             if (String.IsNullOrWhiteSpace(folder))
 316                 throw new ArgumentNullException("folder", "The folder property can't be empty");
 317
 318             var folderUrl=String.Format("{0}/{1}",container,folder);
 319             using (var client = new RestClient(_baseClient))
 320             {
 321                 client.Parameters.Clear();
 322                 client.Headers.Add("Content-Type", @"application/directory");
 323                 client.Headers.Add("Content-Length", "0");
 324                 client.PutWithRetry(folderUrl, 3);
 325
 326                 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
 327                     throw CreateWebException("CreateFolder", client.StatusCode);
 328             }
 329         }
 330
 331         public ContainerInfo GetContainerInfo(string container)
 332         {
 333             if (String.IsNullOrWhiteSpace(container))
 334                 throw new ArgumentNullException("container", "The container property can't be empty");
 335             using (var client = new RestClient(_baseClient))
 336             {
 337                 client.Head(container);
 338                 switch (client.StatusCode)
 339                 {
 340                     case HttpStatusCode.OK:
 341                     case HttpStatusCode.NoContent:
 342                         var containerInfo = new ContainerInfo
 343                                                 {
 344                                                     Name = container,
 345                                                     Count =
 346                                                         long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
 347                                                     Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
 348                                                     BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
 349                                                     BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size"))
 350                                                 };
 351                         return containerInfo;
 352                     case HttpStatusCode.NotFound:
 353                         return ContainerInfo.Empty;
 354                     default:
 355                         throw CreateWebException("GetContainerInfo", client.StatusCode);
 356                 }
 357             }
 358         }
 359
 360         public void CreateContainer(string container)
 361         {
 362             if (String.IsNullOrWhiteSpace(container))
 363                 throw new ArgumentNullException("container", "The container property can't be empty");
 364             using (var client = new RestClient(_baseClient))
 365             {
 366                 client.PutWithRetry(container, 3);
 367                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
 368                 if (!expectedCodes.Contains(client.StatusCode))
 369                     throw CreateWebException("CreateContainer", client.StatusCode);
 370             }
 371         }
 372
 373         public void DeleteContainer(string container)
 374         {
 375             if (String.IsNullOrWhiteSpace(container))
 376                 throw new ArgumentNullException("container", "The container property can't be empty");
 377             using (var client = new RestClient(_baseClient))
 378             {
 379                 client.DeleteWithRetry(container, 3);
 380                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
 381                 if (!expectedCodes.Contains(client.StatusCode))
 382                     throw CreateWebException("DeleteContainer", client.StatusCode);
 383             }
 384
 385         }
 386
 387         /// <summary>
 388         ///
 389         /// </summary>
 390         /// <param name="container"></param>
 391         /// <param name="objectName"></param>
 392         /// <param name="fileName"></param>
 393         /// <returns></returns>
 394         /// <remarks>This method should have no timeout or a very long one</remarks>
 395         //Asynchronously download the object specified by *objectName* in a specific *container* to
 396         // a local file
 397         public Task GetObject(string container, string objectName, string fileName)
 398         {
 399             if (String.IsNullOrWhiteSpace(container))
 400                 throw new ArgumentNullException("container", "The container property can't be empty");
 401             if (String.IsNullOrWhiteSpace(objectName))
 402                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 403             Contract.EndContractBlock();
 404
 405             try
 406             {
 407                 //The container and objectName are relative names. They are joined with the client's
 408                 //BaseAddress to create the object's absolute address
 409                 var builder = GetAddressBuilder(container, objectName);
 410                 var uri = builder.Uri;
 411                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
 412                 //object to avoid concurrency errors.
 413                 //
 414                 //Download operations take a long time therefore they have no timeout.
 415                 var client = new RestClient(_baseClient) { Timeout = 0 };
 416
 417                 //Download progress is reported to the Trace log
 418                 Trace.TraceInformation("[GET] START {0}", objectName);
 419                 client.DownloadProgressChanged += (sender, args) =>
 420                     Trace.TraceInformation("[GET PROGRESS] {0} {1}% {2} of {3}",
 421                                     fileName, args.ProgressPercentage,
 422                                     args.BytesReceived,
 423                                     args.TotalBytesToReceive);
 424
 425
 426                 //Start downloading the object asynchronously
 427                 var downloadTask = client.DownloadFileTask(uri, fileName);
 428
 429                 //Once the download completes
 430                 return downloadTask.ContinueWith(download =>
 431                                       {
 432                                           //Delete the local client object
 433                                           client.Dispose();
 434                                           //And report failure or completion
 435                                           if (download.IsFaulted)
 436                                           {
 437                                               Trace.TraceError("[GET] FAIL for {0} with \r{1}", objectName,
 438                                                                download.Exception);
 439                                           }
 440                                           else
 441                                           {
 442                                               Trace.TraceInformation("[GET] END {0}", objectName);
 443                                           }
 444                                       });
 445             }
 446             catch (Exception exc)
 447             {
 448                 Trace.TraceError("[GET] END {0} with {1}", objectName, exc);
 449                 throw;
 450             }
 451
 452
 453
 454         }
 455
 456         public Task<IList<string>> PutHashMap(string container, string objectName, TreeHash hash)
 457         {
 458             if (String.IsNullOrWhiteSpace(container))
 459                 throw new ArgumentNullException("container");
 460             if (String.IsNullOrWhiteSpace(objectName))
 461                 throw new ArgumentNullException("objectName");
 462             if (hash==null)
 463                 throw new ArgumentNullException("hash");
 464             if (String.IsNullOrWhiteSpace(Token))
 465                 throw new InvalidOperationException("Invalid Token");
 466             if (StorageUrl == null)
 467                 throw new InvalidOperationException("Invalid Storage Url");
 468             Contract.EndContractBlock();
 469             //The container and objectName are relative names. They are joined with the client's
 470             //BaseAddress to create the object's absolute address
 471             var builder = GetAddressBuilder(container, objectName);
 472             builder.Query = "format=json&hashmap";
 473             var uri = builder.Uri;
 474
 475             //Don't use a timeout because putting the hashmap may be a long process
 476             var client = new RestClient(_baseClient) { Timeout = 0 };
 477
 478             //Send the tree hash as Json to the server
 479             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
 480             var uploadTask=client.UploadStringTask(uri, "PUT", hash.ToJson());
 481
 482
 483             return uploadTask.ContinueWith(t =>
 484             {
 485
 486                 var empty = (IList<string>)new List<string>();
 487
 488
 489                 //The server will respond either with 201-created if all blocks were already on the server
 490                 if (client.StatusCode == HttpStatusCode.Created)
 491                 {
 492                     //in which case we return an empty hash list
 493                     return empty;
 494                 }
 495                 //or with a 409-conflict and return the list of missing parts
 496                 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
 497                 if (t.IsFaulted)
 498                 {
 499                     var ex = t.Exception.InnerException;
 500                     var we = ex as WebException;
 501                     var response = we.Response as HttpWebResponse;
 502                     if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
 503                     {
 504                         //In case of 409 the missing parts will be in the response content
 505                         using (var stream = response.GetResponseStream())
 506                         using(var reader=new StreamReader(stream))
 507                         {
 508                             //We need to cleanup the content before returning it because it contains
 509                             //error content after the list of hashes
 510                             var hashes = new List<string>();
 511                             string line=null;
 512                             //All lines up to the first empty line are hashes
 513                             while(!String.IsNullOrWhiteSpace(line=reader.ReadLine()))
 514                             {
 515                                 hashes.Add(line);
 516                             }
 517
 518                             return hashes;
 519                         }
 520                     }
 521                     else
 522                         //Any other status code is unexpected and the exception should be rethrown
 523                         throw ex;
 524
 525                 }
 526                 //Any other status code is unexpected but there was no exception. We can probably continue processing
 527                 else
 528                 {
 529                     Trace.TraceWarning("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
 530                 }
 531                 return empty;
 532             });
 533
 534         }
 535
 536         public Task<byte[]> GetBlock(string container, Uri relativeUrl, long start, long? end=null)
 537         {
 538             if (String.IsNullOrWhiteSpace(Token))
 539                 throw new InvalidOperationException("Invalid Token");
 540             if (StorageUrl == null)
 541                 throw new InvalidOperationException("Invalid Storage Url");
 542             if (String.IsNullOrWhiteSpace(container))
 543                 throw new ArgumentNullException("container");
 544             if (relativeUrl== null)
 545                 throw new ArgumentNullException("relativeUrl");
 546             if (end.HasValue && end<0)
 547                 throw new ArgumentOutOfRangeException("end");
 548             if (start<0)
 549                 throw new ArgumentOutOfRangeException("start");
 550             Contract.EndContractBlock();
 551
 552             var builder = GetAddressBuilder(container, relativeUrl.ToString());
 553
 554             var uri = builder.Uri;
 555
 556             //Don't use a timeout because putting the hashmap may be a long process
 557             var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
 558             return client.DownloadDataTask(uri)
 559                 .ContinueWith(t=>
 560                                   {
 561                                       client.Dispose();
 562                                       return t.Result;
 563                                   });
 564         }
 565
 566
 567         public Task PostBlock(string container,byte[] block,int offset,int count)
 568         {
 569             if (String.IsNullOrWhiteSpace(container))
 570                 throw new ArgumentNullException("container");
 571             if (block == null)
 572                 throw new ArgumentNullException("block");
 573             if (offset < 0 || offset >= block.Length)
 574                 throw new ArgumentOutOfRangeException("offset");
 575             if (count < 0 || count > block.Length)
 576                 throw new ArgumentOutOfRangeException("count");
 577             if (String.IsNullOrWhiteSpace(Token))
 578                 throw new InvalidOperationException("Invalid Token");
 579             if (StorageUrl == null)
 580                 throw new InvalidOperationException("Invalid Storage Url");
 581             Contract.EndContractBlock();
 582
 583             var builder = GetAddressBuilder(container, "");
 584             //We are doing an update
 585             builder.Query = "update";
 586             var uri = builder.Uri;
 587
 588             //Don't use a timeout because putting the hashmap may be a long process
 589             var client = new RestClient(_baseClient) { Timeout = 0 };
 590             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
 591
 592             Trace.TraceInformation("[BLOCK POST] START");
 593
 594             client.UploadProgressChanged += (sender, args) =>
 595                 Trace.TraceInformation("[BLOCK POST PROGRESS] {0}% {1} of {2}",
 596                                     args.ProgressPercentage, args.BytesSent,
 597                                     args.TotalBytesToSend);
 598             client.UploadFileCompleted += (sender, args) =>
 599                 Trace.TraceInformation("[BLOCK POST PROGRESS] Completed ");
 600
 601
 602             //Send the block
 603             var uploadTask = client.UploadDataTask(uri, "POST", block)
 604             .ContinueWith(upload =>
 605             {
 606                 client.Dispose();
 607
 608                 if (upload.IsFaulted)
 609                 {
 610                     var exception = upload.Exception.InnerException;
 611                     Trace.TraceError("[BLOCK POST] FAIL with \r{0}", exception);
 612                     throw exception;
 613                 }
 614
 615                 Trace.TraceInformation("[BLOCK POST] END");
 616             });
 617             return uploadTask;
 618         }
 619
 620
 621         public Task<TreeHash> GetHashMap(string container, string objectName)
 622         {
 623             if (String.IsNullOrWhiteSpace(container))
 624                 throw new ArgumentNullException("container");
 625             if (String.IsNullOrWhiteSpace(objectName))
 626                 throw new ArgumentNullException("objectName");
 627             if (String.IsNullOrWhiteSpace(Token))
 628                 throw new InvalidOperationException("Invalid Token");
 629             if (StorageUrl == null)
 630                 throw new InvalidOperationException("Invalid Storage Url");
 631             Contract.EndContractBlock();
 632
 633             try
 634             {
 635                 //The container and objectName are relative names. They are joined with the client's
 636                 //BaseAddress to create the object's absolute address
 637                 var builder = GetAddressBuilder(container, objectName);
 638                 builder.Query="format=json&hashmap";
 639                 var uri = builder.Uri;
 640                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
 641                 //object to avoid concurrency errors.
 642                 //
 643                 //Download operations take a long time therefore they have no timeout.
 644                 //TODO: Do we really? this is a hashmap operation, not a download
 645                 var client = new RestClient(_baseClient) { Timeout = 0 };
 646
 647
 648                 //Start downloading the object asynchronously
 649                 var downloadTask = client.DownloadStringTask(uri);
 650
 651                 //Once the download completes
 652                 return downloadTask.ContinueWith(download =>
 653                 {
 654                     //Delete the local client object
 655                     client.Dispose();
 656                     //And report failure or completion
 657                     if (download.IsFaulted)
 658                     {
 659                         Trace.TraceError("[GET HASH] FAIL for {0} with \r{1}", objectName,
 660                                         download.Exception);
 661                         throw download.Exception;
 662                     }
 663
 664                     //The server will return an empty string if the file is empty
 665                     var json = download.Result;
 666                     var treeHash = TreeHash.Parse(json);
 667                     Trace.TraceInformation("[GET HASH] END {0}", objectName);
 668                     return treeHash;
 669                 });
 670             }
 671             catch (Exception exc)
 672             {
 673                 Trace.TraceError("[GET HASH] END {0} with {1}", objectName, exc);
 674                 throw;
 675             }
 676
 677
 678
 679         }
 680
 681         private UriBuilder GetAddressBuilder(string container, string objectName)
 682         {
 683             var builder = new UriBuilder(String.Join("/", _baseClient.BaseAddress, container, objectName));
 684             return builder;
 685         }
 686
 687
 688         /// <summary>
 689         ///
 690         /// </summary>
 691         /// <param name="container"></param>
 692         /// <param name="objectName"></param>
 693         /// <param name="fileName"></param>
 694         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
 695         /// <remarks>>This method should have no timeout or a very long one</remarks>
 696         public Task PutObject(string container, string objectName, string fileName, string hash = null)
 697         {
 698             if (String.IsNullOrWhiteSpace(container))
 699                 throw new ArgumentNullException("container", "The container property can't be empty");
 700             if (String.IsNullOrWhiteSpace(objectName))
 701                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 702             if (String.IsNullOrWhiteSpace(fileName))
 703                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
 704             if (!File.Exists(fileName))
 705                 throw new FileNotFoundException("The file does not exist",fileName);
 706
 707
 708             try
 709             {
 710                 var builder= GetAddressBuilder(container,objectName);
 711                 var uri = builder.Uri;
 712
 713                 var client = new RestClient(_baseClient){Timeout=0};
 714                 string etag = hash ?? CalculateHash(fileName);
 715
 716                 client.Headers.Add("Content-Type", "application/octet-stream");
 717                 client.Headers.Add("ETag", etag);
 718
 719
 720                 Trace.TraceInformation("[PUT] START {0}", objectName);
 721                 client.UploadProgressChanged += (sender, args) =>
 722                 {
 723                     Trace.TraceInformation("[PUT PROGRESS] {0} {1}% {2} of {3}", fileName, args.ProgressPercentage, args.BytesSent, args.TotalBytesToSend);
 724                 };
 725
 726                 client.UploadFileCompleted += (sender, args) =>
 727                 {
 728                     Trace.TraceInformation("[PUT PROGRESS] Completed {0}", fileName);
 729                 };
 730                 return client.UploadFileTask(uri, "PUT", fileName)
 731                     .ContinueWith(upload=>
 732                                       {
 733                                           client.Dispose();
 734
 735                                           if (upload.IsFaulted)
 736                                           {
 737                                               var exc = upload.Exception.InnerException;
 738                                               Trace.TraceError("[PUT] FAIL for {0} with \r{1}",objectName,exc);
 739                                               throw exc;
 740                                           }
 741                                           else
 742                                             Trace.TraceInformation("[PUT] END {0}", objectName);
 743                                       });
 744             }
 745             catch (Exception exc)
 746             {
 747                 Trace.TraceError("[PUT] END {0} with {1}", objectName, exc);
 748                 throw;
 749             }
 750
 751         }
 752
 753
 754         private static string CalculateHash(string fileName)
 755         {
 756             string hash;
 757             using (var hasher = MD5.Create())
 758             using(var stream=File.OpenRead(fileName))
 759             {
 760                 var hashBuilder=new StringBuilder();
 761                 foreach (byte b in hasher.ComputeHash(stream))
 762                     hashBuilder.Append(b.ToString("x2").ToLower());
 763                 hash = hashBuilder.ToString();
 764             }
 765             return hash;
 766         }
 767
 768         public void DeleteObject(string container, string objectName)
 769         {
 770             if (String.IsNullOrWhiteSpace(container))
 771                 throw new ArgumentNullException("container", "The container property can't be empty");
 772             if (String.IsNullOrWhiteSpace(objectName))
 773                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 774             using (var client = new RestClient(_baseClient))
 775             {
 776
 777                 client.DeleteWithRetry(container + "/" + objectName, 3);
 778
 779                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
 780                 if (!expectedCodes.Contains(client.StatusCode))
 781                     throw CreateWebException("DeleteObject", client.StatusCode);
 782             }
 783
 784         }
 785
 786         public void MoveObject(string sourceContainer, string oldObjectName, string targetContainer,string newObjectName)
 787         {
 788             if (String.IsNullOrWhiteSpace(sourceContainer))
 789                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
 790             if (String.IsNullOrWhiteSpace(oldObjectName))
 791                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
 792             if (String.IsNullOrWhiteSpace(targetContainer))
 793                 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
 794             if (String.IsNullOrWhiteSpace(newObjectName))
 795                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
 796
 797             var targetUrl = targetContainer + "/" + newObjectName;
 798             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
 799
 800             using (var client = new RestClient(_baseClient))
 801             {
 802                 client.Headers.Add("X-Move-From", sourceUrl);
 803                 client.PutWithRetry(targetUrl, 3);
 804
 805                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
 806                 if (!expectedCodes.Contains(client.StatusCode))
 807                     throw CreateWebException("MoveObject", client.StatusCode);
 808             }
 809         }
 810
 811         public void DeleteObject(string sourceContainer, string objectName, string targetContainer)
 812         {
 813             if (String.IsNullOrWhiteSpace(sourceContainer))
 814                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
 815             if (String.IsNullOrWhiteSpace(objectName))
 816                 throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
 817             if (String.IsNullOrWhiteSpace(targetContainer))
 818                 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
 819
 820             var targetUrl = targetContainer + "/" + objectName;
 821             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
 822
 823             using (var client = new RestClient(_baseClient))
 824             {
 825                 client.Headers.Add("X-Move-From", sourceUrl);
 826                 client.PutWithRetry(targetUrl, 3);
 827
 828                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
 829                 if (!expectedCodes.Contains(client.StatusCode))
 830                     throw CreateWebException("DeleteObject", client.StatusCode);
 831             }
 832         }
 833
 834
 835         private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
 836         {
 837             return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
 838         }
 839
 840
 841     }
 842 }