code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
   2 //
   3 // The class provides methods to upload/download files, delete files, manage containers
   4
   5
   6 using System;
   7 using System.Collections.Generic;
   8 using System.ComponentModel.Composition;
   9 using System.Diagnostics;
  10 using System.Diagnostics.Contracts;
  11 using System.Globalization;
  12 using System.IO;
  13 using System.Linq;
  14 using System.Net;
  15 using System.Security.Cryptography;
  16 using System.Text;
  17 using System.Threading.Algorithms;
  18 using System.Threading.Tasks;
  19 using Newtonsoft.Json;
  20 using Pithos.Interfaces;
  21 using WebHeaderCollection = System.Net.WebHeaderCollection;
  22
  23 namespace Pithos.Network
  24 {
  25     [Export(typeof(ICloudClient))]
  26     public class CloudFilesClient:ICloudClient
  27     {
  28         //CloudFilesClient uses *_baseClient* internally to communicate with the server
  29         //RestClient provides a REST-friendly interface over the standard WebClient.
  30         private RestClient _baseClient;
  31
  32         //During authentication the client provides a UserName
  33         public string UserName { get; set; }
  34
  35         //and and ApiKey to the server
  36         public string ApiKey { get; set; }
  37
  38         //And receives an authentication Token. This token must be provided in ALL other operations,
  39         //in the X-Auth-Token header
  40         public string Token { get; set; }
  41
  42         //The client also receives a StorageUrl after authentication. All subsequent operations must
  43         //use this url
  44         public Uri StorageUrl { get; set; }
  45
  46         protected Uri RootAddressUri { get; set; }
  47
  48         public Uri Proxy { get; set; }
  49
  50         public double DownloadPercentLimit { get; set; }
  51         public double UploadPercentLimit { get; set; }
  52
  53         public string AuthenticationUrl { get; set; }
  54
  55
  56         public string VersionPath
  57         {
  58             get { return UsePithos ? "v1" : "v1.0"; }
  59         }
  60
  61         public bool UsePithos { get; set; }
  62
  63         private bool _authenticated = false;
  64
  65         //
  66         public void Authenticate(string userName,string apiKey)
  67         {
  68             if (String.IsNullOrWhiteSpace(userName))
  69                 throw new ArgumentNullException("userName", "The userName property can't be empty");
  70             if (String.IsNullOrWhiteSpace(apiKey))
  71                 throw new ArgumentNullException("apiKey", "The apiKey property can't be empty");
  72             Contract.Ensures(_baseClient != null);
  73             Contract.EndContractBlock();
  74
  75             Trace.TraceInformation("[AUTHENTICATE] Start for {0}", userName);
  76
  77             if (_authenticated)
  78                 return;
  79
  80             UserName = userName;
  81             ApiKey = apiKey;
  82
  83
  84             using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
  85             {
  86                 if (Proxy != null)
  87                     authClient.Proxy = new WebProxy(Proxy);
  88
  89                 Contract.Assume(authClient.Headers!=null);
  90
  91                 authClient.Headers.Add("X-Auth-User", UserName);
  92                 authClient.Headers.Add("X-Auth-Key", ApiKey);
  93
  94                 authClient.DownloadStringWithRetry(VersionPath, 3);
  95
  96                 authClient.AssertStatusOK("Authentication failed");
  97
  98                 var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
  99                 if (String.IsNullOrWhiteSpace(storageUrl))
 100                     throw new InvalidOperationException("Failed to obtain storage url");
 101                 StorageUrl = new Uri(storageUrl);
 102
 103                 //Get the root address (StorageUrl without the account)
 104                 var usernameIndex=storageUrl.LastIndexOf(UserName);
 105                 var rootUrl = storageUrl.Substring(0, usernameIndex);
 106                 RootAddressUri = new Uri(rootUrl);
 107
 108                 var token = authClient.GetHeaderValue("X-Auth-Token");
 109                 if (String.IsNullOrWhiteSpace(token))
 110                     throw new InvalidOperationException("Failed to obtain token url");
 111                 Token = token;
 112             }
 113
 114             _baseClient = new RestClient{
 115                 BaseAddress  = StorageUrl.AbsoluteUri,
 116                 Timeout=10000,
 117                 Retries=3};
 118             if (Proxy!=null)
 119                 _baseClient.Proxy = new WebProxy(Proxy);
 120
 121
 122
 123             Contract.Assume(_baseClient.Headers!=null);
 124             _baseClient.Headers.Add("X-Auth-Token", Token);
 125
 126             Trace.TraceInformation("[AUTHENTICATE] End for {0}", userName);
 127         }
 128
 129
 130
 131         public IList<ContainerInfo> ListContainers(string account)
 132         {
 133
 134             using (var client = new RestClient(_baseClient))
 135             {
 136                 if (!String.IsNullOrWhiteSpace(account))
 137                     client.BaseAddress = GetAccountUrl(account);
 138
 139                 client.Parameters.Clear();
 140                 client.Parameters.Add("format", "json");
 141                 var content = client.DownloadStringWithRetry("", 3);
 142                 client.AssertStatusOK("List Containers failed");
 143
 144                 if (client.StatusCode == HttpStatusCode.NoContent)
 145                     return new List<ContainerInfo>();
 146                 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
 147                 return infos;
 148             }
 149
 150         }
 151
 152         private string GetAccountUrl(string account)
 153         {
 154             return new Uri(this.RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
 155         }
 156
 157         public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
 158         {
 159             Trace.TraceInformation("[START] ListSharingAccounts");
 160
 161             using (var client = new RestClient(_baseClient))
 162             {
 163                 client.Parameters.Clear();
 164                 client.Parameters.Add("format", "json");
 165                 client.IfModifiedSince = since;
 166
 167                 //Extract the username from the base address
 168                 client.BaseAddress = RootAddressUri.AbsoluteUri;
 169
 170                 var content = client.DownloadStringWithRetry(@"", 3);
 171
 172                 client.AssertStatusOK("ListSharingAccounts failed");
 173
 174                 //If the result is empty, return an empty list,
 175                 var infos = String.IsNullOrWhiteSpace(content)
 176                     ? new List<ShareAccountInfo>()
 177                     //Otherwise deserialize the account list into a list of ShareAccountInfos
 178                     : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
 179
 180                 Trace.TraceInformation("[END] ListSharingAccounts");
 181                 return infos;
 182             }
 183         }
 184
 185         //Request listing of all objects in a container modified since a specific time.
 186         //If the *since* value is missing, return all objects
 187         public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
 188         {
 189
 190             Trace.TraceInformation("[START] ListSharedObjects");
 191
 192             var objects=new List<ObjectInfo>();
 193             var accounts=ListSharingAccounts(since);
 194             foreach (var account in accounts)
 195             {
 196                 var containers=ListContainers(account.name);
 197                 foreach (var container in containers)
 198                 {
 199                     var containerObjects=ListObjects(account.name, container.Name, account.last_modified);
 200                     objects.AddRange(containerObjects);
 201                 }
 202             }
 203             return objects;
 204         }
 205
 206         public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
 207         {
 208             if (String.IsNullOrWhiteSpace(Token))
 209                 throw new InvalidOperationException("The Token is not set");
 210             if (StorageUrl==null)
 211                 throw new InvalidOperationException("The StorageUrl is not set");
 212             if (String.IsNullOrWhiteSpace(container))
 213                 throw new ArgumentNullException("container");
 214             if (String.IsNullOrWhiteSpace(objectName))
 215                 throw new ArgumentNullException("objectName");
 216             if (String.IsNullOrWhiteSpace(account))
 217                 throw new ArgumentNullException("account");
 218             if (String.IsNullOrWhiteSpace(shareTo))
 219                 throw new ArgumentNullException("shareTo");
 220             Contract.EndContractBlock();
 221
 222             using (var client = new RestClient(_baseClient))
 223             {
 224
 225                 client.BaseAddress = GetAccountUrl(account);
 226
 227                 client.Parameters.Clear();
 228                 client.Parameters.Add("format", "json");
 229
 230                 string permission = "";
 231                 if (write)
 232                     permission = String.Format("write={0}", shareTo);
 233                 else if (read)
 234                     permission=String.Format("read={0}", shareTo);
 235                 client.Headers.Add("X-Object-Sharing",permission);
 236
 237                 var content = client.DownloadStringWithRetry(container, 3);
 238
 239                 client.AssertStatusOK("ShareObject failed");
 240
 241                 //If the result is empty, return an empty list,
 242                 var infos = String.IsNullOrWhiteSpace(content)
 243                     ? new List<ObjectInfo>()
 244                     //Otherwise deserialize the object list into a list of ObjectInfos
 245                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 246
 247                 Trace.TraceInformation("[END] ListObjects");
 248             }
 249
 250
 251         }
 252
 253
 254         public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
 255         {
 256             if (String.IsNullOrWhiteSpace(container))
 257                 throw new ArgumentNullException("container");
 258             Contract.EndContractBlock();
 259
 260             Trace.TraceInformation("[START] ListObjects");
 261
 262             using (var client = new RestClient(_baseClient))
 263             {
 264                 if (!String.IsNullOrWhiteSpace(account))
 265                     client.BaseAddress = GetAccountUrl(account);
 266
 267                 client.Parameters.Clear();
 268                 client.Parameters.Add("format", "json");
 269                 client.IfModifiedSince = since;
 270                 var content = client.DownloadStringWithRetry(container, 3);
 271
 272                 client.AssertStatusOK("ListObjects failed");
 273
 274                 //If the result is empty, return an empty list,
 275                 var infos=String.IsNullOrWhiteSpace(content)
 276                     ? new List<ObjectInfo>()
 277                     //Otherwise deserialize the object list into a list of ObjectInfos
 278                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 279
 280                 foreach (var info in infos)
 281                 {
 282                     info.Container = container;
 283                     info.Account = account;
 284                 }
 285                 Trace.TraceInformation("[END] ListObjects");
 286                 return infos;
 287             }
 288         }
 289
 290
 291
 292         public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
 293         {
 294             if (String.IsNullOrWhiteSpace(container))
 295                 throw new ArgumentNullException("container");
 296             if (String.IsNullOrWhiteSpace(folder))
 297                 throw new ArgumentNullException("folder");
 298             Contract.EndContractBlock();
 299
 300             Trace.TraceInformation("[START] ListObjects");
 301
 302             using (var client = new RestClient(_baseClient))
 303             {
 304                 if (!String.IsNullOrWhiteSpace(account))
 305                     client.BaseAddress = GetAccountUrl(account);
 306
 307                 client.Parameters.Clear();
 308                 client.Parameters.Add("format", "json");
 309                 client.Parameters.Add("path", folder);
 310                 client.IfModifiedSince = since;
 311                 var content = client.DownloadStringWithRetry(container, 3);
 312                 client.AssertStatusOK("ListObjects failed");
 313
 314                 var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 315
 316                 Trace.TraceInformation("[END] ListObjects");
 317                 return infos;
 318             }
 319         }
 320
 321
 322         public bool ContainerExists(string account, string container)
 323         {
 324             if (String.IsNullOrWhiteSpace(container))
 325                 throw new ArgumentNullException("container", "The container property can't be empty");
 326             Contract.EndContractBlock();
 327
 328             using (var client = new RestClient(_baseClient))
 329             {
 330                 if (!String.IsNullOrWhiteSpace(account))
 331                     client.BaseAddress = GetAccountUrl(account);
 332
 333                 client.Parameters.Clear();
 334                 client.Head(container, 3);
 335
 336                 switch (client.StatusCode)
 337                 {
 338                     case HttpStatusCode.OK:
 339                     case HttpStatusCode.NoContent:
 340                         return true;
 341                     case HttpStatusCode.NotFound:
 342                         return false;
 343                     default:
 344                         throw CreateWebException("ContainerExists", client.StatusCode);
 345                 }
 346             }
 347         }
 348
 349         public bool ObjectExists(string account, string container, string objectName)
 350         {
 351             if (String.IsNullOrWhiteSpace(container))
 352                 throw new ArgumentNullException("container", "The container property can't be empty");
 353             if (String.IsNullOrWhiteSpace(objectName))
 354                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 355             Contract.EndContractBlock();
 356
 357             using (var client = new RestClient(_baseClient))
 358             {
 359                 if (!String.IsNullOrWhiteSpace(account))
 360                     client.BaseAddress = GetAccountUrl(account);
 361
 362                 client.Parameters.Clear();
 363                 client.Head(container + "/" + objectName, 3);
 364
 365                 switch (client.StatusCode)
 366                 {
 367                     case HttpStatusCode.OK:
 368                     case HttpStatusCode.NoContent:
 369                         return true;
 370                     case HttpStatusCode.NotFound:
 371                         return false;
 372                     default:
 373                         throw CreateWebException("ObjectExists", client.StatusCode);
 374                 }
 375             }
 376
 377         }
 378
 379         public ObjectInfo GetObjectInfo(string account, string container, string objectName)
 380         {
 381             if (String.IsNullOrWhiteSpace(container))
 382                 throw new ArgumentNullException("container", "The container property can't be empty");
 383             if (String.IsNullOrWhiteSpace(objectName))
 384                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 385             Contract.EndContractBlock();
 386
 387             using (var client = new RestClient(_baseClient))
 388             {
 389                 if (!String.IsNullOrWhiteSpace(account))
 390                     client.BaseAddress = GetAccountUrl(account);
 391                 try
 392                 {
 393                     client.Parameters.Clear();
 394
 395                     client.Head(container + "/" + objectName, 3);
 396
 397                     if (client.TimedOut)
 398                         return ObjectInfo.Empty;
 399
 400                     switch (client.StatusCode)
 401                     {
 402                         case HttpStatusCode.OK:
 403                         case HttpStatusCode.NoContent:
 404                             var keys = client.ResponseHeaders.AllKeys.AsQueryable();
 405                             var tags = (from key in keys
 406                                         where key.StartsWith("X-Object-Meta-")
 407                                         let name = key.Substring(14)
 408                                         select new {Name = name, Value = client.ResponseHeaders[name]})
 409                                 .ToDictionary(t => t.Name, t => t.Value);
 410                             var extensions = (from key in keys
 411                                               where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
 412                                               select new {Name = key, Value = client.ResponseHeaders[key]})
 413                                 .ToDictionary(t => t.Name, t => t.Value);
 414                             var info = new ObjectInfo
 415                                                  {
 416                                                      Name = objectName,
 417                                                      Hash = client.GetHeaderValue("ETag"),
 418                                                      Content_Type = client.GetHeaderValue("Content-Type"),
 419                                                      Tags = tags,
 420                                                      Last_Modified = client.LastModified,
 421                                                      Extensions = extensions
 422                                                  };
 423                             return info;
 424                         case HttpStatusCode.NotFound:
 425                             return ObjectInfo.Empty;
 426                         default:
 427                             throw new WebException(
 428                                 String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 429                                               objectName, client.StatusCode));
 430                     }
 431
 432                 }
 433                 catch(RetryException)
 434                 {
 435                     Trace.TraceWarning("[RETRY FAIL] GetObjectInfo for {0} failed.");
 436                     return ObjectInfo.Empty;
 437                 }
 438                 catch(WebException e)
 439                 {
 440                     Trace.TraceError(
 441                         String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 442                                       objectName, client.StatusCode), e);
 443                     throw;
 444                 }
 445             }
 446
 447         }
 448
 449         public void CreateFolder(string account, string container, string folder)
 450         {
 451             if (String.IsNullOrWhiteSpace(container))
 452                 throw new ArgumentNullException("container", "The container property can't be empty");
 453             if (String.IsNullOrWhiteSpace(folder))
 454                 throw new ArgumentNullException("folder", "The folder property can't be empty");
 455             Contract.EndContractBlock();
 456
 457             var folderUrl=String.Format("{0}/{1}",container,folder);
 458             using (var client = new RestClient(_baseClient))
 459             {
 460                 if (!String.IsNullOrWhiteSpace(account))
 461                     client.BaseAddress = GetAccountUrl(account);
 462
 463                 client.Parameters.Clear();
 464                 client.Headers.Add("Content-Type", @"application/directory");
 465                 client.Headers.Add("Content-Length", "0");
 466                 client.PutWithRetry(folderUrl, 3);
 467
 468                 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
 469                     throw CreateWebException("CreateFolder", client.StatusCode);
 470             }
 471         }
 472
 473         public ContainerInfo GetContainerInfo(string account, string container)
 474         {
 475             if (String.IsNullOrWhiteSpace(container))
 476                 throw new ArgumentNullException("container", "The container property can't be empty");
 477             Contract.EndContractBlock();
 478
 479             using (var client = new RestClient(_baseClient))
 480             {
 481                 if (!String.IsNullOrWhiteSpace(account))
 482                     client.BaseAddress = GetAccountUrl(account);
 483
 484                 client.Head(container);
 485                 switch (client.StatusCode)
 486                 {
 487                     case HttpStatusCode.OK:
 488                     case HttpStatusCode.NoContent:
 489                         var containerInfo = new ContainerInfo
 490                                                 {
 491                                                     Name = container,
 492                                                     Count =
 493                                                         long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
 494                                                     Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
 495                                                     BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
 496                                                     BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size"))
 497                                                 };
 498                         return containerInfo;
 499                     case HttpStatusCode.NotFound:
 500                         return ContainerInfo.Empty;
 501                     default:
 502                         throw CreateWebException("GetContainerInfo", client.StatusCode);
 503                 }
 504             }
 505         }
 506
 507         public void CreateContainer(string account, string container)
 508         {
 509             if (String.IsNullOrWhiteSpace(container))
 510                 throw new ArgumentNullException("container", "The container property can't be empty");
 511             Contract.EndContractBlock();
 512
 513             using (var client = new RestClient(_baseClient))
 514             {
 515                 if (!String.IsNullOrWhiteSpace(account))
 516                     client.BaseAddress = GetAccountUrl(account);
 517
 518                 client.PutWithRetry(container, 3);
 519                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
 520                 if (!expectedCodes.Contains(client.StatusCode))
 521                     throw CreateWebException("CreateContainer", client.StatusCode);
 522             }
 523         }
 524
 525         public void DeleteContainer(string account, string container)
 526         {
 527             if (String.IsNullOrWhiteSpace(container))
 528                 throw new ArgumentNullException("container", "The container property can't be empty");
 529             Contract.EndContractBlock();
 530
 531             using (var client = new RestClient(_baseClient))
 532             {
 533                 if (!String.IsNullOrWhiteSpace(account))
 534                     client.BaseAddress = GetAccountUrl(account);
 535
 536                 client.DeleteWithRetry(container, 3);
 537                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
 538                 if (!expectedCodes.Contains(client.StatusCode))
 539                     throw CreateWebException("DeleteContainer", client.StatusCode);
 540             }
 541
 542         }
 543
 544         /// <summary>
 545         ///
 546         /// </summary>
 547         /// <param name="account"></param>
 548         /// <param name="container"></param>
 549         /// <param name="objectName"></param>
 550         /// <param name="fileName"></param>
 551         /// <returns></returns>
 552         /// <remarks>This method should have no timeout or a very long one</remarks>
 553         //Asynchronously download the object specified by *objectName* in a specific *container* to
 554         // a local file
 555         public Task GetObject(string account, string container, string objectName, string fileName)
 556         {
 557             if (String.IsNullOrWhiteSpace(container))
 558                 throw new ArgumentNullException("container", "The container property can't be empty");
 559             if (String.IsNullOrWhiteSpace(objectName))
 560                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 561             Contract.EndContractBlock();
 562
 563             try
 564             {
 565                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
 566                 //object to avoid concurrency errors.
 567                 //
 568                 //Download operations take a long time therefore they have no timeout.
 569                 var client = new RestClient(_baseClient) { Timeout = 0 };
 570                 if (!String.IsNullOrWhiteSpace(account))
 571                     client.BaseAddress = GetAccountUrl(account);
 572
 573                 //The container and objectName are relative names. They are joined with the client's
 574                 //BaseAddress to create the object's absolute address
 575                 var builder = client.GetAddressBuilder(container, objectName);
 576                 var uri = builder.Uri;
 577
 578                 //Download progress is reported to the Trace log
 579                 Trace.TraceInformation("[GET] START {0}", objectName);
 580                 client.DownloadProgressChanged += (sender, args) =>
 581                     Trace.TraceInformation("[GET PROGRESS] {0} {1}% {2} of {3}",
 582                                     fileName, args.ProgressPercentage,
 583                                     args.BytesReceived,
 584                                     args.TotalBytesToReceive);
 585
 586
 587                 //Start downloading the object asynchronously
 588                 var downloadTask = client.DownloadFileTask(uri, fileName);
 589
 590                 //Once the download completes
 591                 return downloadTask.ContinueWith(download =>
 592                                       {
 593                                           //Delete the local client object
 594                                           client.Dispose();
 595                                           //And report failure or completion
 596                                           if (download.IsFaulted)
 597                                           {
 598                                               Trace.TraceError("[GET] FAIL for {0} with \r{1}", objectName,
 599                                                                download.Exception);
 600                                           }
 601                                           else
 602                                           {
 603                                               Trace.TraceInformation("[GET] END {0}", objectName);
 604                                           }
 605                                       });
 606             }
 607             catch (Exception exc)
 608             {
 609                 Trace.TraceError("[GET] END {0} with {1}", objectName, exc);
 610                 throw;
 611             }
 612
 613
 614
 615         }
 616
 617         public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
 618         {
 619             if (String.IsNullOrWhiteSpace(container))
 620                 throw new ArgumentNullException("container");
 621             if (String.IsNullOrWhiteSpace(objectName))
 622                 throw new ArgumentNullException("objectName");
 623             if (hash==null)
 624                 throw new ArgumentNullException("hash");
 625             if (String.IsNullOrWhiteSpace(Token))
 626                 throw new InvalidOperationException("Invalid Token");
 627             if (StorageUrl == null)
 628                 throw new InvalidOperationException("Invalid Storage Url");
 629             Contract.EndContractBlock();
 630
 631
 632             //Don't use a timeout because putting the hashmap may be a long process
 633             var client = new RestClient(_baseClient) { Timeout = 0 };
 634             if (!String.IsNullOrWhiteSpace(account))
 635                 client.BaseAddress = GetAccountUrl(account);
 636
 637             //The container and objectName are relative names. They are joined with the client's
 638             //BaseAddress to create the object's absolute address
 639             var builder = client.GetAddressBuilder(container, objectName);
 640             builder.Query = "format=json&hashmap";
 641             var uri = builder.Uri;
 642
 643
 644             //Send the tree hash as Json to the server
 645             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
 646             var uploadTask=client.UploadStringTask(uri, "PUT", hash.ToJson());
 647
 648
 649             return uploadTask.ContinueWith(t =>
 650             {
 651
 652                 var empty = (IList<string>)new List<string>();
 653
 654
 655                 //The server will respond either with 201-created if all blocks were already on the server
 656                 if (client.StatusCode == HttpStatusCode.Created)
 657                 {
 658                     //in which case we return an empty hash list
 659                     return empty;
 660                 }
 661                 //or with a 409-conflict and return the list of missing parts
 662                 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
 663                 if (t.IsFaulted)
 664                 {
 665                     var ex = t.Exception.InnerException;
 666                     var we = ex as WebException;
 667                     var response = we.Response as HttpWebResponse;
 668                     if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
 669                     {
 670                         //In case of 409 the missing parts will be in the response content
 671                         using (var stream = response.GetResponseStream())
 672                         using(var reader=new StreamReader(stream))
 673                         {
 674                             //We need to cleanup the content before returning it because it contains
 675                             //error content after the list of hashes
 676                             var hashes = new List<string>();
 677                             string line=null;
 678                             //All lines up to the first empty line are hashes
 679                             while(!String.IsNullOrWhiteSpace(line=reader.ReadLine()))
 680                             {
 681                                 hashes.Add(line);
 682                             }
 683
 684                             return hashes;
 685                         }
 686                     }
 687                     else
 688                         //Any other status code is unexpected and the exception should be rethrown
 689                         throw ex;
 690
 691                 }
 692                 //Any other status code is unexpected but there was no exception. We can probably continue processing
 693                 else
 694                 {
 695                     Trace.TraceWarning("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
 696                 }
 697                 return empty;
 698             });
 699
 700         }
 701
 702         public Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end)
 703         {
 704             if (String.IsNullOrWhiteSpace(Token))
 705                 throw new InvalidOperationException("Invalid Token");
 706             if (StorageUrl == null)
 707                 throw new InvalidOperationException("Invalid Storage Url");
 708             if (String.IsNullOrWhiteSpace(container))
 709                 throw new ArgumentNullException("container");
 710             if (relativeUrl== null)
 711                 throw new ArgumentNullException("relativeUrl");
 712             if (end.HasValue && end<0)
 713                 throw new ArgumentOutOfRangeException("end");
 714             if (start<0)
 715                 throw new ArgumentOutOfRangeException("start");
 716             Contract.EndContractBlock();
 717
 718
 719             //Don't use a timeout because putting the hashmap may be a long process
 720             var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
 721             if (!String.IsNullOrWhiteSpace(account))
 722                 client.BaseAddress = GetAccountUrl(account);
 723
 724             var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
 725             var uri = builder.Uri;
 726
 727             return client.DownloadDataTask(uri)
 728                 .ContinueWith(t=>
 729                                   {
 730                                       client.Dispose();
 731                                       return t.Result;
 732                                   });
 733         }
 734
 735
 736         public Task PostBlock(string account, string container, byte[] block, int offset, int count)
 737         {
 738             if (String.IsNullOrWhiteSpace(container))
 739                 throw new ArgumentNullException("container");
 740             if (block == null)
 741                 throw new ArgumentNullException("block");
 742             if (offset < 0 || offset >= block.Length)
 743                 throw new ArgumentOutOfRangeException("offset");
 744             if (count < 0 || count > block.Length)
 745                 throw new ArgumentOutOfRangeException("count");
 746             if (String.IsNullOrWhiteSpace(Token))
 747                 throw new InvalidOperationException("Invalid Token");
 748             if (StorageUrl == null)
 749                 throw new InvalidOperationException("Invalid Storage Url");
 750             Contract.EndContractBlock();
 751
 752
 753             //Don't use a timeout because putting the hashmap may be a long process
 754             var client = new RestClient(_baseClient) { Timeout = 0 };
 755             if (!String.IsNullOrWhiteSpace(account))
 756                 client.BaseAddress = GetAccountUrl(account);
 757
 758             var builder = client.GetAddressBuilder(container, "");
 759             //We are doing an update
 760             builder.Query = "update";
 761             var uri = builder.Uri;
 762
 763             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
 764
 765             Trace.TraceInformation("[BLOCK POST] START");
 766
 767             client.UploadProgressChanged += (sender, args) =>
 768                 Trace.TraceInformation("[BLOCK POST PROGRESS] {0}% {1} of {2}",
 769                                     args.ProgressPercentage, args.BytesSent,
 770                                     args.TotalBytesToSend);
 771             client.UploadFileCompleted += (sender, args) =>
 772                 Trace.TraceInformation("[BLOCK POST PROGRESS] Completed ");
 773
 774
 775             //Send the block
 776             var uploadTask = client.UploadDataTask(uri, "POST", block)
 777             .ContinueWith(upload =>
 778             {
 779                 client.Dispose();
 780
 781                 if (upload.IsFaulted)
 782                 {
 783                     var exception = upload.Exception.InnerException;
 784                     Trace.TraceError("[BLOCK POST] FAIL with \r{0}", exception);
 785                     throw exception;
 786                 }
 787
 788                 Trace.TraceInformation("[BLOCK POST] END");
 789             });
 790             return uploadTask;
 791         }
 792
 793
 794         public Task<TreeHash> GetHashMap(string account, string container, string objectName)
 795         {
 796             if (String.IsNullOrWhiteSpace(container))
 797                 throw new ArgumentNullException("container");
 798             if (String.IsNullOrWhiteSpace(objectName))
 799                 throw new ArgumentNullException("objectName");
 800             if (String.IsNullOrWhiteSpace(Token))
 801                 throw new InvalidOperationException("Invalid Token");
 802             if (StorageUrl == null)
 803                 throw new InvalidOperationException("Invalid Storage Url");
 804             Contract.EndContractBlock();
 805
 806             try
 807             {
 808                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
 809                 //object to avoid concurrency errors.
 810                 //
 811                 //Download operations take a long time therefore they have no timeout.
 812                 //TODO: Do we really? this is a hashmap operation, not a download
 813                 var client = new RestClient(_baseClient) { Timeout = 0 };
 814                 if (!String.IsNullOrWhiteSpace(account))
 815                     client.BaseAddress = GetAccountUrl(account);
 816
 817
 818                 //The container and objectName are relative names. They are joined with the client's
 819                 //BaseAddress to create the object's absolute address
 820                 var builder = client.GetAddressBuilder(container, objectName);
 821                 builder.Query = "format=json&hashmap";
 822                 var uri = builder.Uri;
 823
 824                 //Start downloading the object asynchronously
 825                 var downloadTask = client.DownloadStringTask(uri);
 826
 827                 //Once the download completes
 828                 return downloadTask.ContinueWith(download =>
 829                 {
 830                     //Delete the local client object
 831                     client.Dispose();
 832                     //And report failure or completion
 833                     if (download.IsFaulted)
 834                     {
 835                         Trace.TraceError("[GET HASH] FAIL for {0} with \r{1}", objectName,
 836                                         download.Exception);
 837                         throw download.Exception;
 838                     }
 839
 840                     //The server will return an empty string if the file is empty
 841                     var json = download.Result;
 842                     var treeHash = TreeHash.Parse(json);
 843                     Trace.TraceInformation("[GET HASH] END {0}", objectName);
 844                     return treeHash;
 845                 });
 846             }
 847             catch (Exception exc)
 848             {
 849                 Trace.TraceError("[GET HASH] END {0} with {1}", objectName, exc);
 850                 throw;
 851             }
 852
 853
 854
 855         }
 856
 857
 858         /// <summary>
 859         ///
 860         /// </summary>
 861         /// <param name="account"></param>
 862         /// <param name="container"></param>
 863         /// <param name="objectName"></param>
 864         /// <param name="fileName"></param>
 865         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
 866         /// <remarks>>This method should have no timeout or a very long one</remarks>
 867         public Task PutObject(string account, string container, string objectName, string fileName, string hash = null)
 868         {
 869             if (String.IsNullOrWhiteSpace(container))
 870                 throw new ArgumentNullException("container", "The container property can't be empty");
 871             if (String.IsNullOrWhiteSpace(objectName))
 872                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 873             if (String.IsNullOrWhiteSpace(fileName))
 874                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
 875             if (!File.Exists(fileName))
 876                 throw new FileNotFoundException("The file does not exist",fileName);
 877             Contract.EndContractBlock();
 878
 879             try
 880             {
 881
 882                 var client = new RestClient(_baseClient){Timeout=0};
 883                 if (!String.IsNullOrWhiteSpace(account))
 884                     client.BaseAddress = GetAccountUrl(account);
 885
 886                 var builder = client.GetAddressBuilder(container, objectName);
 887                 var uri = builder.Uri;
 888
 889                 string etag = hash ?? CalculateHash(fileName);
 890
 891                 client.Headers.Add("Content-Type", "application/octet-stream");
 892                 client.Headers.Add("ETag", etag);
 893
 894
 895                 Trace.TraceInformation("[PUT] START {0}", objectName);
 896                 client.UploadProgressChanged += (sender, args) =>
 897                 {
 898                     Trace.TraceInformation("[PUT PROGRESS] {0} {1}% {2} of {3}", fileName, args.ProgressPercentage, args.BytesSent, args.TotalBytesToSend);
 899                 };
 900
 901                 client.UploadFileCompleted += (sender, args) =>
 902                 {
 903                     Trace.TraceInformation("[PUT PROGRESS] Completed {0}", fileName);
 904                 };
 905                 return client.UploadFileTask(uri, "PUT", fileName)
 906                     .ContinueWith(upload=>
 907                                       {
 908                                           client.Dispose();
 909
 910                                           if (upload.IsFaulted)
 911                                           {
 912                                               var exc = upload.Exception.InnerException;
 913                                               Trace.TraceError("[PUT] FAIL for {0} with \r{1}",objectName,exc);
 914                                               throw exc;
 915                                           }
 916                                           else
 917                                             Trace.TraceInformation("[PUT] END {0}", objectName);
 918                                       });
 919             }
 920             catch (Exception exc)
 921             {
 922                 Trace.TraceError("[PUT] END {0} with {1}", objectName, exc);
 923                 throw;
 924             }
 925
 926         }
 927
 928
 929         private static string CalculateHash(string fileName)
 930         {
 931             string hash;
 932             using (var hasher = MD5.Create())
 933             using(var stream=File.OpenRead(fileName))
 934             {
 935                 var hashBuilder=new StringBuilder();
 936                 foreach (byte b in hasher.ComputeHash(stream))
 937                     hashBuilder.Append(b.ToString("x2").ToLower());
 938                 hash = hashBuilder.ToString();
 939             }
 940             return hash;
 941         }
 942
 943        /* public void DeleteObject(string container, string objectName,string account)
 944         {
 945             if (String.IsNullOrWhiteSpace(container))
 946                 throw new ArgumentNullException("container", "The container property can't be empty");
 947             if (String.IsNullOrWhiteSpace(objectName))
 948                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 949             Contract.EndContractBlock();
 950
 951             using (var client = new RestClient(_baseClient))
 952             {
 953                 if (!String.IsNullOrWhiteSpace(account))
 954                     client.BaseAddress = GetAccountUrl(account);
 955                 );
 956                 client.DeleteWithRetry(container + "/" + objectName, 3);
 957
 958                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
 959                 if (!expectedCodes.Contains(client.StatusCode))
 960                     throw CreateWebException("DeleteObject", client.StatusCode);
 961             }
 962
 963         }*/
 964
 965         public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
 966         {
 967             if (String.IsNullOrWhiteSpace(sourceContainer))
 968                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
 969             if (String.IsNullOrWhiteSpace(oldObjectName))
 970                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
 971             if (String.IsNullOrWhiteSpace(targetContainer))
 972                 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
 973             if (String.IsNullOrWhiteSpace(newObjectName))
 974                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
 975             Contract.EndContractBlock();
 976
 977             var targetUrl = targetContainer + "/" + newObjectName;
 978             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
 979
 980             using (var client = new RestClient(_baseClient))
 981             {
 982                 if (!String.IsNullOrWhiteSpace(account))
 983                     client.BaseAddress = GetAccountUrl(account);
 984
 985                 client.Headers.Add("X-Move-From", sourceUrl);
 986                 client.PutWithRetry(targetUrl, 3);
 987
 988                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
 989                 if (!expectedCodes.Contains(client.StatusCode))
 990                     throw CreateWebException("MoveObject", client.StatusCode);
 991             }
 992         }
 993
 994         public void DeleteObject(string account, string sourceContainer, string objectName, string targetContainer)
 995         {
 996             if (String.IsNullOrWhiteSpace(sourceContainer))
 997                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
 998             if (String.IsNullOrWhiteSpace(objectName))
 999                 throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
1000             if (String.IsNullOrWhiteSpace(targetContainer))
1001                 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
1002             Contract.EndContractBlock();
1003
1004             var targetUrl = targetContainer + "/" + objectName;
1005             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
1006
1007             using (var client = new RestClient(_baseClient))
1008             {
1009                 if (!String.IsNullOrWhiteSpace(account))
1010                     client.BaseAddress = GetAccountUrl(account);
1011
1012                 client.Headers.Add("X-Move-From", sourceUrl);
1013                 client.PutWithRetry(targetUrl, 3);
1014
1015                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
1016                 if (!expectedCodes.Contains(client.StatusCode))
1017                     throw CreateWebException("DeleteObject", client.StatusCode);
1018             }
1019         }
1020
1021
1022         private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
1023         {
1024             return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
1025         }
1026
1027
1028     }
1029
1030     public class ShareAccountInfo
1031     {
1032         public DateTime? last_modified { get; set; }
1033         public string name { get; set; }
1034     }
1035 }