code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
   2 //
   3 // The class provides methods to upload/download files, delete files, manage containers
   4
   5
   6 using System;
   7 using System.Collections.Generic;
   8 using System.ComponentModel.Composition;
   9 using System.Diagnostics.Contracts;
  10 using System.IO;
  11 using System.Linq;
  12 using System.Net;
  13 using System.Security.Cryptography;
  14 using System.Text;
  15 using System.Threading.Tasks;
  16 using Newtonsoft.Json;
  17 using Pithos.Interfaces;
  18 using log4net;
  19
  20 namespace Pithos.Network
  21 {
  22     [Export(typeof(ICloudClient))]
  23     public class CloudFilesClient:ICloudClient
  24     {
  25         //CloudFilesClient uses *_baseClient* internally to communicate with the server
  26         //RestClient provides a REST-friendly interface over the standard WebClient.
  27         private RestClient _baseClient;
  28
  29
  30         //During authentication the client provides a UserName
  31         public string UserName { get; set; }
  32
  33         //and and ApiKey to the server
  34         public string ApiKey { get; set; }
  35
  36         //And receives an authentication Token. This token must be provided in ALL other operations,
  37         //in the X-Auth-Token header
  38         private string _token;
  39         public string Token
  40         {
  41             get { return _token; }
  42             set
  43             {
  44                 _token = value;
  45                 _baseClient.Headers["X-Auth-Token"] = value;
  46             }
  47         }
  48
  49         //The client also receives a StorageUrl after authentication. All subsequent operations must
  50         //use this url
  51         public Uri StorageUrl { get; set; }
  52
  53
  54         protected Uri RootAddressUri { get; set; }
  55
  56         private Uri _proxy;
  57         public Uri Proxy
  58         {
  59             get { return _proxy; }
  60             set
  61             {
  62                 _proxy = value;
  63                 if (_baseClient != null)
  64                     _baseClient.Proxy = new WebProxy(value);
  65             }
  66         }
  67
  68         public double DownloadPercentLimit { get; set; }
  69         public double UploadPercentLimit { get; set; }
  70
  71         public string AuthenticationUrl { get; set; }
  72
  73
  74         public string VersionPath
  75         {
  76             get { return UsePithos ? "v1" : "v1.0"; }
  77         }
  78
  79         public bool UsePithos { get; set; }
  80
  81
  82         private static readonly ILog Log = LogManager.GetLogger("CloudFilesClient");
  83
  84         public CloudFilesClient(string userName, string apiKey)
  85         {
  86             UserName = userName;
  87             ApiKey = apiKey;
  88         }
  89
  90         public CloudFilesClient(AccountInfo accountInfo)
  91         {
  92             if (accountInfo==null)
  93                 throw new ArgumentNullException("accountInfo");
  94             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
  95             Contract.Ensures(StorageUrl != null);
  96             Contract.Ensures(_baseClient != null);
  97             Contract.Ensures(RootAddressUri != null);
  98             Contract.EndContractBlock();
  99
 100             _baseClient = new RestClient
 101             {
 102                 BaseAddress = accountInfo.StorageUri.ToString(),
 103                 Timeout = 10000,
 104                 Retries = 3
 105             };
 106             StorageUrl = accountInfo.StorageUri;
 107             Token = accountInfo.Token;
 108             UserName = accountInfo.UserName;
 109
 110             //Get the root address (StorageUrl without the account)
 111             var storageUrl = StorageUrl.AbsoluteUri;
 112             var usernameIndex = storageUrl.LastIndexOf(UserName);
 113             var rootUrl = storageUrl.Substring(0, usernameIndex);
 114             RootAddressUri = new Uri(rootUrl);
 115         }
 116
 117
 118         public AccountInfo Authenticate()
 119         {
 120             if (String.IsNullOrWhiteSpace(UserName))
 121                 throw new InvalidOperationException("UserName is empty");
 122             if (String.IsNullOrWhiteSpace(ApiKey))
 123                 throw new InvalidOperationException("ApiKey is empty");
 124             if (String.IsNullOrWhiteSpace(AuthenticationUrl))
 125                 throw new InvalidOperationException("AuthenticationUrl is empty");
 126             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
 127             Contract.Ensures(StorageUrl != null);
 128             Contract.Ensures(_baseClient != null);
 129             Contract.Ensures(RootAddressUri != null);
 130             Contract.EndContractBlock();
 131
 132
 133             Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
 134
 135             using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
 136             {
 137                 if (Proxy != null)
 138                     authClient.Proxy = new WebProxy(Proxy);
 139
 140                 Contract.Assume(authClient.Headers!=null);
 141
 142                 authClient.Headers.Add("X-Auth-User", UserName);
 143                 authClient.Headers.Add("X-Auth-Key", ApiKey);
 144
 145                 authClient.DownloadStringWithRetry(VersionPath, 3);
 146
 147                 authClient.AssertStatusOK("Authentication failed");
 148
 149                 var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
 150                 if (String.IsNullOrWhiteSpace(storageUrl))
 151                     throw new InvalidOperationException("Failed to obtain storage url");
 152
 153                 _baseClient = new RestClient
 154                 {
 155                     BaseAddress = storageUrl,
 156                     Timeout = 10000,
 157                     Retries = 3
 158                 };
 159
 160                 StorageUrl = new Uri(storageUrl);
 161
 162                 //Get the root address (StorageUrl without the account)
 163                 var usernameIndex=storageUrl.LastIndexOf(UserName);
 164                 var rootUrl = storageUrl.Substring(0, usernameIndex);
 165                 RootAddressUri = new Uri(rootUrl);
 166
 167                 var token = authClient.GetHeaderValue("X-Auth-Token");
 168                 if (String.IsNullOrWhiteSpace(token))
 169                     throw new InvalidOperationException("Failed to obtain token url");
 170                 Token = token;
 171
 172             }
 173
 174             Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
 175
 176
 177             return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName};
 178
 179         }
 180
 181
 182
 183         public IList<ContainerInfo> ListContainers(string account)
 184         {
 185             using (var client = new RestClient(_baseClient))
 186             {
 187                 if (!String.IsNullOrWhiteSpace(account))
 188                     client.BaseAddress = GetAccountUrl(account);
 189
 190                 client.Parameters.Clear();
 191                 client.Parameters.Add("format", "json");
 192                 var content = client.DownloadStringWithRetry("", 3);
 193                 client.AssertStatusOK("List Containers failed");
 194
 195                 if (client.StatusCode == HttpStatusCode.NoContent)
 196                     return new List<ContainerInfo>();
 197                 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
 198                 return infos;
 199             }
 200
 201         }
 202
 203         private string GetAccountUrl(string account)
 204         {
 205             return new Uri(this.RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
 206         }
 207
 208         public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
 209         {
 210             using (log4net.ThreadContext.Stacks["Share"].Push("List Accounts"))
 211             {
 212                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 213
 214                 using (var client = new RestClient(_baseClient))
 215                 {
 216                     client.Parameters.Clear();
 217                     client.Parameters.Add("format", "json");
 218                     client.IfModifiedSince = since;
 219
 220                     //Extract the username from the base address
 221                     client.BaseAddress = RootAddressUri.AbsoluteUri;
 222
 223                     var content = client.DownloadStringWithRetry(@"", 3);
 224
 225                     client.AssertStatusOK("ListSharingAccounts failed");
 226
 227                     //If the result is empty, return an empty list,
 228                     var infos = String.IsNullOrWhiteSpace(content)
 229                                     ? new List<ShareAccountInfo>()
 230                                 //Otherwise deserialize the account list into a list of ShareAccountInfos
 231                                     : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
 232
 233                     Log.DebugFormat("END");
 234                     return infos;
 235                 }
 236             }
 237         }
 238
 239         //Request listing of all objects in a container modified since a specific time.
 240         //If the *since* value is missing, return all objects
 241         public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
 242         {
 243
 244             using (log4net.ThreadContext.Stacks["Share"].Push("List Objects"))
 245             {
 246                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 247
 248                 var objects = new List<ObjectInfo>();
 249                 var accounts = ListSharingAccounts(since);
 250                 foreach (var account in accounts)
 251                 {
 252                     var containers = ListContainers(account.name);
 253                     foreach (var container in containers)
 254                     {
 255                         var containerObjects = ListObjects(account.name, container.Name, null);
 256                         objects.AddRange(containerObjects);
 257                     }
 258                 }
 259                 if (Log.IsDebugEnabled) Log.DebugFormat("END");
 260                 return objects;
 261             }
 262         }
 263
 264         public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
 265         {
 266             if (String.IsNullOrWhiteSpace(Token))
 267                 throw new InvalidOperationException("The Token is not set");
 268             if (StorageUrl==null)
 269                 throw new InvalidOperationException("The StorageUrl is not set");
 270             if (String.IsNullOrWhiteSpace(container))
 271                 throw new ArgumentNullException("container");
 272             if (String.IsNullOrWhiteSpace(objectName))
 273                 throw new ArgumentNullException("objectName");
 274             if (String.IsNullOrWhiteSpace(account))
 275                 throw new ArgumentNullException("account");
 276             if (String.IsNullOrWhiteSpace(shareTo))
 277                 throw new ArgumentNullException("shareTo");
 278             Contract.EndContractBlock();
 279
 280             using (log4net.ThreadContext.Stacks["Share"].Push("Share Object"))
 281             {
 282                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 283
 284                 using (var client = new RestClient(_baseClient))
 285                 {
 286
 287                     client.BaseAddress = GetAccountUrl(account);
 288
 289                     client.Parameters.Clear();
 290                     client.Parameters.Add("format", "json");
 291
 292                     string permission = "";
 293                     if (write)
 294                         permission = String.Format("write={0}", shareTo);
 295                     else if (read)
 296                         permission = String.Format("read={0}", shareTo);
 297                     client.Headers.Add("X-Object-Sharing", permission);
 298
 299                     var content = client.DownloadStringWithRetry(container, 3);
 300
 301                     client.AssertStatusOK("ShareObject failed");
 302
 303                     //If the result is empty, return an empty list,
 304                     var infos = String.IsNullOrWhiteSpace(content)
 305                                     ? new List<ObjectInfo>()
 306                                 //Otherwise deserialize the object list into a list of ObjectInfos
 307                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 308
 309                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 310                 }
 311             }
 312
 313
 314         }
 315
 316         public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
 317         {
 318             if (accountInfo==null)
 319                 throw new ArgumentNullException("accountInfo");
 320             Contract.EndContractBlock();
 321
 322             using (log4net.ThreadContext.Stacks["Account"].Push("GetPolicies"))
 323             {
 324                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 325
 326                 using (var client = new RestClient(_baseClient))
 327                 {
 328                     if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
 329                         client.BaseAddress = GetAccountUrl(accountInfo.UserName);
 330
 331                     client.Parameters.Clear();
 332                     client.Parameters.Add("format", "json");
 333                     client.Head(String.Empty, 3);
 334
 335                     var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
 336                     var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
 337
 338                     long quota, bytes;
 339                     if (long.TryParse(quotaValue, out quota))
 340                         accountInfo.Quota = quota;
 341                     if (long.TryParse(bytesValue, out bytes))
 342                         accountInfo.BytesUsed = bytes;
 343
 344                     return accountInfo;
 345
 346                 }
 347
 348             }
 349         }
 350
 351
 352         public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
 353         {
 354             if (String.IsNullOrWhiteSpace(container))
 355                 throw new ArgumentNullException("container");
 356             Contract.EndContractBlock();
 357
 358             using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
 359             {
 360                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 361
 362                 using (var client = new RestClient(_baseClient))
 363                 {
 364                     if (!String.IsNullOrWhiteSpace(account))
 365                         client.BaseAddress = GetAccountUrl(account);
 366
 367                     client.Parameters.Clear();
 368                     client.Parameters.Add("format", "json");
 369                     client.IfModifiedSince = since;
 370                     var content = client.DownloadStringWithRetry(container, 3);
 371
 372                     client.AssertStatusOK("ListObjects failed");
 373
 374                     //If the result is empty, return an empty list,
 375                     var infos = String.IsNullOrWhiteSpace(content)
 376                                     ? new List<ObjectInfo>()
 377                                 //Otherwise deserialize the object list into a list of ObjectInfos
 378                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 379
 380                     foreach (var info in infos)
 381                     {
 382                         info.Container = container;
 383                         info.Account = account;
 384                     }
 385                     if (Log.IsDebugEnabled) Log.DebugFormat("START");
 386                     return infos;
 387                 }
 388             }
 389         }
 390
 391
 392
 393         public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
 394         {
 395             if (String.IsNullOrWhiteSpace(container))
 396                 throw new ArgumentNullException("container");
 397             if (String.IsNullOrWhiteSpace(folder))
 398                 throw new ArgumentNullException("folder");
 399             Contract.EndContractBlock();
 400
 401             using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
 402             {
 403                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 404
 405                 using (var client = new RestClient(_baseClient))
 406                 {
 407                     if (!String.IsNullOrWhiteSpace(account))
 408                         client.BaseAddress = GetAccountUrl(account);
 409
 410                     client.Parameters.Clear();
 411                     client.Parameters.Add("format", "json");
 412                     client.Parameters.Add("path", folder);
 413                     client.IfModifiedSince = since;
 414                     var content = client.DownloadStringWithRetry(container, 3);
 415                     client.AssertStatusOK("ListObjects failed");
 416
 417                     var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 418
 419                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 420                     return infos;
 421                 }
 422             }
 423         }
 424
 425
 426         public bool ContainerExists(string account, string container)
 427         {
 428             if (String.IsNullOrWhiteSpace(container))
 429                 throw new ArgumentNullException("container", "The container property can't be empty");
 430             Contract.EndContractBlock();
 431
 432             using (log4net.ThreadContext.Stacks["Containters"].Push("Exists"))
 433             {
 434                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 435
 436                 using (var client = new RestClient(_baseClient))
 437                 {
 438                     if (!String.IsNullOrWhiteSpace(account))
 439                         client.BaseAddress = GetAccountUrl(account);
 440
 441                     client.Parameters.Clear();
 442                     client.Head(container, 3);
 443
 444                     bool result;
 445                     switch (client.StatusCode)
 446                     {
 447                         case HttpStatusCode.OK:
 448                         case HttpStatusCode.NoContent:
 449                             result=true;
 450                             break;
 451                         case HttpStatusCode.NotFound:
 452                             result=false;
 453                             break;
 454                         default:
 455                             throw CreateWebException("ContainerExists", client.StatusCode);
 456                     }
 457                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 458
 459                     return result;
 460                 }
 461
 462             }
 463         }
 464
 465         public bool ObjectExists(string account, string container, string objectName)
 466         {
 467             if (String.IsNullOrWhiteSpace(container))
 468                 throw new ArgumentNullException("container", "The container property can't be empty");
 469             if (String.IsNullOrWhiteSpace(objectName))
 470                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 471             Contract.EndContractBlock();
 472
 473             using (var client = new RestClient(_baseClient))
 474             {
 475                 if (!String.IsNullOrWhiteSpace(account))
 476                     client.BaseAddress = GetAccountUrl(account);
 477
 478                 client.Parameters.Clear();
 479                 client.Head(container + "/" + objectName, 3);
 480
 481                 switch (client.StatusCode)
 482                 {
 483                     case HttpStatusCode.OK:
 484                     case HttpStatusCode.NoContent:
 485                         return true;
 486                     case HttpStatusCode.NotFound:
 487                         return false;
 488                     default:
 489                         throw CreateWebException("ObjectExists", client.StatusCode);
 490                 }
 491             }
 492
 493         }
 494
 495         public ObjectInfo GetObjectInfo(string account, string container, string objectName)
 496         {
 497             if (String.IsNullOrWhiteSpace(container))
 498                 throw new ArgumentNullException("container", "The container property can't be empty");
 499             if (String.IsNullOrWhiteSpace(objectName))
 500                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 501             Contract.EndContractBlock();
 502
 503             using (log4net.ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
 504             {
 505
 506                 using (var client = new RestClient(_baseClient))
 507                 {
 508                     if (!String.IsNullOrWhiteSpace(account))
 509                         client.BaseAddress = GetAccountUrl(account);
 510                     try
 511                     {
 512                         client.Parameters.Clear();
 513
 514                         client.Head(container + "/" + objectName, 3);
 515
 516                         if (client.TimedOut)
 517                             return ObjectInfo.Empty;
 518
 519                         switch (client.StatusCode)
 520                         {
 521                             case HttpStatusCode.OK:
 522                             case HttpStatusCode.NoContent:
 523                                 var keys = client.ResponseHeaders.AllKeys.AsQueryable();
 524                                 var tags = (from key in keys
 525                                             where key.StartsWith("X-Object-Meta-")
 526                                             let name = key.Substring(14)
 527                                             select new {Name = name, Value = client.ResponseHeaders[name]})
 528                                     .ToDictionary(t => t.Name, t => t.Value);
 529                                 var extensions = (from key in keys
 530                                                   where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
 531                                                   select new {Name = key, Value = client.ResponseHeaders[key]})
 532                                     .ToDictionary(t => t.Name, t => t.Value);
 533                                 var info = new ObjectInfo
 534                                                {
 535                                                    Name = objectName,
 536                                                    Hash = client.GetHeaderValue("ETag"),
 537                                                    Content_Type = client.GetHeaderValue("Content-Type"),
 538                                                    Tags = tags,
 539                                                    Last_Modified = client.LastModified,
 540                                                    Extensions = extensions
 541                                                };
 542                                 return info;
 543                             case HttpStatusCode.NotFound:
 544                                 return ObjectInfo.Empty;
 545                             default:
 546                                 throw new WebException(
 547                                     String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 548                                                   objectName, client.StatusCode));
 549                         }
 550
 551                     }
 552                     catch (RetryException)
 553                     {
 554                         Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.");
 555                         return ObjectInfo.Empty;
 556                     }
 557                     catch (WebException e)
 558                     {
 559                         Log.Error(
 560                             String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 561                                           objectName, client.StatusCode), e);
 562                         throw;
 563                     }
 564                 }
 565             }
 566
 567         }
 568
 569         public void CreateFolder(string account, string container, string folder)
 570         {
 571             if (String.IsNullOrWhiteSpace(container))
 572                 throw new ArgumentNullException("container", "The container property can't be empty");
 573             if (String.IsNullOrWhiteSpace(folder))
 574                 throw new ArgumentNullException("folder", "The folder property can't be empty");
 575             Contract.EndContractBlock();
 576
 577             var folderUrl=String.Format("{0}/{1}",container,folder);
 578             using (var client = new RestClient(_baseClient))
 579             {
 580                 if (!String.IsNullOrWhiteSpace(account))
 581                     client.BaseAddress = GetAccountUrl(account);
 582
 583                 client.Parameters.Clear();
 584                 client.Headers.Add("Content-Type", @"application/directory");
 585                 client.Headers.Add("Content-Length", "0");
 586                 client.PutWithRetry(folderUrl, 3);
 587
 588                 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
 589                     throw CreateWebException("CreateFolder", client.StatusCode);
 590             }
 591         }
 592
 593         public ContainerInfo GetContainerInfo(string account, string container)
 594         {
 595             if (String.IsNullOrWhiteSpace(container))
 596                 throw new ArgumentNullException("container", "The container property can't be empty");
 597             Contract.EndContractBlock();
 598
 599             using (var client = new RestClient(_baseClient))
 600             {
 601                 if (!String.IsNullOrWhiteSpace(account))
 602                     client.BaseAddress = GetAccountUrl(account);
 603
 604                 client.Head(container);
 605                 switch (client.StatusCode)
 606                 {
 607                     case HttpStatusCode.OK:
 608                     case HttpStatusCode.NoContent:
 609                         var containerInfo = new ContainerInfo
 610                                                 {
 611                                                     Name = container,
 612                                                     Count =
 613                                                         long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
 614                                                     Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
 615                                                     BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
 616                                                     BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size"))
 617                                                 };
 618                         return containerInfo;
 619                     case HttpStatusCode.NotFound:
 620                         return ContainerInfo.Empty;
 621                     default:
 622                         throw CreateWebException("GetContainerInfo", client.StatusCode);
 623                 }
 624             }
 625         }
 626
 627         public void CreateContainer(string account, string container)
 628         {
 629             if (String.IsNullOrWhiteSpace(container))
 630                 throw new ArgumentNullException("container", "The container property can't be empty");
 631             Contract.EndContractBlock();
 632
 633             using (var client = new RestClient(_baseClient))
 634             {
 635                 if (!String.IsNullOrWhiteSpace(account))
 636                     client.BaseAddress = GetAccountUrl(account);
 637
 638                 client.PutWithRetry(container, 3);
 639                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
 640                 if (!expectedCodes.Contains(client.StatusCode))
 641                     throw CreateWebException("CreateContainer", client.StatusCode);
 642             }
 643         }
 644
 645         public void DeleteContainer(string account, string container)
 646         {
 647             if (String.IsNullOrWhiteSpace(container))
 648                 throw new ArgumentNullException("container", "The container property can't be empty");
 649             Contract.EndContractBlock();
 650
 651             using (var client = new RestClient(_baseClient))
 652             {
 653                 if (!String.IsNullOrWhiteSpace(account))
 654                     client.BaseAddress = GetAccountUrl(account);
 655
 656                 client.DeleteWithRetry(container, 3);
 657                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
 658                 if (!expectedCodes.Contains(client.StatusCode))
 659                     throw CreateWebException("DeleteContainer", client.StatusCode);
 660             }
 661
 662         }
 663
 664         /// <summary>
 665         ///
 666         /// </summary>
 667         /// <param name="account"></param>
 668         /// <param name="container"></param>
 669         /// <param name="objectName"></param>
 670         /// <param name="fileName"></param>
 671         /// <returns></returns>
 672         /// <remarks>This method should have no timeout or a very long one</remarks>
 673         //Asynchronously download the object specified by *objectName* in a specific *container* to
 674         // a local file
 675         public Task GetObject(string account, string container, string objectName, string fileName)
 676         {
 677             if (String.IsNullOrWhiteSpace(container))
 678                 throw new ArgumentNullException("container", "The container property can't be empty");
 679             if (String.IsNullOrWhiteSpace(objectName))
 680                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 681             Contract.EndContractBlock();
 682
 683             try
 684             {
 685                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
 686                 //object to avoid concurrency errors.
 687                 //
 688                 //Download operations take a long time therefore they have no timeout.
 689                 var client = new RestClient(_baseClient) { Timeout = 0 };
 690                 if (!String.IsNullOrWhiteSpace(account))
 691                     client.BaseAddress = GetAccountUrl(account);
 692
 693                 //The container and objectName are relative names. They are joined with the client's
 694                 //BaseAddress to create the object's absolute address
 695                 var builder = client.GetAddressBuilder(container, objectName);
 696                 var uri = builder.Uri;
 697
 698                 //Download progress is reported to the Trace log
 699                 Log.InfoFormat("[GET] START {0}", objectName);
 700                 client.DownloadProgressChanged += (sender, args) =>
 701                     Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
 702                                     fileName, args.ProgressPercentage,
 703                                     args.BytesReceived,
 704                                     args.TotalBytesToReceive);
 705
 706
 707                 //Start downloading the object asynchronously
 708                 var downloadTask = client.DownloadFileTask(uri, fileName);
 709
 710                 //Once the download completes
 711                 return downloadTask.ContinueWith(download =>
 712                                       {
 713                                           //Delete the local client object
 714                                           client.Dispose();
 715                                           //And report failure or completion
 716                                           if (download.IsFaulted)
 717                                           {
 718                                               Log.ErrorFormat("[GET] FAIL for {0} with \r{1}", objectName,
 719                                                                download.Exception);
 720                                           }
 721                                           else
 722                                           {
 723                                               Log.InfoFormat("[GET] END {0}", objectName);
 724                                           }
 725                                       });
 726             }
 727             catch (Exception exc)
 728             {
 729                 Log.ErrorFormat("[GET] END {0} with {1}", objectName, exc);
 730                 throw;
 731             }
 732
 733
 734
 735         }
 736
 737         public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
 738         {
 739             if (String.IsNullOrWhiteSpace(container))
 740                 throw new ArgumentNullException("container");
 741             if (String.IsNullOrWhiteSpace(objectName))
 742                 throw new ArgumentNullException("objectName");
 743             if (hash==null)
 744                 throw new ArgumentNullException("hash");
 745             if (String.IsNullOrWhiteSpace(Token))
 746                 throw new InvalidOperationException("Invalid Token");
 747             if (StorageUrl == null)
 748                 throw new InvalidOperationException("Invalid Storage Url");
 749             Contract.EndContractBlock();
 750
 751
 752             //Don't use a timeout because putting the hashmap may be a long process
 753             var client = new RestClient(_baseClient) { Timeout = 0 };
 754             if (!String.IsNullOrWhiteSpace(account))
 755                 client.BaseAddress = GetAccountUrl(account);
 756
 757             //The container and objectName are relative names. They are joined with the client's
 758             //BaseAddress to create the object's absolute address
 759             var builder = client.GetAddressBuilder(container, objectName);
 760             builder.Query = "format=json&hashmap";
 761             var uri = builder.Uri;
 762
 763
 764             //Send the tree hash as Json to the server
 765             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
 766             var uploadTask=client.UploadStringTask(uri, "PUT", hash.ToJson());
 767
 768
 769             return uploadTask.ContinueWith(t =>
 770             {
 771
 772                 var empty = (IList<string>)new List<string>();
 773
 774
 775                 //The server will respond either with 201-created if all blocks were already on the server
 776                 if (client.StatusCode == HttpStatusCode.Created)
 777                 {
 778                     //in which case we return an empty hash list
 779                     return empty;
 780                 }
 781                 //or with a 409-conflict and return the list of missing parts
 782                 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
 783                 if (t.IsFaulted)
 784                 {
 785                     var ex = t.Exception.InnerException;
 786                     var we = ex as WebException;
 787                     var response = we.Response as HttpWebResponse;
 788                     if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
 789                     {
 790                         //In case of 409 the missing parts will be in the response content
 791                         using (var stream = response.GetResponseStream())
 792                         using(var reader=new StreamReader(stream))
 793                         {
 794                             //We need to cleanup the content before returning it because it contains
 795                             //error content after the list of hashes
 796                             var hashes = new List<string>();
 797                             string line=null;
 798                             //All lines up to the first empty line are hashes
 799                             while(!String.IsNullOrWhiteSpace(line=reader.ReadLine()))
 800                             {
 801                                 hashes.Add(line);
 802                             }
 803
 804                             return hashes;
 805                         }
 806                     }
 807                     else
 808                         //Any other status code is unexpected and the exception should be rethrown
 809                         throw ex;
 810
 811                 }
 812                 //Any other status code is unexpected but there was no exception. We can probably continue processing
 813                 else
 814                 {
 815                     Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
 816                 }
 817                 return empty;
 818             });
 819
 820         }
 821
 822         public Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end)
 823         {
 824             if (String.IsNullOrWhiteSpace(Token))
 825                 throw new InvalidOperationException("Invalid Token");
 826             if (StorageUrl == null)
 827                 throw new InvalidOperationException("Invalid Storage Url");
 828             if (String.IsNullOrWhiteSpace(container))
 829                 throw new ArgumentNullException("container");
 830             if (relativeUrl== null)
 831                 throw new ArgumentNullException("relativeUrl");
 832             if (end.HasValue && end<0)
 833                 throw new ArgumentOutOfRangeException("end");
 834             if (start<0)
 835                 throw new ArgumentOutOfRangeException("start");
 836             Contract.EndContractBlock();
 837
 838
 839             //Don't use a timeout because putting the hashmap may be a long process
 840             var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
 841             if (!String.IsNullOrWhiteSpace(account))
 842                 client.BaseAddress = GetAccountUrl(account);
 843
 844             var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
 845             var uri = builder.Uri;
 846
 847             return client.DownloadDataTask(uri)
 848                 .ContinueWith(t=>
 849                                   {
 850                                       client.Dispose();
 851                                       return t.Result;
 852                                   });
 853         }
 854
 855
 856         public Task PostBlock(string account, string container, byte[] block, int offset, int count)
 857         {
 858             if (String.IsNullOrWhiteSpace(container))
 859                 throw new ArgumentNullException("container");
 860             if (block == null)
 861                 throw new ArgumentNullException("block");
 862             if (offset < 0 || offset >= block.Length)
 863                 throw new ArgumentOutOfRangeException("offset");
 864             if (count < 0 || count > block.Length)
 865                 throw new ArgumentOutOfRangeException("count");
 866             if (String.IsNullOrWhiteSpace(Token))
 867                 throw new InvalidOperationException("Invalid Token");
 868             if (StorageUrl == null)
 869                 throw new InvalidOperationException("Invalid Storage Url");
 870             Contract.EndContractBlock();
 871
 872
 873             //Don't use a timeout because putting the hashmap may be a long process
 874             var client = new RestClient(_baseClient) { Timeout = 0 };
 875             if (!String.IsNullOrWhiteSpace(account))
 876                 client.BaseAddress = GetAccountUrl(account);
 877
 878             var builder = client.GetAddressBuilder(container, "");
 879             //We are doing an update
 880             builder.Query = "update";
 881             var uri = builder.Uri;
 882
 883             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
 884
 885             Log.InfoFormat("[BLOCK POST] START");
 886
 887             client.UploadProgressChanged += (sender, args) =>
 888                 Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
 889                                     args.ProgressPercentage, args.BytesSent,
 890                                     args.TotalBytesToSend);
 891             client.UploadFileCompleted += (sender, args) =>
 892                 Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
 893
 894
 895             //Send the block
 896             var uploadTask = client.UploadDataTask(uri, "POST", block)
 897             .ContinueWith(upload =>
 898             {
 899                 client.Dispose();
 900
 901                 if (upload.IsFaulted)
 902                 {
 903                     var exception = upload.Exception.InnerException;
 904                     Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exception);
 905                     throw exception;
 906                 }
 907
 908                 Log.InfoFormat("[BLOCK POST] END");
 909             });
 910             return uploadTask;
 911         }
 912
 913
 914         public Task<TreeHash> GetHashMap(string account, string container, string objectName)
 915         {
 916             if (String.IsNullOrWhiteSpace(container))
 917                 throw new ArgumentNullException("container");
 918             if (String.IsNullOrWhiteSpace(objectName))
 919                 throw new ArgumentNullException("objectName");
 920             if (String.IsNullOrWhiteSpace(Token))
 921                 throw new InvalidOperationException("Invalid Token");
 922             if (StorageUrl == null)
 923                 throw new InvalidOperationException("Invalid Storage Url");
 924             Contract.EndContractBlock();
 925
 926             try
 927             {
 928                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
 929                 //object to avoid concurrency errors.
 930                 //
 931                 //Download operations take a long time therefore they have no timeout.
 932                 //TODO: Do we really? this is a hashmap operation, not a download
 933                 var client = new RestClient(_baseClient) { Timeout = 0 };
 934                 if (!String.IsNullOrWhiteSpace(account))
 935                     client.BaseAddress = GetAccountUrl(account);
 936
 937
 938                 //The container and objectName are relative names. They are joined with the client's
 939                 //BaseAddress to create the object's absolute address
 940                 var builder = client.GetAddressBuilder(container, objectName);
 941                 builder.Query = "format=json&hashmap";
 942                 var uri = builder.Uri;
 943
 944                 //Start downloading the object asynchronously
 945                 var downloadTask = client.DownloadStringTask(uri);
 946
 947                 //Once the download completes
 948                 return downloadTask.ContinueWith(download =>
 949                 {
 950                     //Delete the local client object
 951                     client.Dispose();
 952                     //And report failure or completion
 953                     if (download.IsFaulted)
 954                     {
 955                         Log.ErrorFormat("[GET HASH] FAIL for {0} with \r{1}", objectName,
 956                                         download.Exception);
 957                         throw download.Exception;
 958                     }
 959
 960                     //The server will return an empty string if the file is empty
 961                     var json = download.Result;
 962                     var treeHash = TreeHash.Parse(json);
 963                     Log.InfoFormat("[GET HASH] END {0}", objectName);
 964                     return treeHash;
 965                 });
 966             }
 967             catch (Exception exc)
 968             {
 969                 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
 970                 throw;
 971             }
 972
 973
 974
 975         }
 976
 977
 978         /// <summary>
 979         ///
 980         /// </summary>
 981         /// <param name="account"></param>
 982         /// <param name="container"></param>
 983         /// <param name="objectName"></param>
 984         /// <param name="fileName"></param>
 985         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
 986         /// <remarks>>This method should have no timeout or a very long one</remarks>
 987         public Task PutObject(string account, string container, string objectName, string fileName, string hash = null)
 988         {
 989             if (String.IsNullOrWhiteSpace(container))
 990                 throw new ArgumentNullException("container", "The container property can't be empty");
 991             if (String.IsNullOrWhiteSpace(objectName))
 992                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 993             if (String.IsNullOrWhiteSpace(fileName))
 994                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
 995             if (!File.Exists(fileName))
 996                 throw new FileNotFoundException("The file does not exist",fileName);
 997             Contract.EndContractBlock();
 998
 999             try
1000             {
1001
1002                 var client = new RestClient(_baseClient){Timeout=0};
1003                 if (!String.IsNullOrWhiteSpace(account))
1004                     client.BaseAddress = GetAccountUrl(account);
1005
1006                 var builder = client.GetAddressBuilder(container, objectName);
1007                 var uri = builder.Uri;
1008
1009                 string etag = hash ?? CalculateHash(fileName);
1010
1011                 client.Headers.Add("Content-Type", "application/octet-stream");
1012                 client.Headers.Add("ETag", etag);
1013
1014
1015                 Log.InfoFormat("[PUT] START {0}", objectName);
1016                 client.UploadProgressChanged += (sender, args) =>
1017                 {
1018                     using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
1019                     {
1020                         Log.InfoFormat("{0} {1}% {2} of {3}", fileName, args.ProgressPercentage,
1021                                        args.BytesSent, args.TotalBytesToSend);
1022                     }
1023                 };
1024
1025                 client.UploadFileCompleted += (sender, args) =>
1026                 {
1027                     using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
1028                     {
1029                         Log.InfoFormat("Completed {0}", fileName);
1030                     }
1031                 };
1032                 return client.UploadFileTask(uri, "PUT", fileName)
1033                     .ContinueWith(upload=>
1034                                       {
1035                                           client.Dispose();
1036
1037                                           if (upload.IsFaulted)
1038                                           {
1039                                               var exc = upload.Exception.InnerException;
1040                                               Log.ErrorFormat("[PUT] FAIL for {0} with \r{1}",objectName,exc);
1041                                               throw exc;
1042                                           }
1043                                           else
1044                                             Log.InfoFormat("[PUT] END {0}", objectName);
1045                                       });
1046             }
1047             catch (Exception exc)
1048             {
1049                 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
1050                 throw;
1051             }
1052
1053         }
1054
1055
1056         private static string CalculateHash(string fileName)
1057         {
1058             Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
1059             Contract.EndContractBlock();
1060
1061             string hash;
1062             using (var hasher = MD5.Create())
1063             using(var stream=File.OpenRead(fileName))
1064             {
1065                 var hashBuilder=new StringBuilder();
1066                 foreach (byte b in hasher.ComputeHash(stream))
1067                     hashBuilder.Append(b.ToString("x2").ToLower());
1068                 hash = hashBuilder.ToString();
1069             }
1070             return hash;
1071         }
1072
1073         public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
1074         {
1075             if (String.IsNullOrWhiteSpace(sourceContainer))
1076                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1077             if (String.IsNullOrWhiteSpace(oldObjectName))
1078                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
1079             if (String.IsNullOrWhiteSpace(targetContainer))
1080                 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
1081             if (String.IsNullOrWhiteSpace(newObjectName))
1082                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
1083             Contract.EndContractBlock();
1084
1085             var targetUrl = targetContainer + "/" + newObjectName;
1086             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
1087
1088             using (var client = new RestClient(_baseClient))
1089             {
1090                 if (!String.IsNullOrWhiteSpace(account))
1091                     client.BaseAddress = GetAccountUrl(account);
1092
1093                 client.Headers.Add("X-Move-From", sourceUrl);
1094                 client.PutWithRetry(targetUrl, 3);
1095
1096                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1097                 if (!expectedCodes.Contains(client.StatusCode))
1098                     throw CreateWebException("MoveObject", client.StatusCode);
1099             }
1100         }
1101
1102         public void DeleteObject(string account, string sourceContainer, string objectName)
1103         {
1104             if (String.IsNullOrWhiteSpace(sourceContainer))
1105                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1106             if (String.IsNullOrWhiteSpace(objectName))
1107                 throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
1108             Contract.EndContractBlock();
1109
1110             var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
1111             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
1112
1113             using (var client = new RestClient(_baseClient))
1114             {
1115                 if (!String.IsNullOrWhiteSpace(account))
1116                     client.BaseAddress = GetAccountUrl(account);
1117
1118                 client.Headers.Add("X-Move-From", sourceUrl);
1119                 client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
1120                 client.PutWithRetry(targetUrl, 3);
1121
1122                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
1123                 if (!expectedCodes.Contains(client.StatusCode))
1124                     throw CreateWebException("DeleteObject", client.StatusCode);
1125             }
1126         }
1127
1128
1129         private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
1130         {
1131             return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
1132         }
1133
1134
1135     }
1136
1137     public class ShareAccountInfo
1138     {
1139         public DateTime? last_modified { get; set; }
1140         public string name { get; set; }
1141     }
1142 }