code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
   2 //
   3 // The class provides methods to upload/download files, delete files, manage containers
   4
   5
   6 using System;
   7 using System.Collections.Generic;
   8 using System.ComponentModel.Composition;
   9 using System.Diagnostics;
  10 using System.Diagnostics.Contracts;
  11 using System.Globalization;
  12 using System.IO;
  13 using System.Linq;
  14 using System.Net;
  15 using System.Security.Cryptography;
  16 using System.Text;
  17 using System.Threading.Algorithms;
  18 using System.Threading.Tasks;
  19 using Newtonsoft.Json;
  20 using Pithos.Interfaces;
  21 using log4net;
  22 using WebHeaderCollection = System.Net.WebHeaderCollection;
  23
  24 namespace Pithos.Network
  25 {
  26     [Export(typeof(ICloudClient))]
  27     public class CloudFilesClient:ICloudClient
  28     {
  29         //CloudFilesClient uses *_baseClient* internally to communicate with the server
  30         //RestClient provides a REST-friendly interface over the standard WebClient.
  31         private RestClient _baseClient;
  32
  33         //During authentication the client provides a UserName
  34         public string UserName { get; set; }
  35
  36         //and and ApiKey to the server
  37         public string ApiKey { get; set; }
  38
  39         //And receives an authentication Token. This token must be provided in ALL other operations,
  40         //in the X-Auth-Token header
  41         public string Token { get; set; }
  42
  43         //The client also receives a StorageUrl after authentication. All subsequent operations must
  44         //use this url
  45         public Uri StorageUrl { get; set; }
  46
  47         protected Uri RootAddressUri { get; set; }
  48
  49         public Uri Proxy { get; set; }
  50
  51         public double DownloadPercentLimit { get; set; }
  52         public double UploadPercentLimit { get; set; }
  53
  54         public string AuthenticationUrl { get; set; }
  55
  56
  57         public string VersionPath
  58         {
  59             get { return UsePithos ? "v1" : "v1.0"; }
  60         }
  61
  62         public bool UsePithos { get; set; }
  63
  64         private bool _authenticated = false;
  65
  66         private static readonly ILog Log = LogManager.GetLogger("CloudFilesClient");
  67
  68
  69         public void Authenticate(string userName,string apiKey)
  70         {
  71             if (String.IsNullOrWhiteSpace(userName))
  72                 throw new ArgumentNullException("userName", "The userName property can't be empty");
  73             if (String.IsNullOrWhiteSpace(apiKey))
  74                 throw new ArgumentNullException("apiKey", "The apiKey property can't be empty");
  75             Contract.Ensures(_baseClient != null);
  76             Contract.EndContractBlock();
  77
  78             Log.InfoFormat("[AUTHENTICATE] Start for {0}", userName);
  79
  80             if (_authenticated)
  81                 return;
  82
  83             UserName = userName;
  84             ApiKey = apiKey;
  85
  86
  87             using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
  88             {
  89                 if (Proxy != null)
  90                     authClient.Proxy = new WebProxy(Proxy);
  91
  92                 Contract.Assume(authClient.Headers!=null);
  93
  94                 authClient.Headers.Add("X-Auth-User", UserName);
  95                 authClient.Headers.Add("X-Auth-Key", ApiKey);
  96
  97                 authClient.DownloadStringWithRetry(VersionPath, 3);
  98
  99                 authClient.AssertStatusOK("Authentication failed");
 100
 101                 var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
 102                 if (String.IsNullOrWhiteSpace(storageUrl))
 103                     throw new InvalidOperationException("Failed to obtain storage url");
 104                 StorageUrl = new Uri(storageUrl);
 105
 106                 //Get the root address (StorageUrl without the account)
 107                 var usernameIndex=storageUrl.LastIndexOf(UserName);
 108                 var rootUrl = storageUrl.Substring(0, usernameIndex);
 109                 RootAddressUri = new Uri(rootUrl);
 110
 111                 var token = authClient.GetHeaderValue("X-Auth-Token");
 112                 if (String.IsNullOrWhiteSpace(token))
 113                     throw new InvalidOperationException("Failed to obtain token url");
 114                 Token = token;
 115             }
 116
 117             _baseClient = new RestClient{
 118                 BaseAddress  = StorageUrl.AbsoluteUri,
 119                 Timeout=10000,
 120                 Retries=3};
 121             if (Proxy!=null)
 122                 _baseClient.Proxy = new WebProxy(Proxy);
 123
 124
 125
 126             Contract.Assume(_baseClient.Headers!=null);
 127             _baseClient.Headers.Add("X-Auth-Token", Token);
 128
 129             Log.InfoFormat("[AUTHENTICATE] End for {0}", userName);
 130         }
 131
 132
 133
 134         public IList<ContainerInfo> ListContainers(string account)
 135         {
 136
 137             using (var client = new RestClient(_baseClient))
 138             {
 139                 if (!String.IsNullOrWhiteSpace(account))
 140                     client.BaseAddress = GetAccountUrl(account);
 141
 142                 client.Parameters.Clear();
 143                 client.Parameters.Add("format", "json");
 144                 var content = client.DownloadStringWithRetry("", 3);
 145                 client.AssertStatusOK("List Containers failed");
 146
 147                 if (client.StatusCode == HttpStatusCode.NoContent)
 148                     return new List<ContainerInfo>();
 149                 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
 150                 return infos;
 151             }
 152
 153         }
 154
 155         private string GetAccountUrl(string account)
 156         {
 157             return new Uri(this.RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
 158         }
 159
 160         public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
 161         {
 162             using (log4net.ThreadContext.Stacks["Share"].Push("List Accounts"))
 163             {
 164                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 165
 166                 using (var client = new RestClient(_baseClient))
 167                 {
 168                     client.Parameters.Clear();
 169                     client.Parameters.Add("format", "json");
 170                     client.IfModifiedSince = since;
 171
 172                     //Extract the username from the base address
 173                     client.BaseAddress = RootAddressUri.AbsoluteUri;
 174
 175                     var content = client.DownloadStringWithRetry(@"", 3);
 176
 177                     client.AssertStatusOK("ListSharingAccounts failed");
 178
 179                     //If the result is empty, return an empty list,
 180                     var infos = String.IsNullOrWhiteSpace(content)
 181                                     ? new List<ShareAccountInfo>()
 182                                 //Otherwise deserialize the account list into a list of ShareAccountInfos
 183                                     : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
 184
 185                     Log.DebugFormat("END");
 186                     return infos;
 187                 }
 188             }
 189         }
 190
 191         //Request listing of all objects in a container modified since a specific time.
 192         //If the *since* value is missing, return all objects
 193         public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
 194         {
 195
 196             using (log4net.ThreadContext.Stacks["Share"].Push("List Objects"))
 197             {
 198                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 199
 200                 var objects = new List<ObjectInfo>();
 201                 var accounts = ListSharingAccounts(since);
 202                 foreach (var account in accounts)
 203                 {
 204                     var containers = ListContainers(account.name);
 205                     foreach (var container in containers)
 206                     {
 207                         var containerObjects = ListObjects(account.name, container.Name, account.last_modified);
 208                         objects.AddRange(containerObjects);
 209                     }
 210                 }
 211                 if (Log.IsDebugEnabled) Log.DebugFormat("END");
 212                 return objects;
 213             }
 214         }
 215
 216         public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
 217         {
 218             if (String.IsNullOrWhiteSpace(Token))
 219                 throw new InvalidOperationException("The Token is not set");
 220             if (StorageUrl==null)
 221                 throw new InvalidOperationException("The StorageUrl is not set");
 222             if (String.IsNullOrWhiteSpace(container))
 223                 throw new ArgumentNullException("container");
 224             if (String.IsNullOrWhiteSpace(objectName))
 225                 throw new ArgumentNullException("objectName");
 226             if (String.IsNullOrWhiteSpace(account))
 227                 throw new ArgumentNullException("account");
 228             if (String.IsNullOrWhiteSpace(shareTo))
 229                 throw new ArgumentNullException("shareTo");
 230             Contract.EndContractBlock();
 231
 232             using (log4net.ThreadContext.Stacks["Share"].Push("Share Object"))
 233             {
 234                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 235
 236                 using (var client = new RestClient(_baseClient))
 237                 {
 238
 239                     client.BaseAddress = GetAccountUrl(account);
 240
 241                     client.Parameters.Clear();
 242                     client.Parameters.Add("format", "json");
 243
 244                     string permission = "";
 245                     if (write)
 246                         permission = String.Format("write={0}", shareTo);
 247                     else if (read)
 248                         permission = String.Format("read={0}", shareTo);
 249                     client.Headers.Add("X-Object-Sharing", permission);
 250
 251                     var content = client.DownloadStringWithRetry(container, 3);
 252
 253                     client.AssertStatusOK("ShareObject failed");
 254
 255                     //If the result is empty, return an empty list,
 256                     var infos = String.IsNullOrWhiteSpace(content)
 257                                     ? new List<ObjectInfo>()
 258                                 //Otherwise deserialize the object list into a list of ObjectInfos
 259                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 260
 261                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 262                 }
 263             }
 264
 265
 266         }
 267
 268
 269         public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
 270         {
 271             if (String.IsNullOrWhiteSpace(container))
 272                 throw new ArgumentNullException("container");
 273             Contract.EndContractBlock();
 274
 275             using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
 276             {
 277                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 278
 279                 using (var client = new RestClient(_baseClient))
 280                 {
 281                     if (!String.IsNullOrWhiteSpace(account))
 282                         client.BaseAddress = GetAccountUrl(account);
 283
 284                     client.Parameters.Clear();
 285                     client.Parameters.Add("format", "json");
 286                     client.IfModifiedSince = since;
 287                     var content = client.DownloadStringWithRetry(container, 3);
 288
 289                     client.AssertStatusOK("ListObjects failed");
 290
 291                     //If the result is empty, return an empty list,
 292                     var infos = String.IsNullOrWhiteSpace(content)
 293                                     ? new List<ObjectInfo>()
 294                                 //Otherwise deserialize the object list into a list of ObjectInfos
 295                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 296
 297                     foreach (var info in infos)
 298                     {
 299                         info.Container = container;
 300                         info.Account = account;
 301                     }
 302                     if (Log.IsDebugEnabled) Log.DebugFormat("START");
 303                     return infos;
 304                 }
 305             }
 306         }
 307
 308
 309
 310         public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
 311         {
 312             if (String.IsNullOrWhiteSpace(container))
 313                 throw new ArgumentNullException("container");
 314             if (String.IsNullOrWhiteSpace(folder))
 315                 throw new ArgumentNullException("folder");
 316             Contract.EndContractBlock();
 317
 318             using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
 319             {
 320                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 321
 322                 using (var client = new RestClient(_baseClient))
 323                 {
 324                     if (!String.IsNullOrWhiteSpace(account))
 325                         client.BaseAddress = GetAccountUrl(account);
 326
 327                     client.Parameters.Clear();
 328                     client.Parameters.Add("format", "json");
 329                     client.Parameters.Add("path", folder);
 330                     client.IfModifiedSince = since;
 331                     var content = client.DownloadStringWithRetry(container, 3);
 332                     client.AssertStatusOK("ListObjects failed");
 333
 334                     var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 335
 336                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 337                     return infos;
 338                 }
 339             }
 340         }
 341
 342
 343         public bool ContainerExists(string account, string container)
 344         {
 345             if (String.IsNullOrWhiteSpace(container))
 346                 throw new ArgumentNullException("container", "The container property can't be empty");
 347             Contract.EndContractBlock();
 348
 349             using (log4net.ThreadContext.Stacks["Containters"].Push("Exists"))
 350             {
 351                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 352
 353                 using (var client = new RestClient(_baseClient))
 354                 {
 355                     if (!String.IsNullOrWhiteSpace(account))
 356                         client.BaseAddress = GetAccountUrl(account);
 357
 358                     client.Parameters.Clear();
 359                     client.Head(container, 3);
 360
 361                     bool result;
 362                     switch (client.StatusCode)
 363                     {
 364                         case HttpStatusCode.OK:
 365                         case HttpStatusCode.NoContent:
 366                             result=true;
 367                             break;
 368                         case HttpStatusCode.NotFound:
 369                             result=false;
 370                             break;
 371                         default:
 372                             throw CreateWebException("ContainerExists", client.StatusCode);
 373                     }
 374                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 375
 376                     return result;
 377                 }
 378
 379             }
 380         }
 381
 382         public bool ObjectExists(string account, string container, string objectName)
 383         {
 384             if (String.IsNullOrWhiteSpace(container))
 385                 throw new ArgumentNullException("container", "The container property can't be empty");
 386             if (String.IsNullOrWhiteSpace(objectName))
 387                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 388             Contract.EndContractBlock();
 389
 390             using (var client = new RestClient(_baseClient))
 391             {
 392                 if (!String.IsNullOrWhiteSpace(account))
 393                     client.BaseAddress = GetAccountUrl(account);
 394
 395                 client.Parameters.Clear();
 396                 client.Head(container + "/" + objectName, 3);
 397
 398                 switch (client.StatusCode)
 399                 {
 400                     case HttpStatusCode.OK:
 401                     case HttpStatusCode.NoContent:
 402                         return true;
 403                     case HttpStatusCode.NotFound:
 404                         return false;
 405                     default:
 406                         throw CreateWebException("ObjectExists", client.StatusCode);
 407                 }
 408             }
 409
 410         }
 411
 412         public ObjectInfo GetObjectInfo(string account, string container, string objectName)
 413         {
 414             if (String.IsNullOrWhiteSpace(container))
 415                 throw new ArgumentNullException("container", "The container property can't be empty");
 416             if (String.IsNullOrWhiteSpace(objectName))
 417                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 418             Contract.EndContractBlock();
 419
 420             using (log4net.ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
 421             {
 422
 423                 using (var client = new RestClient(_baseClient))
 424                 {
 425                     if (!String.IsNullOrWhiteSpace(account))
 426                         client.BaseAddress = GetAccountUrl(account);
 427                     try
 428                     {
 429                         client.Parameters.Clear();
 430
 431                         client.Head(container + "/" + objectName, 3);
 432
 433                         if (client.TimedOut)
 434                             return ObjectInfo.Empty;
 435
 436                         switch (client.StatusCode)
 437                         {
 438                             case HttpStatusCode.OK:
 439                             case HttpStatusCode.NoContent:
 440                                 var keys = client.ResponseHeaders.AllKeys.AsQueryable();
 441                                 var tags = (from key in keys
 442                                             where key.StartsWith("X-Object-Meta-")
 443                                             let name = key.Substring(14)
 444                                             select new {Name = name, Value = client.ResponseHeaders[name]})
 445                                     .ToDictionary(t => t.Name, t => t.Value);
 446                                 var extensions = (from key in keys
 447                                                   where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
 448                                                   select new {Name = key, Value = client.ResponseHeaders[key]})
 449                                     .ToDictionary(t => t.Name, t => t.Value);
 450                                 var info = new ObjectInfo
 451                                                {
 452                                                    Name = objectName,
 453                                                    Hash = client.GetHeaderValue("ETag"),
 454                                                    Content_Type = client.GetHeaderValue("Content-Type"),
 455                                                    Tags = tags,
 456                                                    Last_Modified = client.LastModified,
 457                                                    Extensions = extensions
 458                                                };
 459                                 return info;
 460                             case HttpStatusCode.NotFound:
 461                                 return ObjectInfo.Empty;
 462                             default:
 463                                 throw new WebException(
 464                                     String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 465                                                   objectName, client.StatusCode));
 466                         }
 467
 468                     }
 469                     catch (RetryException)
 470                     {
 471                         Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.");
 472                         return ObjectInfo.Empty;
 473                     }
 474                     catch (WebException e)
 475                     {
 476                         Log.Error(
 477                             String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 478                                           objectName, client.StatusCode), e);
 479                         throw;
 480                     }
 481                 }
 482             }
 483
 484         }
 485
 486         public void CreateFolder(string account, string container, string folder)
 487         {
 488             if (String.IsNullOrWhiteSpace(container))
 489                 throw new ArgumentNullException("container", "The container property can't be empty");
 490             if (String.IsNullOrWhiteSpace(folder))
 491                 throw new ArgumentNullException("folder", "The folder property can't be empty");
 492             Contract.EndContractBlock();
 493
 494             var folderUrl=String.Format("{0}/{1}",container,folder);
 495             using (var client = new RestClient(_baseClient))
 496             {
 497                 if (!String.IsNullOrWhiteSpace(account))
 498                     client.BaseAddress = GetAccountUrl(account);
 499
 500                 client.Parameters.Clear();
 501                 client.Headers.Add("Content-Type", @"application/directory");
 502                 client.Headers.Add("Content-Length", "0");
 503                 client.PutWithRetry(folderUrl, 3);
 504
 505                 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
 506                     throw CreateWebException("CreateFolder", client.StatusCode);
 507             }
 508         }
 509
 510         public ContainerInfo GetContainerInfo(string account, string container)
 511         {
 512             if (String.IsNullOrWhiteSpace(container))
 513                 throw new ArgumentNullException("container", "The container property can't be empty");
 514             Contract.EndContractBlock();
 515
 516             using (var client = new RestClient(_baseClient))
 517             {
 518                 if (!String.IsNullOrWhiteSpace(account))
 519                     client.BaseAddress = GetAccountUrl(account);
 520
 521                 client.Head(container);
 522                 switch (client.StatusCode)
 523                 {
 524                     case HttpStatusCode.OK:
 525                     case HttpStatusCode.NoContent:
 526                         var containerInfo = new ContainerInfo
 527                                                 {
 528                                                     Name = container,
 529                                                     Count =
 530                                                         long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
 531                                                     Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
 532                                                     BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
 533                                                     BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size"))
 534                                                 };
 535                         return containerInfo;
 536                     case HttpStatusCode.NotFound:
 537                         return ContainerInfo.Empty;
 538                     default:
 539                         throw CreateWebException("GetContainerInfo", client.StatusCode);
 540                 }
 541             }
 542         }
 543
 544         public void CreateContainer(string account, string container)
 545         {
 546             if (String.IsNullOrWhiteSpace(container))
 547                 throw new ArgumentNullException("container", "The container property can't be empty");
 548             Contract.EndContractBlock();
 549
 550             using (var client = new RestClient(_baseClient))
 551             {
 552                 if (!String.IsNullOrWhiteSpace(account))
 553                     client.BaseAddress = GetAccountUrl(account);
 554
 555                 client.PutWithRetry(container, 3);
 556                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
 557                 if (!expectedCodes.Contains(client.StatusCode))
 558                     throw CreateWebException("CreateContainer", client.StatusCode);
 559             }
 560         }
 561
 562         public void DeleteContainer(string account, string container)
 563         {
 564             if (String.IsNullOrWhiteSpace(container))
 565                 throw new ArgumentNullException("container", "The container property can't be empty");
 566             Contract.EndContractBlock();
 567
 568             using (var client = new RestClient(_baseClient))
 569             {
 570                 if (!String.IsNullOrWhiteSpace(account))
 571                     client.BaseAddress = GetAccountUrl(account);
 572
 573                 client.DeleteWithRetry(container, 3);
 574                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
 575                 if (!expectedCodes.Contains(client.StatusCode))
 576                     throw CreateWebException("DeleteContainer", client.StatusCode);
 577             }
 578
 579         }
 580
 581         /// <summary>
 582         ///
 583         /// </summary>
 584         /// <param name="account"></param>
 585         /// <param name="container"></param>
 586         /// <param name="objectName"></param>
 587         /// <param name="fileName"></param>
 588         /// <returns></returns>
 589         /// <remarks>This method should have no timeout or a very long one</remarks>
 590         //Asynchronously download the object specified by *objectName* in a specific *container* to
 591         // a local file
 592         public Task GetObject(string account, string container, string objectName, string fileName)
 593         {
 594             if (String.IsNullOrWhiteSpace(container))
 595                 throw new ArgumentNullException("container", "The container property can't be empty");
 596             if (String.IsNullOrWhiteSpace(objectName))
 597                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 598             Contract.EndContractBlock();
 599
 600             try
 601             {
 602                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
 603                 //object to avoid concurrency errors.
 604                 //
 605                 //Download operations take a long time therefore they have no timeout.
 606                 var client = new RestClient(_baseClient) { Timeout = 0 };
 607                 if (!String.IsNullOrWhiteSpace(account))
 608                     client.BaseAddress = GetAccountUrl(account);
 609
 610                 //The container and objectName are relative names. They are joined with the client's
 611                 //BaseAddress to create the object's absolute address
 612                 var builder = client.GetAddressBuilder(container, objectName);
 613                 var uri = builder.Uri;
 614
 615                 //Download progress is reported to the Trace log
 616                 Log.InfoFormat("[GET] START {0}", objectName);
 617                 client.DownloadProgressChanged += (sender, args) =>
 618                     Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
 619                                     fileName, args.ProgressPercentage,
 620                                     args.BytesReceived,
 621                                     args.TotalBytesToReceive);
 622
 623
 624                 //Start downloading the object asynchronously
 625                 var downloadTask = client.DownloadFileTask(uri, fileName);
 626
 627                 //Once the download completes
 628                 return downloadTask.ContinueWith(download =>
 629                                       {
 630                                           //Delete the local client object
 631                                           client.Dispose();
 632                                           //And report failure or completion
 633                                           if (download.IsFaulted)
 634                                           {
 635                                               Log.ErrorFormat("[GET] FAIL for {0} with \r{1}", objectName,
 636                                                                download.Exception);
 637                                           }
 638                                           else
 639                                           {
 640                                               Log.InfoFormat("[GET] END {0}", objectName);
 641                                           }
 642                                       });
 643             }
 644             catch (Exception exc)
 645             {
 646                 Log.ErrorFormat("[GET] END {0} with {1}", objectName, exc);
 647                 throw;
 648             }
 649
 650
 651
 652         }
 653
 654         public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
 655         {
 656             if (String.IsNullOrWhiteSpace(container))
 657                 throw new ArgumentNullException("container");
 658             if (String.IsNullOrWhiteSpace(objectName))
 659                 throw new ArgumentNullException("objectName");
 660             if (hash==null)
 661                 throw new ArgumentNullException("hash");
 662             if (String.IsNullOrWhiteSpace(Token))
 663                 throw new InvalidOperationException("Invalid Token");
 664             if (StorageUrl == null)
 665                 throw new InvalidOperationException("Invalid Storage Url");
 666             Contract.EndContractBlock();
 667
 668
 669             //Don't use a timeout because putting the hashmap may be a long process
 670             var client = new RestClient(_baseClient) { Timeout = 0 };
 671             if (!String.IsNullOrWhiteSpace(account))
 672                 client.BaseAddress = GetAccountUrl(account);
 673
 674             //The container and objectName are relative names. They are joined with the client's
 675             //BaseAddress to create the object's absolute address
 676             var builder = client.GetAddressBuilder(container, objectName);
 677             builder.Query = "format=json&hashmap";
 678             var uri = builder.Uri;
 679
 680
 681             //Send the tree hash as Json to the server
 682             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
 683             var uploadTask=client.UploadStringTask(uri, "PUT", hash.ToJson());
 684
 685
 686             return uploadTask.ContinueWith(t =>
 687             {
 688
 689                 var empty = (IList<string>)new List<string>();
 690
 691
 692                 //The server will respond either with 201-created if all blocks were already on the server
 693                 if (client.StatusCode == HttpStatusCode.Created)
 694                 {
 695                     //in which case we return an empty hash list
 696                     return empty;
 697                 }
 698                 //or with a 409-conflict and return the list of missing parts
 699                 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
 700                 if (t.IsFaulted)
 701                 {
 702                     var ex = t.Exception.InnerException;
 703                     var we = ex as WebException;
 704                     var response = we.Response as HttpWebResponse;
 705                     if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
 706                     {
 707                         //In case of 409 the missing parts will be in the response content
 708                         using (var stream = response.GetResponseStream())
 709                         using(var reader=new StreamReader(stream))
 710                         {
 711                             //We need to cleanup the content before returning it because it contains
 712                             //error content after the list of hashes
 713                             var hashes = new List<string>();
 714                             string line=null;
 715                             //All lines up to the first empty line are hashes
 716                             while(!String.IsNullOrWhiteSpace(line=reader.ReadLine()))
 717                             {
 718                                 hashes.Add(line);
 719                             }
 720
 721                             return hashes;
 722                         }
 723                     }
 724                     else
 725                         //Any other status code is unexpected and the exception should be rethrown
 726                         throw ex;
 727
 728                 }
 729                 //Any other status code is unexpected but there was no exception. We can probably continue processing
 730                 else
 731                 {
 732                     Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
 733                 }
 734                 return empty;
 735             });
 736
 737         }
 738
 739         public Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end)
 740         {
 741             if (String.IsNullOrWhiteSpace(Token))
 742                 throw new InvalidOperationException("Invalid Token");
 743             if (StorageUrl == null)
 744                 throw new InvalidOperationException("Invalid Storage Url");
 745             if (String.IsNullOrWhiteSpace(container))
 746                 throw new ArgumentNullException("container");
 747             if (relativeUrl== null)
 748                 throw new ArgumentNullException("relativeUrl");
 749             if (end.HasValue && end<0)
 750                 throw new ArgumentOutOfRangeException("end");
 751             if (start<0)
 752                 throw new ArgumentOutOfRangeException("start");
 753             Contract.EndContractBlock();
 754
 755
 756             //Don't use a timeout because putting the hashmap may be a long process
 757             var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
 758             if (!String.IsNullOrWhiteSpace(account))
 759                 client.BaseAddress = GetAccountUrl(account);
 760
 761             var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
 762             var uri = builder.Uri;
 763
 764             return client.DownloadDataTask(uri)
 765                 .ContinueWith(t=>
 766                                   {
 767                                       client.Dispose();
 768                                       return t.Result;
 769                                   });
 770         }
 771
 772
 773         public Task PostBlock(string account, string container, byte[] block, int offset, int count)
 774         {
 775             if (String.IsNullOrWhiteSpace(container))
 776                 throw new ArgumentNullException("container");
 777             if (block == null)
 778                 throw new ArgumentNullException("block");
 779             if (offset < 0 || offset >= block.Length)
 780                 throw new ArgumentOutOfRangeException("offset");
 781             if (count < 0 || count > block.Length)
 782                 throw new ArgumentOutOfRangeException("count");
 783             if (String.IsNullOrWhiteSpace(Token))
 784                 throw new InvalidOperationException("Invalid Token");
 785             if (StorageUrl == null)
 786                 throw new InvalidOperationException("Invalid Storage Url");
 787             Contract.EndContractBlock();
 788
 789
 790             //Don't use a timeout because putting the hashmap may be a long process
 791             var client = new RestClient(_baseClient) { Timeout = 0 };
 792             if (!String.IsNullOrWhiteSpace(account))
 793                 client.BaseAddress = GetAccountUrl(account);
 794
 795             var builder = client.GetAddressBuilder(container, "");
 796             //We are doing an update
 797             builder.Query = "update";
 798             var uri = builder.Uri;
 799
 800             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
 801
 802             Log.InfoFormat("[BLOCK POST] START");
 803
 804             client.UploadProgressChanged += (sender, args) =>
 805                 Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
 806                                     args.ProgressPercentage, args.BytesSent,
 807                                     args.TotalBytesToSend);
 808             client.UploadFileCompleted += (sender, args) =>
 809                 Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
 810
 811
 812             //Send the block
 813             var uploadTask = client.UploadDataTask(uri, "POST", block)
 814             .ContinueWith(upload =>
 815             {
 816                 client.Dispose();
 817
 818                 if (upload.IsFaulted)
 819                 {
 820                     var exception = upload.Exception.InnerException;
 821                     Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exception);
 822                     throw exception;
 823                 }
 824
 825                 Log.InfoFormat("[BLOCK POST] END");
 826             });
 827             return uploadTask;
 828         }
 829
 830
 831         public Task<TreeHash> GetHashMap(string account, string container, string objectName)
 832         {
 833             if (String.IsNullOrWhiteSpace(container))
 834                 throw new ArgumentNullException("container");
 835             if (String.IsNullOrWhiteSpace(objectName))
 836                 throw new ArgumentNullException("objectName");
 837             if (String.IsNullOrWhiteSpace(Token))
 838                 throw new InvalidOperationException("Invalid Token");
 839             if (StorageUrl == null)
 840                 throw new InvalidOperationException("Invalid Storage Url");
 841             Contract.EndContractBlock();
 842
 843             try
 844             {
 845                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
 846                 //object to avoid concurrency errors.
 847                 //
 848                 //Download operations take a long time therefore they have no timeout.
 849                 //TODO: Do we really? this is a hashmap operation, not a download
 850                 var client = new RestClient(_baseClient) { Timeout = 0 };
 851                 if (!String.IsNullOrWhiteSpace(account))
 852                     client.BaseAddress = GetAccountUrl(account);
 853
 854
 855                 //The container and objectName are relative names. They are joined with the client's
 856                 //BaseAddress to create the object's absolute address
 857                 var builder = client.GetAddressBuilder(container, objectName);
 858                 builder.Query = "format=json&hashmap";
 859                 var uri = builder.Uri;
 860
 861                 //Start downloading the object asynchronously
 862                 var downloadTask = client.DownloadStringTask(uri);
 863
 864                 //Once the download completes
 865                 return downloadTask.ContinueWith(download =>
 866                 {
 867                     //Delete the local client object
 868                     client.Dispose();
 869                     //And report failure or completion
 870                     if (download.IsFaulted)
 871                     {
 872                         Log.ErrorFormat("[GET HASH] FAIL for {0} with \r{1}", objectName,
 873                                         download.Exception);
 874                         throw download.Exception;
 875                     }
 876
 877                     //The server will return an empty string if the file is empty
 878                     var json = download.Result;
 879                     var treeHash = TreeHash.Parse(json);
 880                     Log.InfoFormat("[GET HASH] END {0}", objectName);
 881                     return treeHash;
 882                 });
 883             }
 884             catch (Exception exc)
 885             {
 886                 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
 887                 throw;
 888             }
 889
 890
 891
 892         }
 893
 894
 895         /// <summary>
 896         ///
 897         /// </summary>
 898         /// <param name="account"></param>
 899         /// <param name="container"></param>
 900         /// <param name="objectName"></param>
 901         /// <param name="fileName"></param>
 902         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
 903         /// <remarks>>This method should have no timeout or a very long one</remarks>
 904         public Task PutObject(string account, string container, string objectName, string fileName, string hash = null)
 905         {
 906             if (String.IsNullOrWhiteSpace(container))
 907                 throw new ArgumentNullException("container", "The container property can't be empty");
 908             if (String.IsNullOrWhiteSpace(objectName))
 909                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 910             if (String.IsNullOrWhiteSpace(fileName))
 911                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
 912             if (!File.Exists(fileName))
 913                 throw new FileNotFoundException("The file does not exist",fileName);
 914             Contract.EndContractBlock();
 915
 916             try
 917             {
 918
 919                 var client = new RestClient(_baseClient){Timeout=0};
 920                 if (!String.IsNullOrWhiteSpace(account))
 921                     client.BaseAddress = GetAccountUrl(account);
 922
 923                 var builder = client.GetAddressBuilder(container, objectName);
 924                 var uri = builder.Uri;
 925
 926                 string etag = hash ?? CalculateHash(fileName);
 927
 928                 client.Headers.Add("Content-Type", "application/octet-stream");
 929                 client.Headers.Add("ETag", etag);
 930
 931
 932                 Log.InfoFormat("[PUT] START {0}", objectName);
 933                 client.UploadProgressChanged += (sender, args) =>
 934                 {
 935                     using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
 936                     {
 937                         Log.InfoFormat("{0} {1}% {2} of {3}", fileName, args.ProgressPercentage,
 938                                        args.BytesSent, args.TotalBytesToSend);
 939                     }
 940                 };
 941
 942                 client.UploadFileCompleted += (sender, args) =>
 943                 {
 944                     using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
 945                     {
 946                         Log.InfoFormat("Completed {0}", fileName);
 947                     }
 948                 };
 949                 return client.UploadFileTask(uri, "PUT", fileName)
 950                     .ContinueWith(upload=>
 951                                       {
 952                                           client.Dispose();
 953
 954                                           if (upload.IsFaulted)
 955                                           {
 956                                               var exc = upload.Exception.InnerException;
 957                                               Log.ErrorFormat("[PUT] FAIL for {0} with \r{1}",objectName,exc);
 958                                               throw exc;
 959                                           }
 960                                           else
 961                                             Log.InfoFormat("[PUT] END {0}", objectName);
 962                                       });
 963             }
 964             catch (Exception exc)
 965             {
 966                 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
 967                 throw;
 968             }
 969
 970         }
 971
 972
 973         private static string CalculateHash(string fileName)
 974         {
 975             string hash;
 976             using (var hasher = MD5.Create())
 977             using(var stream=File.OpenRead(fileName))
 978             {
 979                 var hashBuilder=new StringBuilder();
 980                 foreach (byte b in hasher.ComputeHash(stream))
 981                     hashBuilder.Append(b.ToString("x2").ToLower());
 982                 hash = hashBuilder.ToString();
 983             }
 984             return hash;
 985         }
 986
 987        /* public void DeleteObject(string container, string objectName,string account)
 988         {
 989             if (String.IsNullOrWhiteSpace(container))
 990                 throw new ArgumentNullException("container", "The container property can't be empty");
 991             if (String.IsNullOrWhiteSpace(objectName))
 992                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 993             Contract.EndContractBlock();
 994
 995             using (var client = new RestClient(_baseClient))
 996             {
 997                 if (!String.IsNullOrWhiteSpace(account))
 998                     client.BaseAddress = GetAccountUrl(account);
 999                 );
1000                 client.DeleteWithRetry(container + "/" + objectName, 3);
1001
1002                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
1003                 if (!expectedCodes.Contains(client.StatusCode))
1004                     throw CreateWebException("DeleteObject", client.StatusCode);
1005             }
1006
1007         }*/
1008
1009         public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
1010         {
1011             if (String.IsNullOrWhiteSpace(sourceContainer))
1012                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1013             if (String.IsNullOrWhiteSpace(oldObjectName))
1014                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
1015             if (String.IsNullOrWhiteSpace(targetContainer))
1016                 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
1017             if (String.IsNullOrWhiteSpace(newObjectName))
1018                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
1019             Contract.EndContractBlock();
1020
1021             var targetUrl = targetContainer + "/" + newObjectName;
1022             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
1023
1024             using (var client = new RestClient(_baseClient))
1025             {
1026                 if (!String.IsNullOrWhiteSpace(account))
1027                     client.BaseAddress = GetAccountUrl(account);
1028
1029                 client.Headers.Add("X-Move-From", sourceUrl);
1030                 client.PutWithRetry(targetUrl, 3);
1031
1032                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1033                 if (!expectedCodes.Contains(client.StatusCode))
1034                     throw CreateWebException("MoveObject", client.StatusCode);
1035             }
1036         }
1037
1038         public void DeleteObject(string account, string sourceContainer, string objectName, string targetContainer)
1039         {
1040             if (String.IsNullOrWhiteSpace(sourceContainer))
1041                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1042             if (String.IsNullOrWhiteSpace(objectName))
1043                 throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
1044             if (String.IsNullOrWhiteSpace(targetContainer))
1045                 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
1046             Contract.EndContractBlock();
1047
1048             var targetUrl = targetContainer + "/" + objectName;
1049             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
1050
1051             using (var client = new RestClient(_baseClient))
1052             {
1053                 if (!String.IsNullOrWhiteSpace(account))
1054                     client.BaseAddress = GetAccountUrl(account);
1055
1056                 client.Headers.Add("X-Move-From", sourceUrl);
1057                 client.PutWithRetry(targetUrl, 3);
1058
1059                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
1060                 if (!expectedCodes.Contains(client.StatusCode))
1061                     throw CreateWebException("DeleteObject", client.StatusCode);
1062             }
1063         }
1064
1065
1066         private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
1067         {
1068             return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
1069         }
1070
1071
1072     }
1073
1074     public class ShareAccountInfo
1075     {
1076         public DateTime? last_modified { get; set; }
1077         public string name { get; set; }
1078     }
1079 }