code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="CloudFilesClient.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42
  43 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
  44 //
  45 // The class provides methods to upload/download files, delete files, manage containers
  46
  47
  48 using System;
  49 using System.Collections.Generic;
  50 using System.Collections.Specialized;
  51 using System.ComponentModel.Composition;
  52 using System.Diagnostics;
  53 using System.Diagnostics.Contracts;
  54 using System.IO;
  55 using System.Linq;
  56 using System.Net;
  57 using System.Reflection;
  58 using System.Security.Cryptography;
  59 using System.Text;
  60 using System.Threading.Tasks;
  61 using Newtonsoft.Json;
  62 using Pithos.Interfaces;
  63 using log4net;
  64
  65 namespace Pithos.Network
  66 {
  67     [Export(typeof(ICloudClient))]
  68     public class CloudFilesClient:ICloudClient
  69     {
  70         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  71
  72         //CloudFilesClient uses *_baseClient* internally to communicate with the server
  73         //RestClient provides a REST-friendly interface over the standard WebClient.
  74         private RestClient _baseClient;
  75
  76
  77         //During authentication the client provides a UserName
  78         public string UserName { get; set; }
  79
  80         //and and ApiKey to the server
  81         public string ApiKey { get; set; }
  82
  83         //And receives an authentication Token. This token must be provided in ALL other operations,
  84         //in the X-Auth-Token header
  85         private string _token;
  86         public string Token
  87         {
  88             get { return _token; }
  89             set
  90             {
  91                 _token = value;
  92                 _baseClient.Headers["X-Auth-Token"] = value;
  93             }
  94         }
  95
  96         //The client also receives a StorageUrl after authentication. All subsequent operations must
  97         //use this url
  98         public Uri StorageUrl { get; set; }
  99
 100
 101         protected Uri RootAddressUri { get; set; }
 102
 103        /* private WebProxy _proxy;
 104         public WebProxy Proxy
 105         {
 106             get { return _proxy; }
 107             set
 108             {
 109                 _proxy = value;
 110                 if (_baseClient != null)
 111                     _baseClient.Proxy = value;
 112             }
 113         }
 114 */
 115
 116         /* private Uri _proxy;
 117         public Uri Proxy
 118         {
 119             get { return _proxy; }
 120             set
 121             {
 122                 _proxy = value;
 123                 if (_baseClient != null)
 124                     _baseClient.Proxy = new WebProxy(value);
 125             }
 126         }*/
 127
 128         public double DownloadPercentLimit { get; set; }
 129         public double UploadPercentLimit { get; set; }
 130
 131         public string AuthenticationUrl { get; set; }
 132
 133
 134         public string VersionPath
 135         {
 136             get { return UsePithos ? "v1" : "v1.0"; }
 137         }
 138
 139         public bool UsePithos { get; set; }
 140
 141
 142
 143         public CloudFilesClient(string userName, string apiKey)
 144         {
 145             UserName = userName;
 146             ApiKey = apiKey;
 147         }
 148
 149         public CloudFilesClient(AccountInfo accountInfo)
 150         {
 151             if (accountInfo==null)
 152                 throw new ArgumentNullException("accountInfo");
 153             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
 154             Contract.Ensures(StorageUrl != null);
 155             Contract.Ensures(_baseClient != null);
 156             Contract.Ensures(RootAddressUri != null);
 157             Contract.EndContractBlock();
 158
 159             _baseClient = new RestClient
 160             {
 161                 BaseAddress = accountInfo.StorageUri.ToString(),
 162                 Timeout = 10000,
 163                 Retries = 3,
 164             };
 165             StorageUrl = accountInfo.StorageUri;
 166             Token = accountInfo.Token;
 167             UserName = accountInfo.UserName;
 168
 169             //Get the root address (StorageUrl without the account)
 170             var storageUrl = StorageUrl.AbsoluteUri;
 171             var usernameIndex = storageUrl.LastIndexOf(UserName);
 172             var rootUrl = storageUrl.Substring(0, usernameIndex);
 173             RootAddressUri = new Uri(rootUrl);
 174         }
 175
 176
 177         public AccountInfo Authenticate()
 178         {
 179             if (String.IsNullOrWhiteSpace(UserName))
 180                 throw new InvalidOperationException("UserName is empty");
 181             if (String.IsNullOrWhiteSpace(ApiKey))
 182                 throw new InvalidOperationException("ApiKey is empty");
 183             if (String.IsNullOrWhiteSpace(AuthenticationUrl))
 184                 throw new InvalidOperationException("AuthenticationUrl is empty");
 185             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
 186             Contract.Ensures(StorageUrl != null);
 187             Contract.Ensures(_baseClient != null);
 188             Contract.Ensures(RootAddressUri != null);
 189             Contract.EndContractBlock();
 190
 191
 192             Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
 193
 194             var groups = new List<Group>();
 195
 196             using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
 197             {
 198                /* if (Proxy != null)
 199                     authClient.Proxy = Proxy;*/
 200
 201                 Contract.Assume(authClient.Headers!=null);
 202
 203                 authClient.Headers.Add("X-Auth-User", UserName);
 204                 authClient.Headers.Add("X-Auth-Key", ApiKey);
 205
 206                 authClient.DownloadStringWithRetry(VersionPath, 3);
 207
 208                 authClient.AssertStatusOK("Authentication failed");
 209
 210                 var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
 211                 if (String.IsNullOrWhiteSpace(storageUrl))
 212                     throw new InvalidOperationException("Failed to obtain storage url");
 213
 214                 _baseClient = new RestClient
 215                 {
 216                     BaseAddress = storageUrl,
 217                     Timeout = 10000,
 218                     Retries = 3,
 219                     //Proxy=Proxy
 220                 };
 221
 222                 StorageUrl = new Uri(storageUrl);
 223
 224                 //Get the root address (StorageUrl without the account)
 225                 var usernameIndex=storageUrl.LastIndexOf(UserName);
 226                 var rootUrl = storageUrl.Substring(0, usernameIndex);
 227                 RootAddressUri = new Uri(rootUrl);
 228
 229                 var token = authClient.GetHeaderValue("X-Auth-Token");
 230                 if (String.IsNullOrWhiteSpace(token))
 231                     throw new InvalidOperationException("Failed to obtain token url");
 232                 Token = token;
 233
 234                /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
 235                 groups = (from key in keys
 236                             where key.StartsWith("X-Account-Group-")
 237                             let name = key.Substring(16)
 238                             select new Group(name, authClient.ResponseHeaders[key]))
 239                         .ToList();
 240
 241 */
 242             }
 243
 244             Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
 245             Debug.Assert(_baseClient!=null);
 246
 247             return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};
 248
 249         }
 250
 251
 252
 253         public IList<ContainerInfo> ListContainers(string account)
 254         {
 255             using (var client = new RestClient(_baseClient))
 256             {
 257                 if (!String.IsNullOrWhiteSpace(account))
 258                     client.BaseAddress = GetAccountUrl(account);
 259
 260                 client.Parameters.Clear();
 261                 client.Parameters.Add("format", "json");
 262                 var content = client.DownloadStringWithRetry("", 3);
 263                 client.AssertStatusOK("List Containers failed");
 264
 265                 if (client.StatusCode == HttpStatusCode.NoContent)
 266                     return new List<ContainerInfo>();
 267                 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
 268
 269                 foreach (var info in infos)
 270                 {
 271                     info.Account = account;
 272                 }
 273                 return infos;
 274             }
 275
 276         }
 277
 278         private string GetAccountUrl(string account)
 279         {
 280             return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
 281         }
 282
 283         public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
 284         {
 285             using (ThreadContext.Stacks["Share"].Push("List Accounts"))
 286             {
 287                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 288
 289                 using (var client = new RestClient(_baseClient))
 290                 {
 291                     client.Parameters.Clear();
 292                     client.Parameters.Add("format", "json");
 293                     client.IfModifiedSince = since;
 294
 295                     //Extract the username from the base address
 296                     client.BaseAddress = RootAddressUri.AbsoluteUri;
 297
 298                     var content = client.DownloadStringWithRetry(@"", 3);
 299
 300                     client.AssertStatusOK("ListSharingAccounts failed");
 301
 302                     //If the result is empty, return an empty list,
 303                     var infos = String.IsNullOrWhiteSpace(content)
 304                                     ? new List<ShareAccountInfo>()
 305                                 //Otherwise deserialize the account list into a list of ShareAccountInfos
 306                                     : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
 307
 308                     Log.DebugFormat("END");
 309                     return infos;
 310                 }
 311             }
 312         }
 313
 314         //Request listing of all objects in a container modified since a specific time.
 315         //If the *since* value is missing, return all objects
 316         public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
 317         {
 318
 319             using (ThreadContext.Stacks["Share"].Push("List Objects"))
 320             {
 321                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 322                 //'since' is not used here because we need to have ListObjects return a NoChange result
 323                 //for all shared accounts,containers
 324                 var accounts = ListSharingAccounts();
 325                 var items = from account in accounts
 326                             let containers = ListContainers(account.name)
 327                             from container in containers
 328                             select ListObjects(account.name, container.Name,since);
 329                 var objects=items.SelectMany(r=> r).ToList();
 330 /*
 331                 var objects = new List<ObjectInfo>();
 332                 foreach (var containerObjects in items)
 333                 {
 334                     objects.AddRange(containerObjects);
 335                 }
 336 */
 337                 if (Log.IsDebugEnabled) Log.DebugFormat("END");
 338                 return objects;
 339             }
 340         }
 341
 342         public void SetTags(ObjectInfo target,IDictionary<string,string> tags)
 343         {
 344             if (String.IsNullOrWhiteSpace(Token))
 345                 throw new InvalidOperationException("The Token is not set");
 346             if (StorageUrl == null)
 347                 throw new InvalidOperationException("The StorageUrl is not set");
 348             if (target == null)
 349                 throw new ArgumentNullException("target");
 350             Contract.EndContractBlock();
 351
 352             using (ThreadContext.Stacks["Share"].Push("Share Object"))
 353             {
 354                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 355
 356                 using (var client = new RestClient(_baseClient))
 357                 {
 358
 359                     client.BaseAddress = GetAccountUrl(target.Account);
 360
 361                     client.Parameters.Clear();
 362                     client.Parameters.Add("update", "");
 363
 364                     foreach (var tag in tags)
 365                     {
 366                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
 367                         client.Headers.Add(headerTag, tag.Value);
 368                     }
 369
 370                     client.DownloadStringWithRetry(target.Container, 3);
 371
 372
 373                     client.AssertStatusOK("SetTags failed");
 374                     //If the status is NOT ACCEPTED we have a problem
 375                     if (client.StatusCode != HttpStatusCode.Accepted)
 376                     {
 377                         Log.Error("Failed to set tags");
 378                         throw new Exception("Failed to set tags");
 379                     }
 380
 381                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 382                 }
 383             }
 384
 385
 386         }
 387
 388         public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
 389         {
 390             if (String.IsNullOrWhiteSpace(Token))
 391                 throw new InvalidOperationException("The Token is not set");
 392             if (StorageUrl==null)
 393                 throw new InvalidOperationException("The StorageUrl is not set");
 394             if (String.IsNullOrWhiteSpace(container))
 395                 throw new ArgumentNullException("container");
 396             if (String.IsNullOrWhiteSpace(objectName))
 397                 throw new ArgumentNullException("objectName");
 398             if (String.IsNullOrWhiteSpace(account))
 399                 throw new ArgumentNullException("account");
 400             if (String.IsNullOrWhiteSpace(shareTo))
 401                 throw new ArgumentNullException("shareTo");
 402             Contract.EndContractBlock();
 403
 404             using (ThreadContext.Stacks["Share"].Push("Share Object"))
 405             {
 406                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 407
 408                 using (var client = new RestClient(_baseClient))
 409                 {
 410
 411                     client.BaseAddress = GetAccountUrl(account);
 412
 413                     client.Parameters.Clear();
 414                     client.Parameters.Add("format", "json");
 415
 416                     string permission = "";
 417                     if (write)
 418                         permission = String.Format("write={0}", shareTo);
 419                     else if (read)
 420                         permission = String.Format("read={0}", shareTo);
 421                     client.Headers.Add("X-Object-Sharing", permission);
 422
 423                     var content = client.DownloadStringWithRetry(container, 3);
 424
 425                     client.AssertStatusOK("ShareObject failed");
 426
 427                     //If the result is empty, return an empty list,
 428                     var infos = String.IsNullOrWhiteSpace(content)
 429                                     ? new List<ObjectInfo>()
 430                                 //Otherwise deserialize the object list into a list of ObjectInfos
 431                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 432
 433                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 434                 }
 435             }
 436
 437
 438         }
 439
 440         public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
 441         {
 442             if (accountInfo==null)
 443                 throw new ArgumentNullException("accountInfo");
 444             Contract.EndContractBlock();
 445
 446             using (ThreadContext.Stacks["Account"].Push("GetPolicies"))
 447             {
 448                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 449
 450                 using (var client = new RestClient(_baseClient))
 451                 {
 452                     if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
 453                         client.BaseAddress = GetAccountUrl(accountInfo.UserName);
 454
 455                     client.Parameters.Clear();
 456                     client.Parameters.Add("format", "json");
 457                     client.Head(String.Empty, 3);
 458
 459                     var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
 460                     var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
 461
 462                     long quota, bytes;
 463                     if (long.TryParse(quotaValue, out quota))
 464                         accountInfo.Quota = quota;
 465                     if (long.TryParse(bytesValue, out bytes))
 466                         accountInfo.BytesUsed = bytes;
 467
 468                     return accountInfo;
 469
 470                 }
 471
 472             }
 473         }
 474
 475         public void UpdateMetadata(ObjectInfo objectInfo)
 476         {
 477             if (objectInfo == null)
 478                 throw new ArgumentNullException("objectInfo");
 479             Contract.EndContractBlock();
 480
 481             using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
 482             {
 483                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 484
 485
 486                 using(var client=new RestClient(_baseClient))
 487                 {
 488
 489                     client.BaseAddress = GetAccountUrl(objectInfo.Account);
 490
 491                     client.Parameters.Clear();
 492
 493
 494                     //Set Tags
 495                     foreach (var tag in objectInfo.Tags)
 496                     {
 497                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
 498                         client.Headers.Add(headerTag, tag.Value);
 499                     }
 500
 501                     //Set Permissions
 502
 503                     var permissions=objectInfo.GetPermissionString();
 504                     client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);
 505
 506                     client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
 507                     client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
 508                     client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
 509                     var isPublic = objectInfo.IsPublic.ToString().ToLower();
 510                     client.Headers.Add("X-Object-Public", isPublic);
 511
 512
 513                     /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);
 514                     uriBuilder.Query = "update=";
 515                     var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/
 516                     var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);
 517                     client.PostWithRetry(address,"application/xml");
 518
 519                     //client.UploadValues(uri,new NameValueCollection());
 520
 521
 522                     client.AssertStatusOK("UpdateMetadata failed");
 523                     //If the status is NOT ACCEPTED or OK we have a problem
 524                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
 525                     {
 526                         Log.Error("Failed to update metadata");
 527                         throw new Exception("Failed to update metadata");
 528                     }
 529
 530                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 531                 }
 532             }
 533
 534         }
 535
 536         public void UpdateMetadata(ContainerInfo containerInfo)
 537         {
 538             if (containerInfo == null)
 539                 throw new ArgumentNullException("containerInfo");
 540             Contract.EndContractBlock();
 541
 542             using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
 543             {
 544                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 545
 546
 547                 using(var client=new RestClient(_baseClient))
 548                 {
 549
 550                     client.BaseAddress = GetAccountUrl(containerInfo.Account);
 551
 552                     client.Parameters.Clear();
 553
 554
 555                     //Set Tags
 556                     foreach (var tag in containerInfo.Tags)
 557                     {
 558                         var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
 559                         client.Headers.Add(headerTag, tag.Value);
 560                     }
 561
 562
 563                     //Set Policies
 564                     foreach (var policy in containerInfo.Policies)
 565                     {
 566                         var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
 567                         client.Headers.Add(headerPolicy, policy.Value);
 568                     }
 569
 570
 571                     var uriBuilder = client.GetAddressBuilder(containerInfo.Name,"");
 572                     var uri = uriBuilder.Uri;
 573
 574                     client.UploadValues(uri,new NameValueCollection());
 575
 576
 577                     client.AssertStatusOK("UpdateMetadata failed");
 578                     //If the status is NOT ACCEPTED or OK we have a problem
 579                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
 580                     {
 581                         Log.Error("Failed to update metadata");
 582                         throw new Exception("Failed to update metadata");
 583                     }
 584
 585                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 586                 }
 587             }
 588
 589         }
 590
 591
 592         public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
 593         {
 594             if (String.IsNullOrWhiteSpace(container))
 595                 throw new ArgumentNullException("container");
 596             Contract.EndContractBlock();
 597
 598             using (ThreadContext.Stacks["Objects"].Push("List"))
 599             {
 600                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 601
 602                 using (var client = new RestClient(_baseClient))
 603                 {
 604                     if (!String.IsNullOrWhiteSpace(account))
 605                         client.BaseAddress = GetAccountUrl(account);
 606
 607                     client.Parameters.Clear();
 608                     client.Parameters.Add("format", "json");
 609                     client.IfModifiedSince = since;
 610                     var content = client.DownloadStringWithRetry(container, 3);
 611
 612                     client.AssertStatusOK("ListObjects failed");
 613
 614                     if (client.StatusCode==HttpStatusCode.NotModified)
 615                         return new[]{new NoModificationInfo(account,container)};
 616                     //If the result is empty, return an empty list,
 617                     var infos = String.IsNullOrWhiteSpace(content)
 618                                     ? new List<ObjectInfo>()
 619                                 //Otherwise deserialize the object list into a list of ObjectInfos
 620                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 621
 622                     foreach (var info in infos)
 623                     {
 624                         info.Container = container;
 625                         info.Account = account;
 626                         info.StorageUri = this.StorageUrl;
 627                     }
 628                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 629                     return infos;
 630                 }
 631             }
 632         }
 633
 634         public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
 635         {
 636             if (String.IsNullOrWhiteSpace(container))
 637                 throw new ArgumentNullException("container");
 638 /*
 639             if (String.IsNullOrWhiteSpace(folder))
 640                 throw new ArgumentNullException("folder");
 641 */
 642             Contract.EndContractBlock();
 643
 644             using (ThreadContext.Stacks["Objects"].Push("List"))
 645             {
 646                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 647
 648                 using (var client = new RestClient(_baseClient))
 649                 {
 650                     if (!String.IsNullOrWhiteSpace(account))
 651                         client.BaseAddress = GetAccountUrl(account);
 652
 653                     client.Parameters.Clear();
 654                     client.Parameters.Add("format", "json");
 655                     client.Parameters.Add("path", folder);
 656                     client.IfModifiedSince = since;
 657                     var content = client.DownloadStringWithRetry(container, 3);
 658                     client.AssertStatusOK("ListObjects failed");
 659
 660                     if (client.StatusCode==HttpStatusCode.NotModified)
 661                         return new[]{new NoModificationInfo(account,container,folder)};
 662
 663                     var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 664                     foreach (var info in infos)
 665                     {
 666                         info.Account = account;
 667                         if (info.Container == null)
 668                             info.Container = container;
 669                         info.StorageUri = this.StorageUrl;
 670                     }
 671                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 672                     return infos;
 673                 }
 674             }
 675         }
 676
 677
 678         public bool ContainerExists(string account, string container)
 679         {
 680             if (String.IsNullOrWhiteSpace(container))
 681                 throw new ArgumentNullException("container", "The container property can't be empty");
 682             Contract.EndContractBlock();
 683
 684             using (ThreadContext.Stacks["Containters"].Push("Exists"))
 685             {
 686                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 687
 688                 using (var client = new RestClient(_baseClient))
 689                 {
 690                     if (!String.IsNullOrWhiteSpace(account))
 691                         client.BaseAddress = GetAccountUrl(account);
 692
 693                     client.Parameters.Clear();
 694                     client.Head(container, 3);
 695
 696                     bool result;
 697                     switch (client.StatusCode)
 698                     {
 699                         case HttpStatusCode.OK:
 700                         case HttpStatusCode.NoContent:
 701                             result=true;
 702                             break;
 703                         case HttpStatusCode.NotFound:
 704                             result=false;
 705                             break;
 706                         default:
 707                             throw CreateWebException("ContainerExists", client.StatusCode);
 708                     }
 709                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 710
 711                     return result;
 712                 }
 713
 714             }
 715         }
 716
 717         public bool ObjectExists(string account, string container, string objectName)
 718         {
 719             if (String.IsNullOrWhiteSpace(container))
 720                 throw new ArgumentNullException("container", "The container property can't be empty");
 721             if (String.IsNullOrWhiteSpace(objectName))
 722                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 723             Contract.EndContractBlock();
 724
 725             using (var client = new RestClient(_baseClient))
 726             {
 727                 if (!String.IsNullOrWhiteSpace(account))
 728                     client.BaseAddress = GetAccountUrl(account);
 729
 730                 client.Parameters.Clear();
 731                 client.Head(container + "/" + objectName, 3);
 732
 733                 switch (client.StatusCode)
 734                 {
 735                     case HttpStatusCode.OK:
 736                     case HttpStatusCode.NoContent:
 737                         return true;
 738                     case HttpStatusCode.NotFound:
 739                         return false;
 740                     default:
 741                         throw CreateWebException("ObjectExists", client.StatusCode);
 742                 }
 743             }
 744
 745         }
 746
 747         public ObjectInfo GetObjectInfo(string account, string container, string objectName)
 748         {
 749             if (String.IsNullOrWhiteSpace(container))
 750                 throw new ArgumentNullException("container", "The container property can't be empty");
 751             if (String.IsNullOrWhiteSpace(objectName))
 752                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 753             Contract.EndContractBlock();
 754
 755             using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
 756             {
 757
 758                 using (var client = new RestClient(_baseClient))
 759                 {
 760                     if (!String.IsNullOrWhiteSpace(account))
 761                         client.BaseAddress = GetAccountUrl(account);
 762                     try
 763                     {
 764                         client.Parameters.Clear();
 765
 766                         client.Head(container + "/" + objectName, 3);
 767
 768                         if (client.TimedOut)
 769                             return ObjectInfo.Empty;
 770
 771                         switch (client.StatusCode)
 772                         {
 773                             case HttpStatusCode.OK:
 774                             case HttpStatusCode.NoContent:
 775                                 var keys = client.ResponseHeaders.AllKeys.AsQueryable();
 776                                 var tags = client.GetMeta("X-Object-Meta-");
 777                                 var extensions = (from key in keys
 778                                                   where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
 779                                                   select new {Name = key, Value = client.ResponseHeaders[key]})
 780                                     .ToDictionary(t => t.Name, t => t.Value);
 781
 782                                 var permissions=client.GetHeaderValue("X-Object-Sharing", true);
 783
 784
 785                                 var info = new ObjectInfo
 786                                                {
 787                                                    Account = account,
 788                                                    Container = container,
 789                                                    Name = objectName,
 790                                                    Hash = client.GetHeaderValue("ETag"),
 791                                                    Content_Type = client.GetHeaderValue("Content-Type"),
 792                                                    Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),
 793                                                    Tags = tags,
 794                                                    Last_Modified = client.LastModified,
 795                                                    Extensions = extensions,
 796                                                    ContentEncoding=client.GetHeaderValue("Content-Encoding",true),
 797                                                    ContendDisposition = client.GetHeaderValue("Content-Disposition",true),
 798                                                    Manifest=client.GetHeaderValue("X-Object-Manifest",true),
 799                                                    PublicUrl=client.GetHeaderValue("X-Object-Public",true),
 800                                                    StorageUri=this.StorageUrl,
 801                                                };
 802                                 info.SetPermissions(permissions);
 803                                 return info;
 804                             case HttpStatusCode.NotFound:
 805                                 return ObjectInfo.Empty;
 806                             default:
 807                                 throw new WebException(
 808                                     String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 809                                                   objectName, client.StatusCode));
 810                         }
 811
 812                     }
 813                     catch (RetryException)
 814                     {
 815                         Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);
 816                         return ObjectInfo.Empty;
 817                     }
 818                     catch (WebException e)
 819                     {
 820                         Log.Error(
 821                             String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 822                                           objectName, client.StatusCode), e);
 823                         throw;
 824                     }
 825                 }
 826             }
 827
 828         }
 829
 830         public void CreateFolder(string account, string container, string folder)
 831         {
 832             if (String.IsNullOrWhiteSpace(container))
 833                 throw new ArgumentNullException("container", "The container property can't be empty");
 834             if (String.IsNullOrWhiteSpace(folder))
 835                 throw new ArgumentNullException("folder", "The folder property can't be empty");
 836             Contract.EndContractBlock();
 837
 838             var folderUrl=String.Format("{0}/{1}",container,folder);
 839             using (var client = new RestClient(_baseClient))
 840             {
 841                 if (!String.IsNullOrWhiteSpace(account))
 842                     client.BaseAddress = GetAccountUrl(account);
 843
 844                 client.Parameters.Clear();
 845                 client.Headers.Add("Content-Type", @"application/directory");
 846                 client.Headers.Add("Content-Length", "0");
 847                 client.PutWithRetry(folderUrl, 3);
 848
 849                 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
 850                     throw CreateWebException("CreateFolder", client.StatusCode);
 851             }
 852         }
 853
 854
 855
 856         public ContainerInfo GetContainerInfo(string account, string container)
 857         {
 858             if (String.IsNullOrWhiteSpace(container))
 859                 throw new ArgumentNullException("container", "The container property can't be empty");
 860             Contract.EndContractBlock();
 861
 862             using (var client = new RestClient(_baseClient))
 863             {
 864                 if (!String.IsNullOrWhiteSpace(account))
 865                     client.BaseAddress = GetAccountUrl(account);
 866
 867                 client.Head(container);
 868                 switch (client.StatusCode)
 869                 {
 870                     case HttpStatusCode.OK:
 871                     case HttpStatusCode.NoContent:
 872                         var tags = client.GetMeta("X-Container-Meta-");
 873                         var policies = client.GetMeta("X-Container-Policy-");
 874
 875                         var containerInfo = new ContainerInfo
 876                                                 {
 877                                                     Account=account,
 878                                                     Name = container,
 879                                                     StorageUrl=this.StorageUrl.ToString(),
 880                                                     Count =
 881                                                         long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
 882                                                     Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
 883                                                     BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
 884                                                     BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),
 885                                                     Last_Modified=client.LastModified,
 886                                                     Tags=tags,
 887                                                     Policies=policies
 888                                                 };
 889
 890
 891                         return containerInfo;
 892                     case HttpStatusCode.NotFound:
 893                         return ContainerInfo.Empty;
 894                     default:
 895                         throw CreateWebException("GetContainerInfo", client.StatusCode);
 896                 }
 897             }
 898         }
 899
 900         public void CreateContainer(string account, string container)
 901         {
 902             if (String.IsNullOrWhiteSpace(account))
 903                 throw new ArgumentNullException("account");
 904             if (String.IsNullOrWhiteSpace(container))
 905                 throw new ArgumentNullException("container");
 906             Contract.EndContractBlock();
 907
 908             using (var client = new RestClient(_baseClient))
 909             {
 910                 if (!String.IsNullOrWhiteSpace(account))
 911                     client.BaseAddress = GetAccountUrl(account);
 912
 913                 client.PutWithRetry(container, 3);
 914                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
 915                 if (!expectedCodes.Contains(client.StatusCode))
 916                     throw CreateWebException("CreateContainer", client.StatusCode);
 917             }
 918         }
 919
 920         public void DeleteContainer(string account, string container)
 921         {
 922             if (String.IsNullOrWhiteSpace(container))
 923                 throw new ArgumentNullException("container", "The container property can't be empty");
 924             Contract.EndContractBlock();
 925
 926             using (var client = new RestClient(_baseClient))
 927             {
 928                 if (!String.IsNullOrWhiteSpace(account))
 929                     client.BaseAddress = GetAccountUrl(account);
 930
 931                 client.DeleteWithRetry(container, 3);
 932                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
 933                 if (!expectedCodes.Contains(client.StatusCode))
 934                     throw CreateWebException("DeleteContainer", client.StatusCode);
 935             }
 936
 937         }
 938
 939         /// <summary>
 940         ///
 941         /// </summary>
 942         /// <param name="account"></param>
 943         /// <param name="container"></param>
 944         /// <param name="objectName"></param>
 945         /// <param name="fileName"></param>
 946         /// <returns></returns>
 947         /// <remarks>This method should have no timeout or a very long one</remarks>
 948         //Asynchronously download the object specified by *objectName* in a specific *container* to
 949         // a local file
 950         public Task GetObject(string account, string container, string objectName, string fileName)
 951         {
 952             if (String.IsNullOrWhiteSpace(container))
 953                 throw new ArgumentNullException("container", "The container property can't be empty");
 954             if (String.IsNullOrWhiteSpace(objectName))
 955                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 956             Contract.EndContractBlock();
 957
 958             try
 959             {
 960                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
 961                 //object to avoid concurrency errors.
 962                 //
 963                 //Download operations take a long time therefore they have no timeout.
 964                 var client = new RestClient(_baseClient) { Timeout = 0 };
 965                 if (!String.IsNullOrWhiteSpace(account))
 966                     client.BaseAddress = GetAccountUrl(account);
 967
 968                 //The container and objectName are relative names. They are joined with the client's
 969                 //BaseAddress to create the object's absolute address
 970                 var builder = client.GetAddressBuilder(container, objectName);
 971                 var uri = builder.Uri;
 972
 973                 //Download progress is reported to the Trace log
 974                 Log.InfoFormat("[GET] START {0}", objectName);
 975                 client.DownloadProgressChanged += (sender, args) =>
 976                     Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
 977                                     fileName, args.ProgressPercentage,
 978                                     args.BytesReceived,
 979                                     args.TotalBytesToReceive);
 980
 981
 982                 //Start downloading the object asynchronously
 983                 var downloadTask = client.DownloadFileTask(uri, fileName);
 984
 985                 //Once the download completes
 986                 return downloadTask.ContinueWith(download =>
 987                                       {
 988                                           //Delete the local client object
 989                                           client.Dispose();
 990                                           //And report failure or completion
 991                                           if (download.IsFaulted)
 992                                           {
 993                                               Log.ErrorFormat("[GET] FAIL for {0} with \r{1}", objectName,
 994                                                                download.Exception);
 995                                           }
 996                                           else
 997                                           {
 998                                               Log.InfoFormat("[GET] END {0}", objectName);
 999                                           }
1000                                       });
1001             }
1002             catch (Exception exc)
1003             {
1004                 Log.ErrorFormat("[GET] END {0} with {1}", objectName, exc);
1005                 throw;
1006             }
1007
1008
1009
1010         }
1011
1012         public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
1013         {
1014             if (String.IsNullOrWhiteSpace(container))
1015                 throw new ArgumentNullException("container");
1016             if (String.IsNullOrWhiteSpace(objectName))
1017                 throw new ArgumentNullException("objectName");
1018             if (hash==null)
1019                 throw new ArgumentNullException("hash");
1020             if (String.IsNullOrWhiteSpace(Token))
1021                 throw new InvalidOperationException("Invalid Token");
1022             if (StorageUrl == null)
1023                 throw new InvalidOperationException("Invalid Storage Url");
1024             Contract.EndContractBlock();
1025
1026
1027             //Don't use a timeout because putting the hashmap may be a long process
1028             var client = new RestClient(_baseClient) { Timeout = 0 };
1029             if (!String.IsNullOrWhiteSpace(account))
1030                 client.BaseAddress = GetAccountUrl(account);
1031
1032             //The container and objectName are relative names. They are joined with the client's
1033             //BaseAddress to create the object's absolute address
1034             var builder = client.GetAddressBuilder(container, objectName);
1035             builder.Query = "format=json&hashmap";
1036             var uri = builder.Uri;
1037
1038
1039             //Send the tree hash as Json to the server
1040             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
1041             var jsonHash = hash.ToJson();
1042             var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);
1043             if (Log.IsDebugEnabled)
1044                 Log.DebugFormat("Hashes:\r\n{0}", jsonHash);
1045             return uploadTask.ContinueWith(t =>
1046             {
1047
1048                 var empty = (IList<string>)new List<string>();
1049
1050
1051                 //The server will respond either with 201-created if all blocks were already on the server
1052                 if (client.StatusCode == HttpStatusCode.Created)
1053                 {
1054                     //in which case we return an empty hash list
1055                     return empty;
1056                 }
1057                 //or with a 409-conflict and return the list of missing parts
1058                 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
1059                 if (t.IsFaulted)
1060                 {
1061                     var ex = t.Exception.InnerException;
1062                     var we = ex as WebException;
1063                     var response = we.Response as HttpWebResponse;
1064                     if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
1065                     {
1066                         //In case of 409 the missing parts will be in the response content
1067                         using (var stream = response.GetResponseStream())
1068                         using(var reader=stream.GetLoggedReader(Log))
1069                         {
1070                             //We used to have to cleanup the content before returning it because it contains
1071                             //error content after the list of hashes
1072                             //
1073                             //As of 30/1/2012, the result is a proper Json array so we don't need to read the content
1074                             //line by line
1075
1076                             var serializer = new JsonSerializer();
1077                             serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);
1078                             var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));
1079                             return hashes;
1080                         }
1081                     }
1082                     //Any other status code is unexpected and the exception should be rethrown
1083                     Log.LogError(response);
1084                     throw ex;
1085
1086                 }
1087
1088                 //Any other status code is unexpected but there was no exception. We can probably continue processing
1089                 Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
1090
1091                 return empty;
1092             });
1093
1094         }
1095
1096
1097         public Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end)
1098         {
1099             if (String.IsNullOrWhiteSpace(Token))
1100                 throw new InvalidOperationException("Invalid Token");
1101             if (StorageUrl == null)
1102                 throw new InvalidOperationException("Invalid Storage Url");
1103             if (String.IsNullOrWhiteSpace(container))
1104                 throw new ArgumentNullException("container");
1105             if (relativeUrl== null)
1106                 throw new ArgumentNullException("relativeUrl");
1107             if (end.HasValue && end<0)
1108                 throw new ArgumentOutOfRangeException("end");
1109             if (start<0)
1110                 throw new ArgumentOutOfRangeException("start");
1111             Contract.EndContractBlock();
1112
1113
1114             //Don't use a timeout because putting the hashmap may be a long process
1115             var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
1116             if (!String.IsNullOrWhiteSpace(account))
1117                 client.BaseAddress = GetAccountUrl(account);
1118
1119             var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
1120             var uri = builder.Uri;
1121
1122             return client.DownloadDataTask(uri)
1123                 .ContinueWith(t=>
1124                                   {
1125                                       client.Dispose();
1126                                       return t.Result;
1127                                   });
1128         }
1129
1130
1131         public async Task PostBlock(string account, string container, byte[] block, int offset, int count)
1132         {
1133             if (String.IsNullOrWhiteSpace(container))
1134                 throw new ArgumentNullException("container");
1135             if (block == null)
1136                 throw new ArgumentNullException("block");
1137             if (offset < 0 || offset >= block.Length)
1138                 throw new ArgumentOutOfRangeException("offset");
1139             if (count < 0 || count > block.Length)
1140                 throw new ArgumentOutOfRangeException("count");
1141             if (String.IsNullOrWhiteSpace(Token))
1142                 throw new InvalidOperationException("Invalid Token");
1143             if (StorageUrl == null)
1144                 throw new InvalidOperationException("Invalid Storage Url");
1145             Contract.EndContractBlock();
1146
1147
1148             try
1149             {
1150
1151             //Don't use a timeout because putting the hashmap may be a long process
1152                 using (var client = new RestClient(_baseClient) { Timeout = 0 })
1153                 {
1154                     if (!String.IsNullOrWhiteSpace(account))
1155                         client.BaseAddress = GetAccountUrl(account);
1156
1157                     var builder = client.GetAddressBuilder(container, "");
1158                     //We are doing an update
1159                     builder.Query = "update";
1160                     var uri = builder.Uri;
1161
1162                     client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
1163
1164                     Log.InfoFormat("[BLOCK POST] START");
1165
1166                     client.UploadProgressChanged += (sender, args) =>
1167                                                     Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
1168                                                                    args.ProgressPercentage, args.BytesSent,
1169                                                                    args.TotalBytesToSend);
1170                     client.UploadFileCompleted += (sender, args) =>
1171                                                   Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
1172
1173                     var buffer = new byte[count];
1174                     Buffer.BlockCopy(block, offset, buffer, 0, count);
1175                     //Send the block
1176                     await client.UploadDataTask(uri, "POST", buffer);
1177                     Log.InfoFormat("[BLOCK POST] END");
1178                 }
1179             }
1180             catch (Exception exc)
1181             {
1182                 Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);
1183                 throw;
1184             }
1185         }
1186
1187
1188         public async Task<TreeHash> GetHashMap(string account, string container, string objectName)
1189         {
1190             if (String.IsNullOrWhiteSpace(container))
1191                 throw new ArgumentNullException("container");
1192             if (String.IsNullOrWhiteSpace(objectName))
1193                 throw new ArgumentNullException("objectName");
1194             if (String.IsNullOrWhiteSpace(Token))
1195                 throw new InvalidOperationException("Invalid Token");
1196             if (StorageUrl == null)
1197                 throw new InvalidOperationException("Invalid Storage Url");
1198             Contract.EndContractBlock();
1199
1200             try
1201             {
1202                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
1203                 //object to avoid concurrency errors.
1204                 //
1205                 //Download operations take a long time therefore they have no timeout.
1206                 //TODO: Do they really? this is a hashmap operation, not a download
1207
1208                 //Start downloading the object asynchronously
1209                 using (var client = new RestClient(_baseClient) { Timeout = 0 })
1210                 {
1211                     if (!String.IsNullOrWhiteSpace(account))
1212                         client.BaseAddress = GetAccountUrl(account);
1213
1214                     //The container and objectName are relative names. They are joined with the client's
1215                     //BaseAddress to create the object's absolute address
1216                     var builder = client.GetAddressBuilder(container, objectName);
1217                     builder.Query = "format=json&hashmap";
1218                     var uri = builder.Uri;
1219
1220
1221                     var json = await client.DownloadStringTaskAsync(uri);
1222                     var treeHash = TreeHash.Parse(json);
1223                     Log.InfoFormat("[GET HASH] END {0}", objectName);
1224                     return treeHash;
1225                 }
1226             }
1227             catch (Exception exc)
1228             {
1229                 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
1230                 throw;
1231             }
1232
1233         }
1234
1235
1236         /// <summary>
1237         ///
1238         /// </summary>
1239         /// <param name="account"></param>
1240         /// <param name="container"></param>
1241         /// <param name="objectName"></param>
1242         /// <param name="fileName"></param>
1243         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
1244         /// <remarks>>This method should have no timeout or a very long one</remarks>
1245         public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream")
1246         {
1247             if (String.IsNullOrWhiteSpace(container))
1248                 throw new ArgumentNullException("container", "The container property can't be empty");
1249             if (String.IsNullOrWhiteSpace(objectName))
1250                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1251             if (String.IsNullOrWhiteSpace(fileName))
1252                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
1253 /*
1254             if (!File.Exists(fileName) && !Directory.Exists(fileName))
1255                 throw new FileNotFoundException("The file or directory does not exist",fileName);
1256 */
1257
1258             try
1259             {
1260
1261                 using (var client = new RestClient(_baseClient) { Timeout = 0 })
1262                 {
1263                     if (!String.IsNullOrWhiteSpace(account))
1264                         client.BaseAddress = GetAccountUrl(account);
1265
1266                     var builder = client.GetAddressBuilder(container, objectName);
1267                     var uri = builder.Uri;
1268
1269                     string etag = hash ?? CalculateHash(fileName);
1270
1271                     client.Headers.Add("Content-Type", contentType);
1272                     client.Headers.Add("ETag", etag);
1273
1274
1275                     Log.InfoFormat("[PUT] START {0}", objectName);
1276                     client.UploadProgressChanged += (sender, args) =>
1277                                                         {
1278                                                             using (ThreadContext.Stacks["PUT"].Push("Progress"))
1279                                                             {
1280                                                                 Log.InfoFormat("{0} {1}% {2} of {3}", fileName,
1281                                                                                args.ProgressPercentage,
1282                                                                                args.BytesSent, args.TotalBytesToSend);
1283                                                             }
1284                                                         };
1285
1286                     client.UploadFileCompleted += (sender, args) =>
1287                                                       {
1288                                                           using (ThreadContext.Stacks["PUT"].Push("Progress"))
1289                                                           {
1290                                                               Log.InfoFormat("Completed {0}", fileName);
1291                                                           }
1292                                                       };
1293                     if (contentType=="application/directory")
1294                         await client.UploadDataTaskAsync(uri, "PUT", new byte[0]);
1295                     else
1296                         await client.UploadFileTaskAsync(uri, "PUT", fileName);
1297                 }
1298
1299                 Log.InfoFormat("[PUT] END {0}", objectName);
1300             }
1301             catch (Exception exc)
1302             {
1303                 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
1304                 throw;
1305             }
1306
1307         }
1308
1309
1310         private static string CalculateHash(string fileName)
1311         {
1312             Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
1313             Contract.EndContractBlock();
1314
1315             string hash;
1316             using (var hasher = MD5.Create())
1317             using(var stream=File.OpenRead(fileName))
1318             {
1319                 var hashBuilder=new StringBuilder();
1320                 foreach (byte b in hasher.ComputeHash(stream))
1321                     hashBuilder.Append(b.ToString("x2").ToLower());
1322                 hash = hashBuilder.ToString();
1323             }
1324             return hash;
1325         }
1326
1327         public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
1328         {
1329             if (String.IsNullOrWhiteSpace(sourceContainer))
1330                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1331             if (String.IsNullOrWhiteSpace(oldObjectName))
1332                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
1333             if (String.IsNullOrWhiteSpace(targetContainer))
1334                 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
1335             if (String.IsNullOrWhiteSpace(newObjectName))
1336                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
1337             Contract.EndContractBlock();
1338
1339             var targetUrl = targetContainer + "/" + newObjectName;
1340             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
1341
1342             using (var client = new RestClient(_baseClient))
1343             {
1344                 if (!String.IsNullOrWhiteSpace(account))
1345                     client.BaseAddress = GetAccountUrl(account);
1346
1347                 client.Headers.Add("X-Move-From", sourceUrl);
1348                 client.PutWithRetry(targetUrl, 3);
1349
1350                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1351                 if (!expectedCodes.Contains(client.StatusCode))
1352                     throw CreateWebException("MoveObject", client.StatusCode);
1353             }
1354         }
1355
1356         public void DeleteObject(string account, string sourceContainer, string objectName)
1357         {
1358             if (String.IsNullOrWhiteSpace(sourceContainer))
1359                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1360             if (String.IsNullOrWhiteSpace(objectName))
1361                 throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
1362             Contract.EndContractBlock();
1363
1364             var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
1365             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
1366
1367             using (var client = new RestClient(_baseClient))
1368             {
1369                 if (!String.IsNullOrWhiteSpace(account))
1370                     client.BaseAddress = GetAccountUrl(account);
1371
1372                 client.Headers.Add("X-Move-From", sourceUrl);
1373                 client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
1374                 Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);
1375                 client.PutWithRetry(targetUrl, 3);
1376
1377                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
1378                 if (!expectedCodes.Contains(client.StatusCode))
1379                     throw CreateWebException("DeleteObject", client.StatusCode);
1380             }
1381         }
1382
1383
1384         private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
1385         {
1386             return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
1387         }
1388
1389
1390 /*
1391         public IEnumerable<ObjectInfo> ListDirectories(ContainerInfo container)
1392         {
1393             var directories=this.ListObjects(container.Account, container.Name, "/");
1394         }
1395 */
1396
1397         public bool CanUpload(string account, ObjectInfo cloudFile)
1398         {
1399             Contract.Requires(!String.IsNullOrWhiteSpace(account));
1400             Contract.Requires(cloudFile!=null);
1401
1402             using (var client = new RestClient(_baseClient))
1403             {
1404                 if (!String.IsNullOrWhiteSpace(account))
1405                     client.BaseAddress = GetAccountUrl(account);
1406
1407
1408                 var parts = cloudFile.Name.Split('/');
1409                 var folder = String.Join("/", parts,0,parts.Length-1);
1410
1411                 var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid());
1412
1413                 client.Parameters.Clear();
1414                 try
1415                 {
1416                     client.PutWithRetry(fileUrl, 3, @"application/octet-stream");
1417
1418                     var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1419                     return (expectedCodes.Contains(client.StatusCode));
1420                 }
1421                 catch
1422                 {
1423                     return false;
1424                 }
1425                 finally
1426                 {
1427                     DeleteObject(account,cloudFile.Container,fileUrl);
1428                 }
1429             }
1430         }
1431     }
1432
1433     public class ShareAccountInfo
1434     {
1435         public DateTime? last_modified { get; set; }
1436         public string name { get; set; }
1437     }
1438 }