code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="CloudFilesClient.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42
  43 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
  44 //
  45 // The class provides methods to upload/download files, delete files, manage containers
  46
  47
  48 using System;
  49 using System.Collections.Generic;
  50 using System.Collections.Specialized;
  51 using System.ComponentModel.Composition;
  52 using System.Diagnostics;
  53 using System.Diagnostics.Contracts;
  54 using System.IO;
  55 using System.Linq;
  56 using System.Net;
  57 using System.Reflection;
  58 using System.Security.Cryptography;
  59 using System.Text;
  60 using System.Threading;
  61 using System.Threading.Tasks;
  62 using Newtonsoft.Json;
  63 using Pithos.Interfaces;
  64 using log4net;
  65
  66 namespace Pithos.Network
  67 {
  68     [Export(typeof(ICloudClient))]
  69     public class CloudFilesClient:ICloudClient
  70     {
  71         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  72
  73         //CloudFilesClient uses *_baseClient* internally to communicate with the server
  74         //RestClient provides a REST-friendly interface over the standard WebClient.
  75         private RestClient _baseClient;
  76
  77
  78         //During authentication the client provides a UserName
  79         public string UserName { get; set; }
  80
  81         //and and ApiKey to the server
  82         public string ApiKey { get; set; }
  83
  84         //And receives an authentication Token. This token must be provided in ALL other operations,
  85         //in the X-Auth-Token header
  86         private string _token;
  87         private readonly string _emptyGuid = Guid.Empty.ToString();
  88
  89
  90         public string Token
  91         {
  92             get { return _token; }
  93             set
  94             {
  95                 _token = value;
  96                 _baseClient.Headers["X-Auth-Token"] = value;
  97             }
  98         }
  99
 100         //The client also receives a StorageUrl after authentication. All subsequent operations must
 101         //use this url
 102         public Uri StorageUrl { get; set; }
 103
 104
 105         public Uri RootAddressUri { get; set; }
 106
 107        /* private WebProxy _proxy;
 108         public WebProxy Proxy
 109         {
 110             get { return _proxy; }
 111             set
 112             {
 113                 _proxy = value;
 114                 if (_baseClient != null)
 115                     _baseClient.Proxy = value;
 116             }
 117         }
 118 */
 119
 120         /* private Uri _proxy;
 121         public Uri Proxy
 122         {
 123             get { return _proxy; }
 124             set
 125             {
 126                 _proxy = value;
 127                 if (_baseClient != null)
 128                     _baseClient.Proxy = new WebProxy(value);
 129             }
 130         }*/
 131
 132         public double DownloadPercentLimit { get; set; }
 133         public double UploadPercentLimit { get; set; }
 134
 135         public string AuthenticationUrl { get; set; }
 136
 137
 138         public string VersionPath
 139         {
 140             get { return UsePithos ? "v1" : "v1.0"; }
 141         }
 142
 143         public bool UsePithos { get; set; }
 144
 145
 146
 147         public CloudFilesClient(string userName, string apiKey)
 148         {
 149             UserName = userName;
 150             ApiKey = apiKey;
 151         }
 152
 153         public CloudFilesClient(AccountInfo accountInfo)
 154         {
 155             if (accountInfo==null)
 156                 throw new ArgumentNullException("accountInfo");
 157             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
 158             Contract.Ensures(StorageUrl != null);
 159             Contract.Ensures(_baseClient != null);
 160             Contract.Ensures(RootAddressUri != null);
 161             Contract.EndContractBlock();
 162
 163             _baseClient = new RestClient
 164             {
 165                 BaseAddress = accountInfo.StorageUri.ToString(),
 166                 Timeout = 30000,
 167                 Retries = 3,
 168             };
 169             StorageUrl = accountInfo.StorageUri;
 170             Token = accountInfo.Token;
 171             UserName = accountInfo.UserName;
 172
 173             //Get the root address (StorageUrl without the account)
 174             var storageUrl = StorageUrl.AbsoluteUri;
 175             var usernameIndex = storageUrl.LastIndexOf(UserName);
 176             var rootUrl = storageUrl.Substring(0, usernameIndex);
 177             RootAddressUri = new Uri(rootUrl);
 178         }
 179
 180
 181         public AccountInfo Authenticate()
 182         {
 183             if (String.IsNullOrWhiteSpace(UserName))
 184                 throw new InvalidOperationException("UserName is empty");
 185             if (String.IsNullOrWhiteSpace(ApiKey))
 186                 throw new InvalidOperationException("ApiKey is empty");
 187             if (String.IsNullOrWhiteSpace(AuthenticationUrl))
 188                 throw new InvalidOperationException("AuthenticationUrl is empty");
 189             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
 190             Contract.Ensures(StorageUrl != null);
 191             Contract.Ensures(_baseClient != null);
 192             Contract.Ensures(RootAddressUri != null);
 193             Contract.EndContractBlock();
 194
 195
 196             Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
 197
 198             var groups = new List<Group>();
 199
 200             using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
 201             {
 202                /* if (Proxy != null)
 203                     authClient.Proxy = Proxy;*/
 204
 205                 Contract.Assume(authClient.Headers!=null);
 206
 207                 authClient.Headers.Add("X-Auth-User", UserName);
 208                 authClient.Headers.Add("X-Auth-Key", ApiKey);
 209                 //TODO: Remove after testing. Added to overcome server auth bug
 210                 //authClient.Headers.Add("X-Auth-Token", ApiKey);
 211
 212                 authClient.DownloadStringWithRetry(VersionPath, 3);
 213
 214                 authClient.AssertStatusOK("Authentication failed");
 215
 216                 var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
 217                 if (String.IsNullOrWhiteSpace(storageUrl))
 218                     throw new InvalidOperationException("Failed to obtain storage url");
 219
 220                 _baseClient = new RestClient
 221                 {
 222                     BaseAddress = storageUrl,
 223                     Timeout = 10000,
 224                     Retries = 3,
 225                     //Proxy=Proxy
 226                 };
 227
 228                 StorageUrl = new Uri(storageUrl);
 229
 230                 //Get the root address (StorageUrl without the account)
 231                 var usernameIndex=storageUrl.LastIndexOf(UserName);
 232                 var rootUrl = storageUrl.Substring(0, usernameIndex);
 233                 RootAddressUri = new Uri(rootUrl);
 234
 235                 var token = authClient.GetHeaderValue("X-Auth-Token");
 236                 if (String.IsNullOrWhiteSpace(token))
 237                     throw new InvalidOperationException("Failed to obtain token url");
 238                 Token = token;
 239
 240                /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
 241                 groups = (from key in keys
 242                             where key.StartsWith("X-Account-Group-")
 243                             let name = key.Substring(16)
 244                             select new Group(name, authClient.ResponseHeaders[key]))
 245                         .ToList();
 246
 247 */
 248             }
 249
 250             Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
 251             Debug.Assert(_baseClient!=null);
 252
 253             return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};
 254
 255         }
 256
 257
 258
 259         public IList<ContainerInfo> ListContainers(string account)
 260         {
 261             using (var client = new RestClient(_baseClient))
 262             {
 263                 if (!String.IsNullOrWhiteSpace(account))
 264                     client.BaseAddress = GetAccountUrl(account);
 265
 266                 client.Parameters.Clear();
 267                 client.Parameters.Add("format", "json");
 268                 var content = client.DownloadStringWithRetry("", 3);
 269                 client.AssertStatusOK("List Containers failed");
 270
 271                 if (client.StatusCode == HttpStatusCode.NoContent)
 272                     return new List<ContainerInfo>();
 273                 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
 274
 275                 foreach (var info in infos)
 276                 {
 277                     info.Account = account;
 278                 }
 279                 return infos;
 280             }
 281
 282         }
 283
 284         private string GetAccountUrl(string account)
 285         {
 286             return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
 287         }
 288
 289         public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
 290         {
 291             using (ThreadContext.Stacks["Share"].Push("List Accounts"))
 292             {
 293                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 294
 295                 using (var client = new RestClient(_baseClient))
 296                 {
 297                     client.Parameters.Clear();
 298                     client.Parameters.Add("format", "json");
 299                     client.IfModifiedSince = since;
 300
 301                     //Extract the username from the base address
 302                     client.BaseAddress = RootAddressUri.AbsoluteUri;
 303
 304                     var content = client.DownloadStringWithRetry(@"", 3);
 305
 306                     client.AssertStatusOK("ListSharingAccounts failed");
 307
 308                     //If the result is empty, return an empty list,
 309                     var infos = String.IsNullOrWhiteSpace(content)
 310                                     ? new List<ShareAccountInfo>()
 311                                 //Otherwise deserialize the account list into a list of ShareAccountInfos
 312                                     : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
 313
 314                     Log.DebugFormat("END");
 315                     return infos;
 316                 }
 317             }
 318         }
 319
 320
 321         /// <summary>
 322         /// Request listing of all objects in a container modified since a specific time.
 323         /// If the *since* value is missing, return all objects
 324         /// </summary>
 325         /// <param name="knownContainers">Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new
 326         /// and should be polled anyway
 327         /// </param>
 328         /// <param name="since"></param>
 329         /// <returns></returns>
 330         public IList<ObjectInfo> ListSharedObjects(HashSet<string> knownContainers,DateTime? since = null )
 331         {
 332
 333             using (ThreadContext.Stacks["Share"].Push("List Objects"))
 334             {
 335                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 336                 //'since' is not used here because we need to have ListObjects return a NoChange result
 337                 //for all shared accounts,containers
 338
 339                 Func<ContainerInfo, string> GetKey = c => String.Format("{0}\\{1}", c.Account, c.Name);
 340
 341                 var accounts = ListSharingAccounts();
 342                 var containers = (from account in accounts
 343                                  let conts = ListContainers(account.name)
 344                                  from container in conts
 345                                  select container).ToList();
 346                 var items = from container in containers
 347                             let actualSince=knownContainers.Contains(GetKey(container))?since:null
 348                             select ListObjects(container.Account , container.Name,  actualSince);
 349                 var objects=items.SelectMany(r=> r).ToList();
 350
 351                 //For each object
 352                 //Check parents recursively up to (but not including) the container.
 353                 //If parents are missing, add them to the list
 354                 //Need function to calculate all parent URLs
 355                 objects = AddMissingParents(objects);
 356
 357                 //Store any new containers
 358                 foreach (var container in containers)
 359                 {
 360                     knownContainers.Add(GetKey(container));
 361                 }
 362
 363
 364
 365                 if (Log.IsDebugEnabled) Log.DebugFormat("END");
 366                 return objects;
 367             }
 368         }
 369
 370         private List<ObjectInfo> AddMissingParents(List<ObjectInfo> objects)
 371         {
 372             //TODO: Remove short-circuit when we decide to use Missing Parents functionality
 373             //return objects;
 374
 375             var existingUris = objects.ToDictionary(o => o.Uri, o => o);
 376             foreach (var objectInfo in objects)
 377             {
 378                 //Can be null when retrieving objects to show in selective sync
 379                 if (objectInfo.Name == null)
 380                     continue;
 381
 382                 var parts = objectInfo.Name.Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries);
 383                 //If there is no parent, skip
 384                 if (parts.Length == 1)
 385                     continue;
 386                 var baseParts = new[]
 387                                   {
 388                                       objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container
 389                                   };
 390                 for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++)
 391                 {
 392                     var nameparts = parts.Range(0, partIdx).ToArray();
 393                     var parentName= String.Join("/", nameparts);
 394
 395                     var parentParts = baseParts.Concat(nameparts);
 396                     var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts);
 397
 398                     var parentUri = new Uri(parentUrl, UriKind.Absolute);
 399
 400                     ObjectInfo existingInfo;
 401                     if (!existingUris.TryGetValue(parentUri,out existingInfo))
 402                     {
 403                         var h = parentUrl.GetHashCode();
 404                         var reverse = new string(parentUrl.Reverse().ToArray());
 405                         var rh = reverse.GetHashCode();
 406                         var b1 = BitConverter.GetBytes(h);
 407                         var b2 = BitConverter.GetBytes(rh);
 408                         var g = new Guid(0,0,0,b1.Concat(b2).ToArray());
 409
 410
 411                         existingUris[parentUri] = new ObjectInfo
 412                                                       {
 413                                                           Account = objectInfo.Account,
 414                                                           Container = objectInfo.Container,
 415                                                           Content_Type = @"application/directory",
 416                                                           ETag = Signature.MD5_EMPTY,
 417                                                           X_Object_Hash = Signature.MERKLE_EMPTY,
 418                                                           Name=parentName,
 419                                                           StorageUri=objectInfo.StorageUri,
 420                                                           Bytes = 0,
 421                                                           UUID=g.ToString(),
 422                                                       };
 423                     }
 424                 }
 425             }
 426             return existingUris.Values.ToList();
 427         }
 428
 429         public void SetTags(ObjectInfo target,IDictionary<string,string> tags)
 430         {
 431             if (String.IsNullOrWhiteSpace(Token))
 432                 throw new InvalidOperationException("The Token is not set");
 433             if (StorageUrl == null)
 434                 throw new InvalidOperationException("The StorageUrl is not set");
 435             if (target == null)
 436                 throw new ArgumentNullException("target");
 437             Contract.EndContractBlock();
 438
 439             using (ThreadContext.Stacks["Share"].Push("Share Object"))
 440             {
 441                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 442
 443                 using (var client = new RestClient(_baseClient))
 444                 {
 445
 446                     client.BaseAddress = GetAccountUrl(target.Account);
 447
 448                     client.Parameters.Clear();
 449                     client.Parameters.Add("update", "");
 450
 451                     foreach (var tag in tags)
 452                     {
 453                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
 454                         client.Headers.Add(headerTag, tag.Value);
 455                     }
 456
 457                     client.DownloadStringWithRetry(target.Container, 3);
 458
 459
 460                     client.AssertStatusOK("SetTags failed");
 461                     //If the status is NOT ACCEPTED we have a problem
 462                     if (client.StatusCode != HttpStatusCode.Accepted)
 463                     {
 464                         Log.Error("Failed to set tags");
 465                         throw new Exception("Failed to set tags");
 466                     }
 467
 468                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 469                 }
 470             }
 471
 472
 473         }
 474
 475         public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
 476         {
 477             if (String.IsNullOrWhiteSpace(Token))
 478                 throw new InvalidOperationException("The Token is not set");
 479             if (StorageUrl==null)
 480                 throw new InvalidOperationException("The StorageUrl is not set");
 481             if (String.IsNullOrWhiteSpace(container))
 482                 throw new ArgumentNullException("container");
 483             if (String.IsNullOrWhiteSpace(objectName))
 484                 throw new ArgumentNullException("objectName");
 485             if (String.IsNullOrWhiteSpace(account))
 486                 throw new ArgumentNullException("account");
 487             if (String.IsNullOrWhiteSpace(shareTo))
 488                 throw new ArgumentNullException("shareTo");
 489             Contract.EndContractBlock();
 490
 491             using (ThreadContext.Stacks["Share"].Push("Share Object"))
 492             {
 493                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 494
 495                 using (var client = new RestClient(_baseClient))
 496                 {
 497
 498                     client.BaseAddress = GetAccountUrl(account);
 499
 500                     client.Parameters.Clear();
 501                     client.Parameters.Add("format", "json");
 502
 503                     string permission = "";
 504                     if (write)
 505                         permission = String.Format("write={0}", shareTo);
 506                     else if (read)
 507                         permission = String.Format("read={0}", shareTo);
 508                     client.Headers.Add("X-Object-Sharing", permission);
 509
 510                     var content = client.DownloadStringWithRetry(container, 3);
 511
 512                     client.AssertStatusOK("ShareObject failed");
 513
 514                     //If the result is empty, return an empty list,
 515                     var infos = String.IsNullOrWhiteSpace(content)
 516                                     ? new List<ObjectInfo>()
 517                                 //Otherwise deserialize the object list into a list of ObjectInfos
 518                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 519
 520                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 521                 }
 522             }
 523
 524
 525         }
 526
 527         public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
 528         {
 529             if (accountInfo==null)
 530                 throw new ArgumentNullException("accountInfo");
 531             Contract.EndContractBlock();
 532
 533             using (ThreadContext.Stacks["Account"].Push("GetPolicies"))
 534             {
 535                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 536
 537                 using (var client = new RestClient(_baseClient))
 538                 {
 539                     if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
 540                         client.BaseAddress = GetAccountUrl(accountInfo.UserName);
 541
 542                     client.Parameters.Clear();
 543                     client.Parameters.Add("format", "json");
 544                     client.Head(String.Empty, 3);
 545
 546                     var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
 547                     var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
 548
 549                     long quota, bytes;
 550                     if (long.TryParse(quotaValue, out quota))
 551                         accountInfo.Quota = quota;
 552                     if (long.TryParse(bytesValue, out bytes))
 553                         accountInfo.BytesUsed = bytes;
 554
 555                     return accountInfo;
 556
 557                 }
 558
 559             }
 560         }
 561
 562         public void UpdateMetadata(ObjectInfo objectInfo)
 563         {
 564             if (objectInfo == null)
 565                 throw new ArgumentNullException("objectInfo");
 566             Contract.EndContractBlock();
 567
 568             using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
 569             {
 570                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 571
 572
 573                 using(var client=new RestClient(_baseClient))
 574                 {
 575
 576                     client.BaseAddress = GetAccountUrl(objectInfo.Account);
 577
 578                     client.Parameters.Clear();
 579
 580
 581                     //Set Tags
 582                     foreach (var tag in objectInfo.Tags)
 583                     {
 584                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
 585                         client.Headers.Add(headerTag, tag.Value);
 586                     }
 587
 588                     //Set Permissions
 589
 590                     var permissions=objectInfo.GetPermissionString();
 591                     client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);
 592
 593                     client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
 594                     client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
 595                     client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
 596                     var isPublic = objectInfo.IsPublic.ToString().ToLower();
 597                     client.Headers.Add("X-Object-Public", isPublic);
 598
 599
 600                     /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);
 601                     uriBuilder.Query = "update=";
 602                     var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/
 603                     var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);
 604                     client.PostWithRetry(address,"application/xml");
 605
 606                     //client.UploadValues(uri,new NameValueCollection());
 607
 608
 609                     client.AssertStatusOK("UpdateMetadata failed");
 610                     //If the status is NOT ACCEPTED or OK we have a problem
 611                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
 612                     {
 613                         Log.Error("Failed to update metadata");
 614                         throw new Exception("Failed to update metadata");
 615                     }
 616
 617                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 618                 }
 619             }
 620
 621         }
 622
 623         public void UpdateMetadata(ContainerInfo containerInfo)
 624         {
 625             if (containerInfo == null)
 626                 throw new ArgumentNullException("containerInfo");
 627             Contract.EndContractBlock();
 628
 629             using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
 630             {
 631                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 632
 633
 634                 using(var client=new RestClient(_baseClient))
 635                 {
 636
 637                     client.BaseAddress = GetAccountUrl(containerInfo.Account);
 638
 639                     client.Parameters.Clear();
 640
 641
 642                     //Set Tags
 643                     foreach (var tag in containerInfo.Tags)
 644                     {
 645                         var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
 646                         client.Headers.Add(headerTag, tag.Value);
 647                     }
 648
 649
 650                     //Set Policies
 651                     foreach (var policy in containerInfo.Policies)
 652                     {
 653                         var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
 654                         client.Headers.Add(headerPolicy, policy.Value);
 655                     }
 656
 657
 658                     var uriBuilder = client.GetAddressBuilder(containerInfo.Name,"");
 659                     var uri = uriBuilder.Uri;
 660
 661                     client.UploadValues(uri,new NameValueCollection());
 662
 663
 664                     client.AssertStatusOK("UpdateMetadata failed");
 665                     //If the status is NOT ACCEPTED or OK we have a problem
 666                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
 667                     {
 668                         Log.Error("Failed to update metadata");
 669                         throw new Exception("Failed to update metadata");
 670                     }
 671
 672                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 673                 }
 674             }
 675
 676         }
 677
 678
 679         public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
 680         {
 681             if (String.IsNullOrWhiteSpace(container))
 682                 throw new ArgumentNullException("container");
 683             Contract.EndContractBlock();
 684
 685             using (ThreadContext.Stacks["Objects"].Push("List"))
 686             {
 687                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 688
 689                 using (var client = new RestClient(_baseClient))
 690                 {
 691                     if (!String.IsNullOrWhiteSpace(account))
 692                         client.BaseAddress = GetAccountUrl(account);
 693
 694                     client.Parameters.Clear();
 695                     client.Parameters.Add("format", "json");
 696                     client.IfModifiedSince = since;
 697                     var content = client.DownloadStringWithRetry(container, 3);
 698
 699                     client.AssertStatusOK("ListObjects failed");
 700
 701                     if (client.StatusCode==HttpStatusCode.NotModified)
 702                         return new[]{new NoModificationInfo(account,container)};
 703                     //If the result is empty, return an empty list,
 704                     var infos = String.IsNullOrWhiteSpace(content)
 705                                     ? new List<ObjectInfo>()
 706                                 //Otherwise deserialize the object list into a list of ObjectInfos
 707                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 708
 709                     foreach (var info in infos)
 710                     {
 711                         info.Container = container;
 712                         info.Account = account;
 713                         info.StorageUri = this.StorageUrl;
 714                     }
 715                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 716                     return infos;
 717                 }
 718             }
 719         }
 720
 721         public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
 722         {
 723             if (String.IsNullOrWhiteSpace(container))
 724                 throw new ArgumentNullException("container");
 725 /*
 726             if (String.IsNullOrWhiteSpace(folder))
 727                 throw new ArgumentNullException("folder");
 728 */
 729             Contract.EndContractBlock();
 730
 731             using (ThreadContext.Stacks["Objects"].Push("List"))
 732             {
 733                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 734
 735                 using (var client = new RestClient(_baseClient))
 736                 {
 737                     if (!String.IsNullOrWhiteSpace(account))
 738                         client.BaseAddress = GetAccountUrl(account);
 739
 740                     client.Parameters.Clear();
 741                     client.Parameters.Add("format", "json");
 742                     client.Parameters.Add("path", folder);
 743                     client.IfModifiedSince = since;
 744                     var content = client.DownloadStringWithRetry(container, 3);
 745                     client.AssertStatusOK("ListObjects failed");
 746
 747                     if (client.StatusCode==HttpStatusCode.NotModified)
 748                         return new[]{new NoModificationInfo(account,container,folder)};
 749
 750                     var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
 751                     foreach (var info in infos)
 752                     {
 753                         info.Account = account;
 754                         if (info.Container == null)
 755                             info.Container = container;
 756                         info.StorageUri = this.StorageUrl;
 757                     }
 758                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 759                     return infos;
 760                 }
 761             }
 762         }
 763
 764
 765         public bool ContainerExists(string account, string container)
 766         {
 767             if (String.IsNullOrWhiteSpace(container))
 768                 throw new ArgumentNullException("container", "The container property can't be empty");
 769             Contract.EndContractBlock();
 770
 771             using (ThreadContext.Stacks["Containters"].Push("Exists"))
 772             {
 773                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
 774
 775                 using (var client = new RestClient(_baseClient))
 776                 {
 777                     if (!String.IsNullOrWhiteSpace(account))
 778                         client.BaseAddress = GetAccountUrl(account);
 779
 780                     client.Parameters.Clear();
 781                     client.Head(container, 3);
 782
 783                     bool result;
 784                     switch (client.StatusCode)
 785                     {
 786                         case HttpStatusCode.OK:
 787                         case HttpStatusCode.NoContent:
 788                             result=true;
 789                             break;
 790                         case HttpStatusCode.NotFound:
 791                             result=false;
 792                             break;
 793                         default:
 794                             throw CreateWebException("ContainerExists", client.StatusCode);
 795                     }
 796                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
 797
 798                     return result;
 799                 }
 800
 801             }
 802         }
 803
 804         public bool ObjectExists(string account, string container, string objectName)
 805         {
 806             if (String.IsNullOrWhiteSpace(container))
 807                 throw new ArgumentNullException("container", "The container property can't be empty");
 808             if (String.IsNullOrWhiteSpace(objectName))
 809                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 810             Contract.EndContractBlock();
 811
 812             using (var client = new RestClient(_baseClient))
 813             {
 814                 if (!String.IsNullOrWhiteSpace(account))
 815                     client.BaseAddress = GetAccountUrl(account);
 816
 817                 client.Parameters.Clear();
 818                 client.Head(container + "/" + objectName, 3);
 819
 820                 switch (client.StatusCode)
 821                 {
 822                     case HttpStatusCode.OK:
 823                     case HttpStatusCode.NoContent:
 824                         return true;
 825                     case HttpStatusCode.NotFound:
 826                         return false;
 827                     default:
 828                         throw CreateWebException("ObjectExists", client.StatusCode);
 829                 }
 830             }
 831
 832         }
 833
 834         public ObjectInfo GetObjectInfo(string account, string container, string objectName)
 835         {
 836             if (String.IsNullOrWhiteSpace(container))
 837                 throw new ArgumentNullException("container", "The container property can't be empty");
 838             if (String.IsNullOrWhiteSpace(objectName))
 839                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
 840             Contract.EndContractBlock();
 841
 842             using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
 843             {
 844
 845                 using (var client = new RestClient(_baseClient))
 846                 {
 847                     if (!String.IsNullOrWhiteSpace(account))
 848                         client.BaseAddress = GetAccountUrl(account);
 849                     try
 850                     {
 851                         client.Parameters.Clear();
 852
 853                         client.Head(container + "/" + objectName, 3);
 854
 855                         if (client.TimedOut)
 856                             return ObjectInfo.Empty;
 857
 858                         switch (client.StatusCode)
 859                         {
 860                             case HttpStatusCode.OK:
 861                             case HttpStatusCode.NoContent:
 862                                 var keys = client.ResponseHeaders.AllKeys.AsQueryable();
 863                                 var tags = client.GetMeta("X-Object-Meta-");
 864                                 var extensions = (from key in keys
 865                                                   where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
 866                                                   select new {Name = key, Value = client.ResponseHeaders[key]})
 867                                     .ToDictionary(t => t.Name, t => t.Value);
 868
 869                                 var permissions=client.GetHeaderValue("X-Object-Sharing", true);
 870
 871
 872                                 var info = new ObjectInfo
 873                                                {
 874                                                    Account = account,
 875                                                    Container = container,
 876                                                    Name = objectName,
 877                                                    ETag = client.GetHeaderValue("ETag"),
 878                                                    UUID=client.GetHeaderValue("X-Object-UUID"),
 879                                                    X_Object_Hash = client.GetHeaderValue("X-Object-Hash"),
 880                                                    Content_Type = client.GetHeaderValue("Content-Type"),
 881                                                    Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),
 882                                                    Tags = tags,
 883                                                    Last_Modified = client.LastModified,
 884                                                    Extensions = extensions,
 885                                                    ContentEncoding=client.GetHeaderValue("Content-Encoding",true),
 886                                                    ContendDisposition = client.GetHeaderValue("Content-Disposition",true),
 887                                                    Manifest=client.GetHeaderValue("X-Object-Manifest",true),
 888                                                    PublicUrl=client.GetHeaderValue("X-Object-Public",true),
 889                                                    StorageUri=this.StorageUrl,
 890                                                };
 891                                 info.SetPermissions(permissions);
 892                                 return info;
 893                             case HttpStatusCode.NotFound:
 894                                 return ObjectInfo.Empty;
 895                             default:
 896                                 throw new WebException(
 897                                     String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 898                                                   objectName, client.StatusCode));
 899                         }
 900
 901                     }
 902                     catch (RetryException)
 903                     {
 904                         Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);
 905                         return ObjectInfo.Empty;
 906                     }
 907                     catch (WebException e)
 908                     {
 909                         Log.Error(
 910                             String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
 911                                           objectName, client.StatusCode), e);
 912                         throw;
 913                     }
 914                 }
 915             }
 916
 917         }
 918
 919         public void CreateFolder(string account, string container, string folder)
 920         {
 921             if (String.IsNullOrWhiteSpace(container))
 922                 throw new ArgumentNullException("container", "The container property can't be empty");
 923             if (String.IsNullOrWhiteSpace(folder))
 924                 throw new ArgumentNullException("folder", "The folder property can't be empty");
 925             Contract.EndContractBlock();
 926
 927             var folderUrl=String.Format("{0}/{1}",container,folder);
 928             using (var client = new RestClient(_baseClient))
 929             {
 930                 if (!String.IsNullOrWhiteSpace(account))
 931                     client.BaseAddress = GetAccountUrl(account);
 932
 933                 client.Parameters.Clear();
 934                 client.Headers.Add("Content-Type", @"application/directory");
 935                 client.Headers.Add("Content-Length", "0");
 936                 client.PutWithRetry(folderUrl, 3);
 937
 938                 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
 939                     throw CreateWebException("CreateFolder", client.StatusCode);
 940             }
 941         }
 942
 943
 944
 945         public ContainerInfo GetContainerInfo(string account, string container)
 946         {
 947             if (String.IsNullOrWhiteSpace(container))
 948                 throw new ArgumentNullException("container", "The container property can't be empty");
 949             Contract.EndContractBlock();
 950
 951             using (var client = new RestClient(_baseClient))
 952             {
 953                 if (!String.IsNullOrWhiteSpace(account))
 954                     client.BaseAddress = GetAccountUrl(account);
 955
 956                 client.Head(container);
 957                 switch (client.StatusCode)
 958                 {
 959                     case HttpStatusCode.OK:
 960                     case HttpStatusCode.NoContent:
 961                         var tags = client.GetMeta("X-Container-Meta-");
 962                         var policies = client.GetMeta("X-Container-Policy-");
 963
 964                         var containerInfo = new ContainerInfo
 965                                                 {
 966                                                     Account=account,
 967                                                     Name = container,
 968                                                     StorageUrl=this.StorageUrl.ToString(),
 969                                                     Count =
 970                                                         long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
 971                                                     Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
 972                                                     BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
 973                                                     BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),
 974                                                     Last_Modified=client.LastModified,
 975                                                     Tags=tags,
 976                                                     Policies=policies
 977                                                 };
 978
 979
 980                         return containerInfo;
 981                     case HttpStatusCode.NotFound:
 982                         return ContainerInfo.Empty;
 983                     default:
 984                         throw CreateWebException("GetContainerInfo", client.StatusCode);
 985                 }
 986             }
 987         }
 988
 989         public void CreateContainer(string account, string container)
 990         {
 991             if (String.IsNullOrWhiteSpace(account))
 992                 throw new ArgumentNullException("account");
 993             if (String.IsNullOrWhiteSpace(container))
 994                 throw new ArgumentNullException("container");
 995             Contract.EndContractBlock();
 996
 997             using (var client = new RestClient(_baseClient))
 998             {
 999                 if (!String.IsNullOrWhiteSpace(account))
1000                     client.BaseAddress = GetAccountUrl(account);
1001
1002                 client.PutWithRetry(container, 3);
1003                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
1004                 if (!expectedCodes.Contains(client.StatusCode))
1005                     throw CreateWebException("CreateContainer", client.StatusCode);
1006             }
1007         }
1008
1009         public void DeleteContainer(string account, string container)
1010         {
1011             if (String.IsNullOrWhiteSpace(container))
1012                 throw new ArgumentNullException("container", "The container property can't be empty");
1013             Contract.EndContractBlock();
1014
1015             using (var client = new RestClient(_baseClient))
1016             {
1017                 if (!String.IsNullOrWhiteSpace(account))
1018                     client.BaseAddress = GetAccountUrl(account);
1019
1020                 client.DeleteWithRetry(container, 3);
1021                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
1022                 if (!expectedCodes.Contains(client.StatusCode))
1023                     throw CreateWebException("DeleteContainer", client.StatusCode);
1024             }
1025
1026         }
1027
1028         /// <summary>
1029         ///
1030         /// </summary>
1031         /// <param name="account"></param>
1032         /// <param name="container"></param>
1033         /// <param name="objectName"></param>
1034         /// <param name="fileName"></param>
1035         /// <returns></returns>
1036         /// <remarks>This method should have no timeout or a very long one</remarks>
1037         //Asynchronously download the object specified by *objectName* in a specific *container* to
1038         // a local file
1039         public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken)
1040         {
1041             if (String.IsNullOrWhiteSpace(container))
1042                 throw new ArgumentNullException("container", "The container property can't be empty");
1043             if (String.IsNullOrWhiteSpace(objectName))
1044                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1045             Contract.EndContractBlock();
1046
1047             try
1048             {
1049                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
1050                 //object to avoid concurrency errors.
1051                 //
1052                 //Download operations take a long time therefore they have no timeout.
1053                 using(var client = new RestClient(_baseClient) { Timeout = 0 })
1054                 {
1055                     if (!String.IsNullOrWhiteSpace(account))
1056                         client.BaseAddress = GetAccountUrl(account);
1057
1058                     //The container and objectName are relative names. They are joined with the client's
1059                     //BaseAddress to create the object's absolute address
1060                     var builder = client.GetAddressBuilder(container, objectName);
1061                     var uri = builder.Uri;
1062
1063                     //Download progress is reported to the Trace log
1064                     Log.InfoFormat("[GET] START {0}", objectName);
1065                     /*client.DownloadProgressChanged += (sender, args) =>
1066                                                       Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
1067                                                                      fileName, args.ProgressPercentage,
1068                                                                      args.BytesReceived,
1069                                                                      args.TotalBytesToReceive);*/
1070                     var progress = new Progress<DownloadProgressChangedEventArgs>(args =>
1071                                 {
1072                                     Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
1073                                                    fileName, args.ProgressPercentage,
1074                                                    args.BytesReceived,
1075                                                    args.TotalBytesToReceive);
1076                                     if (DownloadProgressChanged!=null)
1077                                         DownloadProgressChanged(this, args);
1078                                 });
1079
1080                     //Start downloading the object asynchronously
1081                     await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false);
1082
1083                     //Once the download completes
1084                     //Delete the local client object
1085                 }
1086                 //And report failure or completion
1087             }
1088             catch (Exception exc)
1089             {
1090                 Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);
1091                 throw;
1092             }
1093
1094             Log.InfoFormat("[GET] END {0}", objectName);
1095
1096
1097         }
1098
1099         public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
1100         {
1101             if (String.IsNullOrWhiteSpace(container))
1102                 throw new ArgumentNullException("container");
1103             if (String.IsNullOrWhiteSpace(objectName))
1104                 throw new ArgumentNullException("objectName");
1105             if (hash==null)
1106                 throw new ArgumentNullException("hash");
1107             if (String.IsNullOrWhiteSpace(Token))
1108                 throw new InvalidOperationException("Invalid Token");
1109             if (StorageUrl == null)
1110                 throw new InvalidOperationException("Invalid Storage Url");
1111             Contract.EndContractBlock();
1112
1113
1114             //Don't use a timeout because putting the hashmap may be a long process
1115             var client = new RestClient(_baseClient) { Timeout = 0 };
1116             if (!String.IsNullOrWhiteSpace(account))
1117                 client.BaseAddress = GetAccountUrl(account);
1118
1119             //The container and objectName are relative names. They are joined with the client's
1120             //BaseAddress to create the object's absolute address
1121             var builder = client.GetAddressBuilder(container, objectName);
1122             builder.Query = "format=json&hashmap";
1123             var uri = builder.Uri;
1124
1125
1126             //Send the tree hash as Json to the server
1127             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
1128             var jsonHash = hash.ToJson();
1129
1130             client.Headers.Add("ETag",hash.MD5);
1131             var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);
1132             if (Log.IsDebugEnabled)
1133                 Log.DebugFormat("Hashes:\r\n{0}", jsonHash);
1134             return uploadTask.ContinueWith(t =>
1135             {
1136
1137                 var empty = (IList<string>)new List<string>();
1138
1139
1140                 //The server will respond either with 201-created if all blocks were already on the server
1141                 if (client.StatusCode == HttpStatusCode.Created)
1142                 {
1143                     //in which case we return an empty hash list
1144                     return empty;
1145                 }
1146                 //or with a 409-conflict and return the list of missing parts
1147                 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
1148                 if (t.IsFaulted)
1149                 {
1150                     var ex = t.Exception.InnerException;
1151                     var we = ex as WebException;
1152                     var response = we.Response as HttpWebResponse;
1153                     if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
1154                     {
1155                         //In case of 409 the missing parts will be in the response content
1156                         using (var stream = response.GetResponseStream())
1157                         using(var reader=stream.GetLoggedReader(Log))
1158                         {
1159                             //We used to have to cleanup the content before returning it because it contains
1160                             //error content after the list of hashes
1161                             //
1162                             //As of 30/1/2012, the result is a proper Json array so we don't need to read the content
1163                             //line by line
1164
1165                             var serializer = new JsonSerializer();
1166                             serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);
1167                             var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));
1168                             return hashes;
1169                         }
1170                     }
1171                     //Any other status code is unexpected and the exception should be rethrown
1172                     Log.LogError(response);
1173                     throw ex;
1174
1175                 }
1176
1177                 //Any other status code is unexpected but there was no exception. We can probably continue processing
1178                 Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
1179
1180                 return empty;
1181             });
1182
1183         }
1184
1185
1186         public async Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)
1187         {
1188             if (String.IsNullOrWhiteSpace(Token))
1189                 throw new InvalidOperationException("Invalid Token");
1190             if (StorageUrl == null)
1191                 throw new InvalidOperationException("Invalid Storage Url");
1192             if (String.IsNullOrWhiteSpace(container))
1193                 throw new ArgumentNullException("container");
1194             if (relativeUrl == null)
1195                 throw new ArgumentNullException("relativeUrl");
1196             if (end.HasValue && end < 0)
1197                 throw new ArgumentOutOfRangeException("end");
1198             if (start < 0)
1199                 throw new ArgumentOutOfRangeException("start");
1200             Contract.EndContractBlock();
1201
1202             //Don't use a timeout because putting the hashmap may be a long process
1203             using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end})
1204             {
1205                 if (!String.IsNullOrWhiteSpace(account))
1206                     client.BaseAddress = GetAccountUrl(account);
1207
1208                 var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
1209                 var uri = builder.Uri;
1210
1211 /*                client.DownloadProgressChanged += (sender, args) =>
1212                                                       {
1213                                                           Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
1214                                                                           uri.Segments.Last(), args.ProgressPercentage,
1215                                                                           args.BytesReceived,
1216                                                                           args.TotalBytesToReceive);
1217                                                           DownloadProgressChanged(sender, args);
1218                                                       };*/
1219                 var progress = new Progress<DownloadProgressChangedEventArgs>(args =>
1220                 {
1221                     Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
1222                                     uri.Segments.Last(), args.ProgressPercentage,
1223                                     args.BytesReceived,
1224                                     args.TotalBytesToReceive);
1225                     if (DownloadProgressChanged!=null)
1226                         DownloadProgressChanged(this, args);
1227                 });
1228
1229
1230                 var result = await client.DownloadDataTaskAsync(uri, cancellationToken,progress).ConfigureAwait(false);
1231                 return result;
1232             }
1233         }
1234
1235         public event UploadProgressChangedEventHandler UploadProgressChanged;
1236         public event DownloadProgressChangedEventHandler DownloadProgressChanged;
1237
1238         public async Task PostBlock(string account, string container, byte[] block, int offset, int count,CancellationToken token)
1239         {
1240             if (String.IsNullOrWhiteSpace(container))
1241                 throw new ArgumentNullException("container");
1242             if (block == null)
1243                 throw new ArgumentNullException("block");
1244             if (offset < 0 || offset >= block.Length)
1245                 throw new ArgumentOutOfRangeException("offset");
1246             if (count < 0 || count > block.Length)
1247                 throw new ArgumentOutOfRangeException("count");
1248             if (String.IsNullOrWhiteSpace(Token))
1249                 throw new InvalidOperationException("Invalid Token");
1250             if (StorageUrl == null)
1251                 throw new InvalidOperationException("Invalid Storage Url");
1252             Contract.EndContractBlock();
1253
1254
1255             try
1256             {
1257
1258                 //Don't use a timeout because putting the hashmap may be a long process
1259                 using (var client = new RestClient(_baseClient) { Timeout = 0 })
1260                 {
1261                     if (!String.IsNullOrWhiteSpace(account))
1262                         client.BaseAddress = GetAccountUrl(account);
1263
1264                     token.Register(client.CancelAsync);
1265
1266                     var builder = client.GetAddressBuilder(container, "");
1267                     //We are doing an update
1268                     builder.Query = "update";
1269                     var uri = builder.Uri;
1270
1271                     client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
1272
1273                     Log.InfoFormat("[BLOCK POST] START");
1274
1275 /*
1276                     client.UploadProgressChanged += (sender, args) =>
1277                                                         {
1278                                                             Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
1279                                                                            args.ProgressPercentage, args.BytesSent,
1280                                                                            args.TotalBytesToSend);
1281                                                             UploadProgressChanged(sender, args);
1282                                                         };
1283 */
1284                     client.UploadFileCompleted += (sender, args) =>
1285                                                   Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
1286
1287                     var progress=new Progress<UploadProgressChangedEventArgs>(args=>
1288                     {
1289                         Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
1290                                         args.ProgressPercentage, args.BytesSent,
1291                                         args.TotalBytesToSend);
1292                         if (UploadProgressChanged!=null)
1293                             UploadProgressChanged(this, args);
1294                     });
1295                     var buffer = new byte[count];
1296                     Buffer.BlockCopy(block, offset, buffer, 0, count);
1297                     //Send the block
1298                     await client.UploadDataTaskAsync(uri, "POST", buffer,token,progress).ConfigureAwait(false);
1299                     Log.InfoFormat("[BLOCK POST] END");
1300                 }
1301             }
1302             catch (TaskCanceledException )
1303             {
1304                 Log.Info("Aborting block");
1305                 throw;
1306             }
1307             catch (Exception exc)
1308             {
1309                 Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);
1310                 throw;
1311             }
1312         }
1313
1314
1315         public async Task<TreeHash> GetHashMap(string account, string container, string objectName)
1316         {
1317             if (String.IsNullOrWhiteSpace(container))
1318                 throw new ArgumentNullException("container");
1319             if (String.IsNullOrWhiteSpace(objectName))
1320                 throw new ArgumentNullException("objectName");
1321             if (String.IsNullOrWhiteSpace(Token))
1322                 throw new InvalidOperationException("Invalid Token");
1323             if (StorageUrl == null)
1324                 throw new InvalidOperationException("Invalid Storage Url");
1325             Contract.EndContractBlock();
1326
1327             try
1328             {
1329                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
1330                 //object to avoid concurrency errors.
1331                 //
1332                 //Download operations take a long time therefore they have no timeout.
1333                 //TODO: Do they really? this is a hashmap operation, not a download
1334
1335                 //Start downloading the object asynchronously
1336                 using (var client = new RestClient(_baseClient) { Timeout = 0 })
1337                 {
1338                     if (!String.IsNullOrWhiteSpace(account))
1339                         client.BaseAddress = GetAccountUrl(account);
1340
1341                     //The container and objectName are relative names. They are joined with the client's
1342                     //BaseAddress to create the object's absolute address
1343                     var builder = client.GetAddressBuilder(container, objectName);
1344                     builder.Query = "format=json&hashmap";
1345                     var uri = builder.Uri;
1346
1347
1348                     var json = await client.DownloadStringTaskAsync(uri).ConfigureAwait(false);
1349                     var treeHash = TreeHash.Parse(json);
1350                     Log.InfoFormat("[GET HASH] END {0}", objectName);
1351                     return treeHash;
1352                 }
1353             }
1354             catch (Exception exc)
1355             {
1356                 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
1357                 throw;
1358             }
1359
1360         }
1361
1362
1363         /// <summary>
1364         ///
1365         /// </summary>
1366         /// <param name="account"></param>
1367         /// <param name="container"></param>
1368         /// <param name="objectName"></param>
1369         /// <param name="fileName"></param>
1370         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
1371         /// <remarks>>This method should have no timeout or a very long one</remarks>
1372         public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream")
1373         {
1374             if (String.IsNullOrWhiteSpace(container))
1375                 throw new ArgumentNullException("container", "The container property can't be empty");
1376             if (String.IsNullOrWhiteSpace(objectName))
1377                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1378             if (String.IsNullOrWhiteSpace(fileName))
1379                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
1380 /*
1381             if (!File.Exists(fileName) && !Directory.Exists(fileName))
1382                 throw new FileNotFoundException("The file or directory does not exist",fileName);
1383 */
1384
1385             try
1386             {
1387
1388                 using (var client = new RestClient(_baseClient) { Timeout = 0 })
1389                 {
1390                     if (!String.IsNullOrWhiteSpace(account))
1391                         client.BaseAddress = GetAccountUrl(account);
1392
1393                     var builder = client.GetAddressBuilder(container, objectName);
1394                     var uri = builder.Uri;
1395
1396                     string etag = hash ?? CalculateHash(fileName);
1397
1398                     client.Headers.Add("Content-Type", contentType);
1399                     client.Headers.Add("ETag", etag);
1400
1401
1402                     Log.InfoFormat("[PUT] START {0}", objectName);
1403                     client.UploadProgressChanged += (sender, args) =>
1404                                                         {
1405                                                             using (ThreadContext.Stacks["PUT"].Push("Progress"))
1406                                                             {
1407                                                                 Log.InfoFormat("{0} {1}% {2} of {3}", fileName,
1408                                                                                args.ProgressPercentage,
1409                                                                                args.BytesSent, args.TotalBytesToSend);
1410                                                             }
1411                                                         };
1412
1413                     client.UploadFileCompleted += (sender, args) =>
1414                                                       {
1415                                                           using (ThreadContext.Stacks["PUT"].Push("Progress"))
1416                                                           {
1417                                                               Log.InfoFormat("Completed {0}", fileName);
1418                                                           }
1419                                                       };
1420                     if (contentType=="application/directory")
1421                         await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false);
1422                     else
1423                         await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false);
1424                 }
1425
1426                 Log.InfoFormat("[PUT] END {0}", objectName);
1427             }
1428             catch (Exception exc)
1429             {
1430                 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
1431                 throw;
1432             }
1433
1434         }
1435
1436
1437         private static string CalculateHash(string fileName)
1438         {
1439             Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
1440             Contract.EndContractBlock();
1441
1442             string hash;
1443             using (var hasher = MD5.Create())
1444             using(var stream=File.OpenRead(fileName))
1445             {
1446                 var hashBuilder=new StringBuilder();
1447                 foreach (byte b in hasher.ComputeHash(stream))
1448                     hashBuilder.Append(b.ToString("x2").ToLower());
1449                 hash = hashBuilder.ToString();
1450             }
1451             return hash;
1452         }
1453
1454         public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
1455         {
1456             if (String.IsNullOrWhiteSpace(sourceContainer))
1457                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1458             if (String.IsNullOrWhiteSpace(oldObjectName))
1459                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
1460             if (String.IsNullOrWhiteSpace(targetContainer))
1461                 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
1462             if (String.IsNullOrWhiteSpace(newObjectName))
1463                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
1464             Contract.EndContractBlock();
1465
1466             var targetUrl = targetContainer + "/" + newObjectName;
1467             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
1468
1469             using (var client = new RestClient(_baseClient))
1470             {
1471                 if (!String.IsNullOrWhiteSpace(account))
1472                     client.BaseAddress = GetAccountUrl(account);
1473
1474                 client.Headers.Add("X-Move-From", sourceUrl);
1475                 client.PutWithRetry(targetUrl, 3);
1476
1477                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1478                 if (!expectedCodes.Contains(client.StatusCode))
1479                     throw CreateWebException("MoveObject", client.StatusCode);
1480             }
1481         }
1482
1483         public void DeleteObject(string account, string sourceContainer, string objectName, bool isDirectory)
1484         {
1485             if (String.IsNullOrWhiteSpace(sourceContainer))
1486                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1487             if (String.IsNullOrWhiteSpace(objectName))
1488                 throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
1489             Contract.EndContractBlock();
1490
1491             var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
1492 /*
1493             if (isDirectory)
1494                 targetUrl = targetUrl + "?delimiter=/";
1495 */
1496
1497             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
1498
1499             using (var client = new RestClient(_baseClient))
1500             {
1501                 if (!String.IsNullOrWhiteSpace(account))
1502                     client.BaseAddress = GetAccountUrl(account);
1503
1504                 client.Headers.Add("X-Move-From", sourceUrl);
1505                 client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
1506                 Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);
1507                 client.PutWithRetry(targetUrl, 3);
1508
1509                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
1510                 if (!expectedCodes.Contains(client.StatusCode))
1511                     throw CreateWebException("DeleteObject", client.StatusCode);
1512             }
1513         }
1514
1515
1516         private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
1517         {
1518             return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
1519         }
1520
1521
1522 /*
1523         public IEnumerable<ObjectInfo> ListDirectories(ContainerInfo container)
1524         {
1525             var directories=this.ListObjects(container.Account, container.Name, "/");
1526         }
1527 */
1528
1529         public bool CanUpload(string account, ObjectInfo cloudFile)
1530         {
1531             Contract.Requires(!String.IsNullOrWhiteSpace(account));
1532             Contract.Requires(cloudFile!=null);
1533
1534             using (var client = new RestClient(_baseClient))
1535             {
1536                 if (!String.IsNullOrWhiteSpace(account))
1537                     client.BaseAddress = GetAccountUrl(account);
1538
1539
1540                 var parts = cloudFile.Name.Split('/');
1541                 var folder = String.Join("/", parts,0,parts.Length-1);
1542
1543                 var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid());
1544
1545                 client.Parameters.Clear();
1546                 try
1547                 {
1548                     client.PutWithRetry(fileUrl, 3, @"application/octet-stream");
1549
1550                     var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1551                     var result=(expectedCodes.Contains(client.StatusCode));
1552                     DeleteObject(account, cloudFile.Container, fileUrl, cloudFile.IsDirectory);
1553                     return result;
1554                 }
1555                 catch
1556                 {
1557                     return false;
1558                 }
1559             }
1560         }
1561     }
1562
1563     public class ShareAccountInfo
1564     {
1565         public DateTime? last_modified { get; set; }
1566         public string name { get; set; }
1567     }
1568 }