code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="CloudFilesClient.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 \r
  43 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos\r
  44 //\r
  45 // The class provides methods to upload/download files, delete files, manage containers\r
  46 \r
  47 \r
  48 using System;\r
  49 using System.Collections.Generic;\r
  50 using System.Collections.Specialized;\r
  51 using System.ComponentModel.Composition;\r
  52 using System.Diagnostics;\r
  53 using System.Diagnostics.Contracts;\r
  54 using System.IO;\r
  55 using System.Linq;\r
  56 using System.Net;\r
  57 using System.Reflection;\r
  58 using System.Security.Cryptography;\r
  59 using System.Text;\r
  60 using System.Threading;\r
  61 using System.Threading.Tasks;\r
  62 using Newtonsoft.Json;\r
  63 using Pithos.Interfaces;\r
  64 using log4net;\r
  65 \r
  66 namespace Pithos.Network\r
  67 {\r
  68     [Export(typeof(ICloudClient))]\r
  69     public class CloudFilesClient:ICloudClient\r
  70     {\r
  71         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  72 \r
  73         //CloudFilesClient uses *_baseClient* internally to communicate with the server\r
  74         //RestClient provides a REST-friendly interface over the standard WebClient.\r
  75         private RestClient _baseClient;\r
  76         \r
  77 \r
  78         //During authentication the client provides a UserName \r
  79         public string UserName { get; set; }\r
  80         \r
  81         //and and ApiKey to the server\r
  82         public string ApiKey { get; set; }\r
  83         \r
  84         //And receives an authentication Token. This token must be provided in ALL other operations,\r
  85         //in the X-Auth-Token header\r
  86         private string _token;\r
  87         private readonly string _emptyGuid = Guid.Empty.ToString();\r
  88 \r
  89 \r
  90         public string Token\r
  91         {\r
  92             get { return _token; }\r
  93             set\r
  94             {\r
  95                 _token = value;\r
  96                 _baseClient.Headers["X-Auth-Token"] = value;\r
  97             }\r
  98         }\r
  99 \r
 100         //The client also receives a StorageUrl after authentication. All subsequent operations must\r
 101         //use this url\r
 102         public Uri StorageUrl { get; set; }\r
 103 \r
 104 \r
 105         public Uri RootAddressUri { get; set; }\r
 106 \r
 107        /* private WebProxy _proxy;\r
 108         public WebProxy Proxy\r
 109         {\r
 110             get { return _proxy; }\r
 111             set\r
 112             {\r
 113                 _proxy = value;\r
 114                 if (_baseClient != null)\r
 115                     _baseClient.Proxy = value;                \r
 116             }\r
 117         }\r
 118 */\r
 119 \r
 120         /* private Uri _proxy;\r
 121         public Uri Proxy\r
 122         {\r
 123             get { return _proxy; }\r
 124             set\r
 125             {\r
 126                 _proxy = value;\r
 127                 if (_baseClient != null)\r
 128                     _baseClient.Proxy = new WebProxy(value);                \r
 129             }\r
 130         }*/\r
 131 \r
 132         public double DownloadPercentLimit { get; set; }\r
 133         public double UploadPercentLimit { get; set; }\r
 134 \r
 135         public string AuthenticationUrl { get; set; }\r
 136 \r
 137  \r
 138         public string VersionPath\r
 139         {\r
 140             get { return UsePithos ? "v1" : "v1.0"; }\r
 141         }\r
 142 \r
 143         public bool UsePithos { get; set; }\r
 144 \r
 145 \r
 146 \r
 147         public CloudFilesClient(string userName, string apiKey)\r
 148         {\r
 149             UserName = userName;\r
 150             ApiKey = apiKey;\r
 151         }\r
 152 \r
 153         public CloudFilesClient(AccountInfo accountInfo)\r
 154         {\r
 155             if (accountInfo==null)\r
 156                 throw new ArgumentNullException("accountInfo");\r
 157             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));\r
 158             Contract.Ensures(StorageUrl != null);\r
 159             Contract.Ensures(_baseClient != null);\r
 160             Contract.Ensures(RootAddressUri != null);\r
 161             Contract.EndContractBlock();          \r
 162 \r
 163             _baseClient = new RestClient\r
 164             {\r
 165                 BaseAddress = accountInfo.StorageUri.ToString(),\r
 166                 Timeout = 30000,\r
 167                 Retries = 3,\r
 168             };\r
 169             StorageUrl = accountInfo.StorageUri;\r
 170             Token = accountInfo.Token;\r
 171             UserName = accountInfo.UserName;\r
 172 \r
 173             //Get the root address (StorageUrl without the account)\r
 174             var storageUrl = StorageUrl.AbsoluteUri;\r
 175             var usernameIndex = storageUrl.LastIndexOf(UserName);\r
 176             var rootUrl = storageUrl.Substring(0, usernameIndex);\r
 177             RootAddressUri = new Uri(rootUrl);\r
 178         }\r
 179 \r
 180 \r
 181         public AccountInfo Authenticate()\r
 182         {\r
 183             if (String.IsNullOrWhiteSpace(UserName))\r
 184                 throw new InvalidOperationException("UserName is empty");\r
 185             if (String.IsNullOrWhiteSpace(ApiKey))\r
 186                 throw new InvalidOperationException("ApiKey is empty");\r
 187             if (String.IsNullOrWhiteSpace(AuthenticationUrl))\r
 188                 throw new InvalidOperationException("AuthenticationUrl is empty");\r
 189             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));\r
 190             Contract.Ensures(StorageUrl != null);\r
 191             Contract.Ensures(_baseClient != null);\r
 192             Contract.Ensures(RootAddressUri != null);\r
 193             Contract.EndContractBlock();\r
 194 \r
 195 \r
 196             Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);\r
 197 \r
 198             var groups = new List<Group>();\r
 199 \r
 200             using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})\r
 201             {                \r
 202                /* if (Proxy != null)\r
 203                     authClient.Proxy = Proxy;*/\r
 204 \r
 205                 Contract.Assume(authClient.Headers!=null);\r
 206 \r
 207                 authClient.Headers.Add("X-Auth-User", UserName);\r
 208                 authClient.Headers.Add("X-Auth-Key", ApiKey);\r
 209                 //TODO: Remove after testing. Added to overcome server auth bug\r
 210                 //authClient.Headers.Add("X-Auth-Token", ApiKey);\r
 211 \r
 212                 authClient.DownloadStringWithRetry(VersionPath, 3);\r
 213 \r
 214                 authClient.AssertStatusOK("Authentication failed");\r
 215 \r
 216                 var storageUrl = authClient.GetHeaderValue("X-Storage-Url");\r
 217                 if (String.IsNullOrWhiteSpace(storageUrl))\r
 218                     throw new InvalidOperationException("Failed to obtain storage url");\r
 219                 \r
 220                 _baseClient = new RestClient\r
 221                 {\r
 222                     BaseAddress = storageUrl,\r
 223                     Timeout = 10000,\r
 224                     Retries = 3,\r
 225                     //Proxy=Proxy\r
 226                 };\r
 227 \r
 228                 StorageUrl = new Uri(storageUrl);\r
 229                 \r
 230                 //Get the root address (StorageUrl without the account)\r
 231                 var usernameIndex=storageUrl.LastIndexOf(UserName);\r
 232                 var rootUrl = storageUrl.Substring(0, usernameIndex);\r
 233                 RootAddressUri = new Uri(rootUrl);\r
 234                 \r
 235                 var token = authClient.GetHeaderValue("X-Auth-Token");\r
 236                 if (String.IsNullOrWhiteSpace(token))\r
 237                     throw new InvalidOperationException("Failed to obtain token url");\r
 238                 Token = token;\r
 239 \r
 240                /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();\r
 241                 groups = (from key in keys\r
 242                             where key.StartsWith("X-Account-Group-")\r
 243                             let name = key.Substring(16)\r
 244                             select new Group(name, authClient.ResponseHeaders[key]))\r
 245                         .ToList();\r
 246                     \r
 247 */\r
 248             }\r
 249 \r
 250             Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);\r
 251             Debug.Assert(_baseClient!=null);\r
 252 \r
 253             return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};            \r
 254 \r
 255         }\r
 256 \r
 257 \r
 258 \r
 259         public IList<ContainerInfo> ListContainers(string account)\r
 260         {\r
 261             using (var client = new RestClient(_baseClient))\r
 262             {\r
 263                 if (!String.IsNullOrWhiteSpace(account))\r
 264                     client.BaseAddress = GetAccountUrl(account);\r
 265                 \r
 266                 client.Parameters.Clear();\r
 267                 client.Parameters.Add("format", "json");\r
 268                 var content = client.DownloadStringWithRetry("", 3);\r
 269                 client.AssertStatusOK("List Containers failed");\r
 270 \r
 271                 if (client.StatusCode == HttpStatusCode.NoContent)\r
 272                     return new List<ContainerInfo>();\r
 273                 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);\r
 274                 \r
 275                 foreach (var info in infos)\r
 276                 {\r
 277                     info.Account = account;\r
 278                 }\r
 279                 return infos;\r
 280             }\r
 281 \r
 282         }\r
 283 \r
 284         private string GetAccountUrl(string account)\r
 285         {\r
 286             return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;\r
 287         }\r
 288 \r
 289         public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)\r
 290         {\r
 291             using (ThreadContext.Stacks["Share"].Push("List Accounts"))\r
 292             {\r
 293                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 294 \r
 295                 using (var client = new RestClient(_baseClient))\r
 296                 {\r
 297                     client.Parameters.Clear();\r
 298                     client.Parameters.Add("format", "json");\r
 299                     client.IfModifiedSince = since;\r
 300 \r
 301                     //Extract the username from the base address\r
 302                     client.BaseAddress = RootAddressUri.AbsoluteUri; \r
 303 \r
 304                     var content = client.DownloadStringWithRetry(@"", 3);\r
 305 \r
 306                     client.AssertStatusOK("ListSharingAccounts failed");\r
 307 \r
 308                     //If the result is empty, return an empty list,\r
 309                     var infos = String.IsNullOrWhiteSpace(content)\r
 310                                     ? new List<ShareAccountInfo>()\r
 311                                 //Otherwise deserialize the account list into a list of ShareAccountInfos\r
 312                                     : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);\r
 313 \r
 314                     Log.DebugFormat("END");\r
 315                     return infos;\r
 316                 }\r
 317             }\r
 318         }\r
 319 \r
 320 \r
 321         /// <summary>\r
 322         /// Request listing of all objects in a container modified since a specific time.\r
 323         /// If the *since* value is missing, return all objects\r
 324         /// </summary>\r
 325         /// <param name="knownContainers">Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new\r
 326         /// and should be polled anyway\r
 327         /// </param>\r
 328         /// <param name="since"></param>\r
 329         /// <returns></returns>\r
 330         public IList<ObjectInfo> ListSharedObjects(HashSet<string> knownContainers,DateTime? since = null )\r
 331         {\r
 332 \r
 333             using (ThreadContext.Stacks["Share"].Push("List Objects"))\r
 334             {\r
 335                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 336                 //'since' is not used here because we need to have ListObjects return a NoChange result\r
 337                 //for all shared accounts,containers\r
 338 \r
 339                 Func<ContainerInfo, string> GetKey = c => String.Format("{0}\\{1}", c.Account, c.Name);\r
 340 \r
 341                 var accounts = ListSharingAccounts();\r
 342                 var containers = (from account in accounts\r
 343                                  let conts = ListContainers(account.name)\r
 344                                  from container in conts\r
 345                                  select container).ToList();                \r
 346                 var items = from container in containers \r
 347                             let actualSince=knownContainers.Contains(GetKey(container))?since:null\r
 348                             select ListObjects(container.Account , container.Name,  actualSince);\r
 349                 var objects=items.SelectMany(r=> r).ToList();\r
 350 \r
 351                 //For each object\r
 352                 //Check parents recursively up to (but not including) the container.\r
 353                 //If parents are missing, add them to the list\r
 354                 //Need function to calculate all parent URLs\r
 355                 objects = AddMissingParents(objects);\r
 356                 \r
 357                 //Store any new containers\r
 358                 foreach (var container in containers)\r
 359                 {\r
 360                     knownContainers.Add(GetKey(container));\r
 361                 }\r
 362 \r
 363 \r
 364 \r
 365                 if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 366                 return objects;\r
 367             }\r
 368         }\r
 369 \r
 370         private List<ObjectInfo> AddMissingParents(List<ObjectInfo> objects)\r
 371         {\r
 372             //TODO: Remove short-circuit when we decide to use Missing Parents functionality\r
 373             //return objects;\r
 374 \r
 375             var existingUris = objects.ToDictionary(o => o.Uri, o => o);\r
 376             foreach (var objectInfo in objects)\r
 377             {\r
 378                 //Can be null when retrieving objects to show in selective sync\r
 379                 if (objectInfo.Name == null)\r
 380                     continue;\r
 381 \r
 382                 var parts = objectInfo.Name.Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries);\r
 383                 //If there is no parent, skip\r
 384                 if (parts.Length == 1)\r
 385                     continue;\r
 386                 var baseParts = new[]\r
 387                                   {\r
 388                                       objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container\r
 389                                   };\r
 390                 for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++)\r
 391                 {\r
 392                     var nameparts = parts.Range(0, partIdx).ToArray();\r
 393                     var parentName= String.Join("/", nameparts);\r
 394 \r
 395                     var parentParts = baseParts.Concat(nameparts);\r
 396                     var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts);\r
 397                     \r
 398                     var parentUri = new Uri(parentUrl, UriKind.Absolute);\r
 399 \r
 400                     ObjectInfo existingInfo;\r
 401                     if (!existingUris.TryGetValue(parentUri,out existingInfo))\r
 402                     {\r
 403                         var h = parentUrl.GetHashCode();\r
 404                         var reverse = new string(parentUrl.Reverse().ToArray());\r
 405                         var rh = reverse.GetHashCode();\r
 406                         var b1 = BitConverter.GetBytes(h);\r
 407                         var b2 = BitConverter.GetBytes(rh);\r
 408                         var g = new Guid(0,0,0,b1.Concat(b2).ToArray());\r
 409                         \r
 410 \r
 411                         existingUris[parentUri] = new ObjectInfo\r
 412                                                       {\r
 413                                                           Account = objectInfo.Account,\r
 414                                                           Container = objectInfo.Container,\r
 415                                                           Content_Type = @"application/directory",\r
 416                                                           ETag = Signature.MD5_EMPTY,\r
 417                                                           X_Object_Hash = Signature.MERKLE_EMPTY,\r
 418                                                           Name=parentName,\r
 419                                                           StorageUri=objectInfo.StorageUri,\r
 420                                                           Bytes = 0,\r
 421                                                           UUID=g.ToString(),                                                          \r
 422                                                       };\r
 423                     }\r
 424                 }\r
 425             }\r
 426             return existingUris.Values.ToList();\r
 427         }\r
 428 \r
 429         public void SetTags(ObjectInfo target,IDictionary<string,string> tags)\r
 430         {\r
 431             if (String.IsNullOrWhiteSpace(Token))\r
 432                 throw new InvalidOperationException("The Token is not set");\r
 433             if (StorageUrl == null)\r
 434                 throw new InvalidOperationException("The StorageUrl is not set");\r
 435             if (target == null)\r
 436                 throw new ArgumentNullException("target");\r
 437             Contract.EndContractBlock();\r
 438 \r
 439             using (ThreadContext.Stacks["Share"].Push("Share Object"))\r
 440             {\r
 441                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 442 \r
 443                 using (var client = new RestClient(_baseClient))\r
 444                 {\r
 445 \r
 446                     client.BaseAddress = GetAccountUrl(target.Account);\r
 447 \r
 448                     client.Parameters.Clear();\r
 449                     client.Parameters.Add("update", "");\r
 450 \r
 451                     foreach (var tag in tags)\r
 452                     {\r
 453                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);\r
 454                         client.Headers.Add(headerTag, tag.Value);\r
 455                     }\r
 456                     \r
 457                     client.DownloadStringWithRetry(target.Container, 3);\r
 458 \r
 459                     \r
 460                     client.AssertStatusOK("SetTags failed");\r
 461                     //If the status is NOT ACCEPTED we have a problem\r
 462                     if (client.StatusCode != HttpStatusCode.Accepted)\r
 463                     {\r
 464                         Log.Error("Failed to set tags");\r
 465                         throw new Exception("Failed to set tags");\r
 466                     }\r
 467 \r
 468                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 469                 }\r
 470             }\r
 471 \r
 472 \r
 473         }\r
 474 \r
 475         public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)\r
 476         {\r
 477             if (String.IsNullOrWhiteSpace(Token))\r
 478                 throw new InvalidOperationException("The Token is not set");\r
 479             if (StorageUrl==null)\r
 480                 throw new InvalidOperationException("The StorageUrl is not set");\r
 481             if (String.IsNullOrWhiteSpace(container))\r
 482                 throw new ArgumentNullException("container");\r
 483             if (String.IsNullOrWhiteSpace(objectName))\r
 484                 throw new ArgumentNullException("objectName");\r
 485             if (String.IsNullOrWhiteSpace(account))\r
 486                 throw new ArgumentNullException("account");\r
 487             if (String.IsNullOrWhiteSpace(shareTo))\r
 488                 throw new ArgumentNullException("shareTo");\r
 489             Contract.EndContractBlock();\r
 490 \r
 491             using (ThreadContext.Stacks["Share"].Push("Share Object"))\r
 492             {\r
 493                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 494                 \r
 495                 using (var client = new RestClient(_baseClient))\r
 496                 {\r
 497 \r
 498                     client.BaseAddress = GetAccountUrl(account);\r
 499 \r
 500                     client.Parameters.Clear();\r
 501                     client.Parameters.Add("format", "json");\r
 502 \r
 503                     string permission = "";\r
 504                     if (write)\r
 505                         permission = String.Format("write={0}", shareTo);\r
 506                     else if (read)\r
 507                         permission = String.Format("read={0}", shareTo);\r
 508                     client.Headers.Add("X-Object-Sharing", permission);\r
 509 \r
 510                     var content = client.DownloadStringWithRetry(container, 3);\r
 511 \r
 512                     client.AssertStatusOK("ShareObject failed");\r
 513 \r
 514                     //If the result is empty, return an empty list,\r
 515                     var infos = String.IsNullOrWhiteSpace(content)\r
 516                                     ? new List<ObjectInfo>()\r
 517                                 //Otherwise deserialize the object list into a list of ObjectInfos\r
 518                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 519 \r
 520                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 521                 }\r
 522             }\r
 523 \r
 524 \r
 525         }\r
 526 \r
 527         public AccountInfo GetAccountPolicies(AccountInfo accountInfo)\r
 528         {\r
 529             if (accountInfo==null)\r
 530                 throw new ArgumentNullException("accountInfo");\r
 531             Contract.EndContractBlock();\r
 532 \r
 533             using (ThreadContext.Stacks["Account"].Push("GetPolicies"))\r
 534             {\r
 535                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 536 \r
 537                 if (_baseClient == null)\r
 538                 {\r
 539                     _baseClient = new RestClient\r
 540                     {\r
 541                         BaseAddress = accountInfo.StorageUri.ToString(),\r
 542                         Timeout = 10000,\r
 543                         Retries = 3,\r
 544                     };\r
 545                 }\r
 546 \r
 547                 using (var client = new RestClient(_baseClient))\r
 548                 {\r
 549                     if (!String.IsNullOrWhiteSpace(accountInfo.UserName))\r
 550                         client.BaseAddress = GetAccountUrl(accountInfo.UserName);\r
 551 \r
 552                     client.Parameters.Clear();\r
 553                     client.Parameters.Add("format", "json");                    \r
 554                     client.Head(String.Empty, 3);\r
 555 \r
 556                     var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];\r
 557                     var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];\r
 558 \r
 559                     long quota, bytes;\r
 560                     if (long.TryParse(quotaValue, out quota))\r
 561                         accountInfo.Quota = quota;\r
 562                     if (long.TryParse(bytesValue, out bytes))\r
 563                         accountInfo.BytesUsed = bytes;\r
 564                     \r
 565                     return accountInfo;\r
 566 \r
 567                 }\r
 568 \r
 569             }\r
 570         }\r
 571 \r
 572         public void UpdateMetadata(ObjectInfo objectInfo)\r
 573         {\r
 574             if (objectInfo == null)\r
 575                 throw new ArgumentNullException("objectInfo");\r
 576             Contract.EndContractBlock();\r
 577 \r
 578             using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))\r
 579             {\r
 580                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 581 \r
 582 \r
 583                 using(var client=new RestClient(_baseClient))\r
 584                 {\r
 585 \r
 586                     client.BaseAddress = GetAccountUrl(objectInfo.Account);\r
 587                     \r
 588                     client.Parameters.Clear();\r
 589                     \r
 590 \r
 591                     //Set Tags\r
 592                     foreach (var tag in objectInfo.Tags)\r
 593                     {\r
 594                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);\r
 595                         client.Headers.Add(headerTag, tag.Value);\r
 596                     }\r
 597 \r
 598                     //Set Permissions\r
 599 \r
 600                     var permissions=objectInfo.GetPermissionString();\r
 601                     client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);\r
 602 \r
 603                     client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);\r
 604                     client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);\r
 605                     client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);\r
 606                     var isPublic = objectInfo.IsPublic.ToString().ToLower();\r
 607                     client.Headers.Add("X-Object-Public", isPublic);\r
 608 \r
 609 \r
 610                     /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);\r
 611                     uriBuilder.Query = "update=";\r
 612                     var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/\r
 613                     var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);\r
 614                     client.PostWithRetry(address,"application/xml");\r
 615                     \r
 616                     //client.UploadValues(uri,new NameValueCollection());\r
 617 \r
 618 \r
 619                     client.AssertStatusOK("UpdateMetadata failed");\r
 620                     //If the status is NOT ACCEPTED or OK we have a problem\r
 621                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))\r
 622                     {\r
 623                         Log.Error("Failed to update metadata");\r
 624                         throw new Exception("Failed to update metadata");\r
 625                     }\r
 626 \r
 627                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 628                 }\r
 629             }\r
 630 \r
 631         }\r
 632 \r
 633         public void UpdateMetadata(ContainerInfo containerInfo)\r
 634         {\r
 635             if (containerInfo == null)\r
 636                 throw new ArgumentNullException("containerInfo");\r
 637             Contract.EndContractBlock();\r
 638 \r
 639             using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))\r
 640             {\r
 641                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 642 \r
 643 \r
 644                 using(var client=new RestClient(_baseClient))\r
 645                 {\r
 646 \r
 647                     client.BaseAddress = GetAccountUrl(containerInfo.Account);\r
 648                     \r
 649                     client.Parameters.Clear();\r
 650                     \r
 651 \r
 652                     //Set Tags\r
 653                     foreach (var tag in containerInfo.Tags)\r
 654                     {\r
 655                         var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);\r
 656                         client.Headers.Add(headerTag, tag.Value);\r
 657                     }\r
 658 \r
 659                     \r
 660                     //Set Policies\r
 661                     foreach (var policy in containerInfo.Policies)\r
 662                     {\r
 663                         var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);\r
 664                         client.Headers.Add(headerPolicy, policy.Value);\r
 665                     }\r
 666 \r
 667 \r
 668                     var uriBuilder = client.GetAddressBuilder(containerInfo.Name,"");\r
 669                     var uri = uriBuilder.Uri;\r
 670 \r
 671                     client.UploadValues(uri,new NameValueCollection());\r
 672 \r
 673 \r
 674                     client.AssertStatusOK("UpdateMetadata failed");\r
 675                     //If the status is NOT ACCEPTED or OK we have a problem\r
 676                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))\r
 677                     {\r
 678                         Log.Error("Failed to update metadata");\r
 679                         throw new Exception("Failed to update metadata");\r
 680                     }\r
 681 \r
 682                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 683                 }\r
 684             }\r
 685 \r
 686         }\r
 687 \r
 688 \r
 689         public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)\r
 690         {\r
 691             if (String.IsNullOrWhiteSpace(container))\r
 692                 throw new ArgumentNullException("container");\r
 693             Contract.EndContractBlock();\r
 694 \r
 695             using (ThreadContext.Stacks["Objects"].Push("List"))\r
 696             {\r
 697                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 698 \r
 699                 using (var client = new RestClient(_baseClient))\r
 700                 {\r
 701                     if (!String.IsNullOrWhiteSpace(account))\r
 702                         client.BaseAddress = GetAccountUrl(account);\r
 703 \r
 704                     client.Parameters.Clear();\r
 705                     client.Parameters.Add("format", "json");\r
 706                     client.IfModifiedSince = since;\r
 707                     var content = client.DownloadStringWithRetry(container, 3);\r
 708 \r
 709                     client.AssertStatusOK("ListObjects failed");\r
 710 \r
 711                     if (client.StatusCode==HttpStatusCode.NotModified)\r
 712                         return new[]{new NoModificationInfo(account,container)};\r
 713                     //If the result is empty, return an empty list,\r
 714                     var infos = String.IsNullOrWhiteSpace(content)\r
 715                                     ? new List<ObjectInfo>()\r
 716                                 //Otherwise deserialize the object list into a list of ObjectInfos\r
 717                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 718 \r
 719                     foreach (var info in infos)\r
 720                     {\r
 721                         info.Container = container;\r
 722                         info.Account = account;\r
 723                         info.StorageUri = this.StorageUrl;\r
 724                     }\r
 725                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 726                     return infos;\r
 727                 }\r
 728             }\r
 729         }\r
 730 \r
 731         public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)\r
 732         {\r
 733             if (String.IsNullOrWhiteSpace(container))\r
 734                 throw new ArgumentNullException("container");\r
 735 /*\r
 736             if (String.IsNullOrWhiteSpace(folder))\r
 737                 throw new ArgumentNullException("folder");\r
 738 */\r
 739             Contract.EndContractBlock();\r
 740 \r
 741             using (ThreadContext.Stacks["Objects"].Push("List"))\r
 742             {\r
 743                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 744 \r
 745                 using (var client = new RestClient(_baseClient))\r
 746                 {\r
 747                     if (!String.IsNullOrWhiteSpace(account))\r
 748                         client.BaseAddress = GetAccountUrl(account);\r
 749 \r
 750                     client.Parameters.Clear();\r
 751                     client.Parameters.Add("format", "json");\r
 752                     client.Parameters.Add("path", folder);\r
 753                     client.IfModifiedSince = since;\r
 754                     var content = client.DownloadStringWithRetry(container, 3);\r
 755                     client.AssertStatusOK("ListObjects failed");\r
 756 \r
 757                     if (client.StatusCode==HttpStatusCode.NotModified)\r
 758                         return new[]{new NoModificationInfo(account,container,folder)};\r
 759 \r
 760                     var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 761                     foreach (var info in infos)\r
 762                     {\r
 763                         info.Account = account;\r
 764                         if (info.Container == null)\r
 765                             info.Container = container;\r
 766                         info.StorageUri = this.StorageUrl;\r
 767                     }\r
 768                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 769                     return infos;\r
 770                 }\r
 771             }\r
 772         }\r
 773 \r
 774  \r
 775         public bool ContainerExists(string account, string container)\r
 776         {\r
 777             if (String.IsNullOrWhiteSpace(container))\r
 778                 throw new ArgumentNullException("container", "The container property can't be empty");\r
 779             Contract.EndContractBlock();\r
 780 \r
 781             using (ThreadContext.Stacks["Containters"].Push("Exists"))\r
 782             {\r
 783                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 784 \r
 785                 using (var client = new RestClient(_baseClient))\r
 786                 {\r
 787                     if (!String.IsNullOrWhiteSpace(account))\r
 788                         client.BaseAddress = GetAccountUrl(account);\r
 789 \r
 790                     client.Parameters.Clear();\r
 791                     client.Head(container, 3);\r
 792                                         \r
 793                     bool result;\r
 794                     switch (client.StatusCode)\r
 795                     {\r
 796                         case HttpStatusCode.OK:\r
 797                         case HttpStatusCode.NoContent:\r
 798                             result=true;\r
 799                             break;\r
 800                         case HttpStatusCode.NotFound:\r
 801                             result=false;\r
 802                             break;\r
 803                         default:\r
 804                             throw CreateWebException("ContainerExists", client.StatusCode);\r
 805                     }\r
 806                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 807 \r
 808                     return result;\r
 809                 }\r
 810                 \r
 811             }\r
 812         }\r
 813 \r
 814         public bool ObjectExists(string account, string container, string objectName)\r
 815         {\r
 816             if (String.IsNullOrWhiteSpace(container))\r
 817                 throw new ArgumentNullException("container", "The container property can't be empty");\r
 818             if (String.IsNullOrWhiteSpace(objectName))\r
 819                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
 820             Contract.EndContractBlock();\r
 821 \r
 822             using (var client = new RestClient(_baseClient))\r
 823             {\r
 824                 if (!String.IsNullOrWhiteSpace(account))\r
 825                     client.BaseAddress = GetAccountUrl(account);\r
 826 \r
 827                 client.Parameters.Clear();\r
 828                 client.Head(container + "/" + objectName, 3);\r
 829 \r
 830                 switch (client.StatusCode)\r
 831                 {\r
 832                     case HttpStatusCode.OK:\r
 833                     case HttpStatusCode.NoContent:\r
 834                         return true;\r
 835                     case HttpStatusCode.NotFound:\r
 836                         return false;\r
 837                     default:\r
 838                         throw CreateWebException("ObjectExists", client.StatusCode);\r
 839                 }\r
 840             }\r
 841 \r
 842         }\r
 843 \r
 844         public ObjectInfo GetObjectInfo(string account, string container, string objectName)\r
 845         {\r
 846             if (String.IsNullOrWhiteSpace(container))\r
 847                 throw new ArgumentNullException("container", "The container property can't be empty");\r
 848             if (String.IsNullOrWhiteSpace(objectName))\r
 849                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
 850             Contract.EndContractBlock();\r
 851 \r
 852             using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))\r
 853             {                \r
 854 \r
 855                 using (var client = new RestClient(_baseClient))\r
 856                 {\r
 857                     if (!String.IsNullOrWhiteSpace(account))\r
 858                         client.BaseAddress = GetAccountUrl(account);\r
 859                     try\r
 860                     {\r
 861                         client.Parameters.Clear();\r
 862 \r
 863                         client.Head(container + "/" + objectName, 3);\r
 864 \r
 865                         if (client.TimedOut)\r
 866                             return ObjectInfo.Empty;\r
 867 \r
 868                         switch (client.StatusCode)\r
 869                         {\r
 870                             case HttpStatusCode.OK:\r
 871                             case HttpStatusCode.NoContent:\r
 872                                 var keys = client.ResponseHeaders.AllKeys.AsQueryable();\r
 873                                 var tags = client.GetMeta("X-Object-Meta-");\r
 874                                 var extensions = (from key in keys\r
 875                                                   where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")\r
 876                                                   select new {Name = key, Value = client.ResponseHeaders[key]})\r
 877                                     .ToDictionary(t => t.Name, t => t.Value);\r
 878 \r
 879                                 var permissions=client.GetHeaderValue("X-Object-Sharing", true);\r
 880                                 \r
 881                                 \r
 882                                 var info = new ObjectInfo\r
 883                                                {\r
 884                                                    Account = account,\r
 885                                                    Container = container,\r
 886                                                    Name = objectName,\r
 887                                                    ETag = client.GetHeaderValue("ETag"),\r
 888                                                    UUID=client.GetHeaderValue("X-Object-UUID"),\r
 889                                                    X_Object_Hash = client.GetHeaderValue("X-Object-Hash"),\r
 890                                                    Content_Type = client.GetHeaderValue("Content-Type"),\r
 891                                                    Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),\r
 892                                                    Tags = tags,\r
 893                                                    Last_Modified = client.LastModified,\r
 894                                                    Extensions = extensions,\r
 895                                                    ContentEncoding=client.GetHeaderValue("Content-Encoding",true),\r
 896                                                    ContendDisposition = client.GetHeaderValue("Content-Disposition",true),\r
 897                                                    Manifest=client.GetHeaderValue("X-Object-Manifest",true),\r
 898                                                    PublicUrl=client.GetHeaderValue("X-Object-Public",true),  \r
 899                                                    StorageUri=this.StorageUrl,\r
 900                                                };\r
 901                                 info.SetPermissions(permissions);\r
 902                                 return info;\r
 903                             case HttpStatusCode.NotFound:\r
 904                                 return ObjectInfo.Empty;\r
 905                             default:\r
 906                                 throw new WebException(\r
 907                                     String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",\r
 908                                                   objectName, client.StatusCode));\r
 909                         }\r
 910 \r
 911                     }\r
 912                     catch (RetryException)\r
 913                     {\r
 914                         Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);\r
 915                         return ObjectInfo.Empty;\r
 916                     }\r
 917                     catch (WebException e)\r
 918                     {\r
 919                         Log.Error(\r
 920                             String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",\r
 921                                           objectName, client.StatusCode), e);\r
 922                         throw;\r
 923                     }\r
 924                 }                \r
 925             }\r
 926 \r
 927         }\r
 928 \r
 929         public void CreateFolder(string account, string container, string folder)\r
 930         {\r
 931             if (String.IsNullOrWhiteSpace(container))\r
 932                 throw new ArgumentNullException("container", "The container property can't be empty");\r
 933             if (String.IsNullOrWhiteSpace(folder))\r
 934                 throw new ArgumentNullException("folder", "The folder property can't be empty");\r
 935             Contract.EndContractBlock();\r
 936 \r
 937             var folderUrl=String.Format("{0}/{1}",container,folder);\r
 938             using (var client = new RestClient(_baseClient))\r
 939             {\r
 940                 if (!String.IsNullOrWhiteSpace(account))\r
 941                     client.BaseAddress = GetAccountUrl(account);\r
 942 \r
 943                 client.Parameters.Clear();\r
 944                 client.Headers.Add("Content-Type", @"application/directory");\r
 945                 client.Headers.Add("Content-Length", "0");\r
 946                 client.PutWithRetry(folderUrl, 3);\r
 947 \r
 948                 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)\r
 949                     throw CreateWebException("CreateFolder", client.StatusCode);\r
 950             }\r
 951         }\r
 952 \r
 953      \r
 954 \r
 955         public ContainerInfo GetContainerInfo(string account, string container)\r
 956         {\r
 957             if (String.IsNullOrWhiteSpace(container))\r
 958                 throw new ArgumentNullException("container", "The container property can't be empty");\r
 959             Contract.EndContractBlock();\r
 960 \r
 961             using (var client = new RestClient(_baseClient))\r
 962             {\r
 963                 if (!String.IsNullOrWhiteSpace(account))\r
 964                     client.BaseAddress = GetAccountUrl(account);                \r
 965 \r
 966                 client.Head(container);\r
 967                 switch (client.StatusCode)\r
 968                 {\r
 969                     case HttpStatusCode.OK:\r
 970                     case HttpStatusCode.NoContent:\r
 971                         var tags = client.GetMeta("X-Container-Meta-");\r
 972                         var policies = client.GetMeta("X-Container-Policy-");\r
 973 \r
 974                         var containerInfo = new ContainerInfo\r
 975                                                 {\r
 976                                                     Account=account,\r
 977                                                     Name = container,\r
 978                                                     StorageUrl=this.StorageUrl.ToString(),\r
 979                                                     Count =\r
 980                                                         long.Parse(client.GetHeaderValue("X-Container-Object-Count")),\r
 981                                                     Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),\r
 982                                                     BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),\r
 983                                                     BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),\r
 984                                                     Last_Modified=client.LastModified,\r
 985                                                     Tags=tags,\r
 986                                                     Policies=policies\r
 987                                                 };\r
 988                         \r
 989 \r
 990                         return containerInfo;\r
 991                     case HttpStatusCode.NotFound:\r
 992                         return ContainerInfo.Empty;\r
 993                     default:\r
 994                         throw CreateWebException("GetContainerInfo", client.StatusCode);\r
 995                 }\r
 996             }\r
 997         }\r
 998 \r
 999         public void CreateContainer(string account, string container)\r
1000         {\r
1001             if (String.IsNullOrWhiteSpace(account))\r
1002                 throw new ArgumentNullException("account");\r
1003             if (String.IsNullOrWhiteSpace(container))\r
1004                 throw new ArgumentNullException("container");\r
1005             Contract.EndContractBlock();\r
1006 \r
1007             using (var client = new RestClient(_baseClient))\r
1008             {\r
1009                 if (!String.IsNullOrWhiteSpace(account))\r
1010                     client.BaseAddress = GetAccountUrl(account);\r
1011 \r
1012                 client.PutWithRetry(container, 3);\r
1013                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};\r
1014                 if (!expectedCodes.Contains(client.StatusCode))\r
1015                     throw CreateWebException("CreateContainer", client.StatusCode);\r
1016             }\r
1017         }\r
1018 \r
1019         public void DeleteContainer(string account, string container)\r
1020         {\r
1021             if (String.IsNullOrWhiteSpace(container))\r
1022                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1023             Contract.EndContractBlock();\r
1024 \r
1025             using (var client = new RestClient(_baseClient))\r
1026             {\r
1027                 if (!String.IsNullOrWhiteSpace(account))\r
1028                     client.BaseAddress = GetAccountUrl(account);\r
1029 \r
1030                 client.DeleteWithRetry(container, 3);\r
1031                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};\r
1032                 if (!expectedCodes.Contains(client.StatusCode))\r
1033                     throw CreateWebException("DeleteContainer", client.StatusCode);\r
1034             }\r
1035 \r
1036         }\r
1037 \r
1038         /// <summary>\r
1039         /// \r
1040         /// </summary>\r
1041         /// <param name="account"></param>\r
1042         /// <param name="container"></param>\r
1043         /// <param name="objectName"></param>\r
1044         /// <param name="fileName"></param>\r
1045         /// <returns></returns>\r
1046         /// <remarks>This method should have no timeout or a very long one</remarks>\r
1047         //Asynchronously download the object specified by *objectName* in a specific *container* to \r
1048         // a local file\r
1049         public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken)\r
1050         {\r
1051             if (String.IsNullOrWhiteSpace(container))\r
1052                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1053             if (String.IsNullOrWhiteSpace(objectName))\r
1054                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");            \r
1055             Contract.EndContractBlock();\r
1056 \r
1057             try\r
1058             {\r
1059                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient\r
1060                 //object to avoid concurrency errors.\r
1061                 //\r
1062                 //Download operations take a long time therefore they have no timeout.\r
1063                 using(var client = new RestClient(_baseClient) { Timeout = 0 })\r
1064                 {\r
1065                     if (!String.IsNullOrWhiteSpace(account))\r
1066                         client.BaseAddress = GetAccountUrl(account);\r
1067 \r
1068                     //The container and objectName are relative names. They are joined with the client's\r
1069                     //BaseAddress to create the object's absolute address\r
1070                     var builder = client.GetAddressBuilder(container, objectName);\r
1071                     var uri = builder.Uri;\r
1072 \r
1073                     //Download progress is reported to the Trace log\r
1074                     Log.InfoFormat("[GET] START {0}", objectName);\r
1075                     /*client.DownloadProgressChanged += (sender, args) =>\r
1076                                                       Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1077                                                                      fileName, args.ProgressPercentage,\r
1078                                                                      args.BytesReceived,\r
1079                                                                      args.TotalBytesToReceive);*/\r
1080                     var progress = new Progress<DownloadProgressChangedEventArgs>(args =>\r
1081                                 {\r
1082                                     Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1083                                                    fileName, args.ProgressPercentage,\r
1084                                                    args.BytesReceived,\r
1085                                                    args.TotalBytesToReceive);\r
1086                                     if (DownloadProgressChanged!=null)\r
1087                                         DownloadProgressChanged(this, args);\r
1088                                 });\r
1089                     \r
1090                     //Start downloading the object asynchronously                    \r
1091                     await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false);\r
1092 \r
1093                     //Once the download completes\r
1094                     //Delete the local client object\r
1095                 }\r
1096                 //And report failure or completion\r
1097             }\r
1098             catch (Exception exc)\r
1099             {\r
1100                 Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);\r
1101                 throw;\r
1102             }\r
1103 \r
1104             Log.InfoFormat("[GET] END {0}", objectName);                                             \r
1105 \r
1106 \r
1107         }\r
1108 \r
1109         public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)\r
1110         {\r
1111             if (String.IsNullOrWhiteSpace(container))\r
1112                 throw new ArgumentNullException("container");\r
1113             if (String.IsNullOrWhiteSpace(objectName))\r
1114                 throw new ArgumentNullException("objectName");\r
1115             if (hash==null)\r
1116                 throw new ArgumentNullException("hash");\r
1117             if (String.IsNullOrWhiteSpace(Token))\r
1118                 throw new InvalidOperationException("Invalid Token");\r
1119             if (StorageUrl == null)\r
1120                 throw new InvalidOperationException("Invalid Storage Url");\r
1121             Contract.EndContractBlock();\r
1122 \r
1123 \r
1124             //Don't use a timeout because putting the hashmap may be a long process\r
1125             var client = new RestClient(_baseClient) { Timeout = 0 };           \r
1126             if (!String.IsNullOrWhiteSpace(account))\r
1127                 client.BaseAddress = GetAccountUrl(account);\r
1128 \r
1129             //The container and objectName are relative names. They are joined with the client's\r
1130             //BaseAddress to create the object's absolute address\r
1131             var builder = client.GetAddressBuilder(container, objectName);\r
1132             builder.Query = "format=json&hashmap";\r
1133             var uri = builder.Uri;\r
1134 \r
1135 \r
1136             //Send the tree hash as Json to the server            \r
1137             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";\r
1138             var jsonHash = hash.ToJson();\r
1139                         \r
1140             client.Headers.Add("ETag",hash.MD5);\r
1141             var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);\r
1142             if (Log.IsDebugEnabled)\r
1143                 Log.DebugFormat("Hashes:\r\n{0}", jsonHash);\r
1144             return uploadTask.ContinueWith(t =>\r
1145             {\r
1146 \r
1147                 var empty = (IList<string>)new List<string>();\r
1148                 \r
1149 \r
1150                 //The server will respond either with 201-created if all blocks were already on the server\r
1151                 if (client.StatusCode == HttpStatusCode.Created)                    \r
1152                 {\r
1153                     //in which case we return an empty hash list\r
1154                     return empty;\r
1155                 }\r
1156                 //or with a 409-conflict and return the list of missing parts\r
1157                 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception                \r
1158                 if (t.IsFaulted)\r
1159                 {\r
1160                     var ex = t.Exception.InnerException;\r
1161                     var we = ex as WebException;\r
1162                     var response = we.Response as HttpWebResponse;\r
1163                     if (response!=null && response.StatusCode==HttpStatusCode.Conflict)\r
1164                     {\r
1165                         //In case of 409 the missing parts will be in the response content                        \r
1166                         using (var stream = response.GetResponseStream())\r
1167                         using(var reader=stream.GetLoggedReader(Log))\r
1168                         {\r
1169                             //We used to have to cleanup the content before returning it because it contains\r
1170                             //error content after the list of hashes\r
1171                             //\r
1172                             //As of 30/1/2012, the result is a proper Json array so we don't need to read the content\r
1173                             //line by line\r
1174                             \r
1175                             var serializer = new JsonSerializer();                            \r
1176                             serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);\r
1177                             var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));\r
1178                             return hashes;\r
1179                         }                        \r
1180                     }                    \r
1181                     //Any other status code is unexpected and the exception should be rethrown\r
1182                     Log.LogError(response);\r
1183                     throw ex;\r
1184                     \r
1185                 }\r
1186 \r
1187                 //Any other status code is unexpected but there was no exception. We can probably continue processing\r
1188                 Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);                    \r
1189                 \r
1190                 return empty;\r
1191             });\r
1192 \r
1193         }\r
1194 \r
1195 \r
1196         public async Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)\r
1197         {\r
1198             if (String.IsNullOrWhiteSpace(Token))\r
1199                 throw new InvalidOperationException("Invalid Token");\r
1200             if (StorageUrl == null)\r
1201                 throw new InvalidOperationException("Invalid Storage Url");\r
1202             if (String.IsNullOrWhiteSpace(container))\r
1203                 throw new ArgumentNullException("container");\r
1204             if (relativeUrl == null)\r
1205                 throw new ArgumentNullException("relativeUrl");\r
1206             if (end.HasValue && end < 0)\r
1207                 throw new ArgumentOutOfRangeException("end");\r
1208             if (start < 0)\r
1209                 throw new ArgumentOutOfRangeException("start");\r
1210             Contract.EndContractBlock();\r
1211 \r
1212             //Don't use a timeout because putting the hashmap may be a long process\r
1213             using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end})\r
1214             {\r
1215                 if (!String.IsNullOrWhiteSpace(account))\r
1216                     client.BaseAddress = GetAccountUrl(account);\r
1217 \r
1218                 var builder = client.GetAddressBuilder(container, relativeUrl.ToString());\r
1219                 var uri = builder.Uri;\r
1220 \r
1221 /*                client.DownloadProgressChanged += (sender, args) =>\r
1222                                                       {\r
1223                                                           Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1224                                                                           uri.Segments.Last(), args.ProgressPercentage,\r
1225                                                                           args.BytesReceived,\r
1226                                                                           args.TotalBytesToReceive);\r
1227                                                           DownloadProgressChanged(sender, args);\r
1228                                                       };*/\r
1229                 var progress = new Progress<DownloadProgressChangedEventArgs>(args =>\r
1230                 {\r
1231                     Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1232                                     uri.Segments.Last(), args.ProgressPercentage,\r
1233                                     args.BytesReceived,\r
1234                                     args.TotalBytesToReceive);\r
1235                     if (DownloadProgressChanged!=null)\r
1236                         DownloadProgressChanged(this, args);\r
1237                 });\r
1238 \r
1239 \r
1240                 var result = await client.DownloadDataTaskAsync(uri, cancellationToken,progress).ConfigureAwait(false);\r
1241                 return result;\r
1242             }\r
1243         }\r
1244 \r
1245         public event UploadProgressChangedEventHandler UploadProgressChanged;\r
1246         public event DownloadProgressChangedEventHandler DownloadProgressChanged;\r
1247 \r
1248         public async Task PostBlock(string account, string container, byte[] block, int offset, int count,CancellationToken token)\r
1249         {\r
1250             if (String.IsNullOrWhiteSpace(container))\r
1251                 throw new ArgumentNullException("container");\r
1252             if (block == null)\r
1253                 throw new ArgumentNullException("block");\r
1254             if (offset < 0 || offset >= block.Length)\r
1255                 throw new ArgumentOutOfRangeException("offset");\r
1256             if (count < 0 || count > block.Length)\r
1257                 throw new ArgumentOutOfRangeException("count");\r
1258             if (String.IsNullOrWhiteSpace(Token))\r
1259                 throw new InvalidOperationException("Invalid Token");\r
1260             if (StorageUrl == null)\r
1261                 throw new InvalidOperationException("Invalid Storage Url");                        \r
1262             Contract.EndContractBlock();\r
1263 \r
1264 \r
1265             try\r
1266             {\r
1267 \r
1268                 //Don't use a timeout because putting the hashmap may be a long process\r
1269                 using (var client = new RestClient(_baseClient) { Timeout = 0 })\r
1270                 {\r
1271                     if (!String.IsNullOrWhiteSpace(account))\r
1272                         client.BaseAddress = GetAccountUrl(account);\r
1273 \r
1274                     var builder = client.GetAddressBuilder(container, "");\r
1275                     //We are doing an update\r
1276                     builder.Query = "update";\r
1277                     var uri = builder.Uri;\r
1278 \r
1279                     client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";\r
1280 \r
1281                     Log.InfoFormat("[BLOCK POST] START");\r
1282 \r
1283 /*\r
1284                     client.UploadProgressChanged += (sender, args) =>\r
1285                                                         {\r
1286                                                             Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",\r
1287                                                                            args.ProgressPercentage, args.BytesSent,\r
1288                                                                            args.TotalBytesToSend);\r
1289                                                             UploadProgressChanged(sender, args);\r
1290                                                         };\r
1291 */\r
1292                     client.UploadFileCompleted += (sender, args) =>\r
1293                                                   Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");\r
1294 \r
1295                     var progress=new Progress<UploadProgressChangedEventArgs>(args=>\r
1296                     {\r
1297                         Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",\r
1298                                         args.ProgressPercentage, args.BytesSent,\r
1299                                         args.TotalBytesToSend);\r
1300                         if (UploadProgressChanged!=null)\r
1301                             UploadProgressChanged(this, args);                                                                                      \r
1302                     });\r
1303                     var buffer = new byte[count];\r
1304                     Buffer.BlockCopy(block, offset, buffer, 0, count);\r
1305                     //Send the block\r
1306                     await client.UploadDataTaskAsync(uri, "POST", buffer,token,progress).ConfigureAwait(false);\r
1307                     Log.InfoFormat("[BLOCK POST] END");\r
1308                 }\r
1309             }\r
1310             catch (TaskCanceledException )\r
1311             {\r
1312                 Log.Info("Aborting block");\r
1313                 throw;\r
1314             }\r
1315             catch (Exception exc)\r
1316             {\r
1317                 Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);\r
1318                 throw;\r
1319             }\r
1320         }\r
1321 \r
1322 \r
1323         public async Task<TreeHash> GetHashMap(string account, string container, string objectName)\r
1324         {\r
1325             if (String.IsNullOrWhiteSpace(container))\r
1326                 throw new ArgumentNullException("container");\r
1327             if (String.IsNullOrWhiteSpace(objectName))\r
1328                 throw new ArgumentNullException("objectName");\r
1329             if (String.IsNullOrWhiteSpace(Token))\r
1330                 throw new InvalidOperationException("Invalid Token");\r
1331             if (StorageUrl == null)\r
1332                 throw new InvalidOperationException("Invalid Storage Url");\r
1333             Contract.EndContractBlock();\r
1334 \r
1335             try\r
1336             {\r
1337                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient\r
1338                 //object to avoid concurrency errors.\r
1339                 //\r
1340                 //Download operations take a long time therefore they have no timeout.\r
1341                 //TODO: Do they really? this is a hashmap operation, not a download\r
1342                 \r
1343                 //Start downloading the object asynchronously\r
1344                 using (var client = new RestClient(_baseClient) { Timeout = 0 })\r
1345                 {\r
1346                     if (!String.IsNullOrWhiteSpace(account))\r
1347                         client.BaseAddress = GetAccountUrl(account);\r
1348 \r
1349                     //The container and objectName are relative names. They are joined with the client's\r
1350                     //BaseAddress to create the object's absolute address\r
1351                     \r
1352                     var builder = client.GetAddressBuilder(container, objectName);\r
1353                     builder.Query = "format=json&hashmap";\r
1354                     var uri = builder.Uri;\r
1355 \r
1356 \r
1357                     var json = await client.DownloadStringTaskAsync(uri).ConfigureAwait(false);\r
1358                     var treeHash = TreeHash.Parse(json);\r
1359                     Log.InfoFormat("[GET HASH] END {0}", objectName);\r
1360                     return treeHash;\r
1361                 }\r
1362             }\r
1363             catch (Exception exc)\r
1364             {\r
1365                 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);\r
1366                 throw;\r
1367             }\r
1368 \r
1369         }\r
1370 \r
1371 \r
1372         /// <summary>\r
1373         /// \r
1374         /// </summary>\r
1375         /// <param name="account"></param>\r
1376         /// <param name="container"></param>\r
1377         /// <param name="objectName"></param>\r
1378         /// <param name="fileName"></param>\r
1379         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>\r
1380         /// <remarks>>This method should have no timeout or a very long one</remarks>\r
1381         public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream")\r
1382         {\r
1383             if (String.IsNullOrWhiteSpace(container))\r
1384                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1385             if (String.IsNullOrWhiteSpace(objectName))\r
1386                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1387             if (String.IsNullOrWhiteSpace(fileName))\r
1388                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");\r
1389 /*\r
1390             if (!File.Exists(fileName) && !Directory.Exists(fileName))\r
1391                 throw new FileNotFoundException("The file or directory does not exist",fileName);\r
1392 */\r
1393             \r
1394             try\r
1395             {\r
1396 \r
1397                 using (var client = new RestClient(_baseClient) { Timeout = 0 })\r
1398                 {\r
1399                     if (!String.IsNullOrWhiteSpace(account))\r
1400                         client.BaseAddress = GetAccountUrl(account);\r
1401 \r
1402                     var builder = client.GetAddressBuilder(container, objectName);\r
1403                     var uri = builder.Uri;\r
1404 \r
1405                     string etag = hash ?? CalculateHash(fileName);\r
1406 \r
1407                     client.Headers.Add("Content-Type", contentType);\r
1408                     client.Headers.Add("ETag", etag);\r
1409 \r
1410 \r
1411                     Log.InfoFormat("[PUT] START {0}", objectName);\r
1412                     client.UploadProgressChanged += (sender, args) =>\r
1413                                                         {\r
1414                                                             using (ThreadContext.Stacks["PUT"].Push("Progress"))\r
1415                                                             {\r
1416                                                                 Log.InfoFormat("{0} {1}% {2} of {3}", fileName,\r
1417                                                                                args.ProgressPercentage,\r
1418                                                                                args.BytesSent, args.TotalBytesToSend);\r
1419                                                             }\r
1420                                                         };\r
1421 \r
1422                     client.UploadFileCompleted += (sender, args) =>\r
1423                                                       {\r
1424                                                           using (ThreadContext.Stacks["PUT"].Push("Progress"))\r
1425                                                           {\r
1426                                                               Log.InfoFormat("Completed {0}", fileName);\r
1427                                                           }\r
1428                                                       };                    \r
1429                     if (contentType=="application/directory")\r
1430                         await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false);\r
1431                     else\r
1432                         await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false);\r
1433                 }\r
1434 \r
1435                 Log.InfoFormat("[PUT] END {0}", objectName);\r
1436             }\r
1437             catch (Exception exc)\r
1438             {\r
1439                 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);\r
1440                 throw;\r
1441             }                \r
1442 \r
1443         }\r
1444        \r
1445         \r
1446         private static string CalculateHash(string fileName)\r
1447         {\r
1448             Contract.Requires(!String.IsNullOrWhiteSpace(fileName));\r
1449             Contract.EndContractBlock();\r
1450 \r
1451             string hash;\r
1452             using (var hasher = MD5.Create())\r
1453             using(var stream=File.OpenRead(fileName))\r
1454             {\r
1455                 var hashBuilder=new StringBuilder();\r
1456                 foreach (byte b in hasher.ComputeHash(stream))\r
1457                     hashBuilder.Append(b.ToString("x2").ToLower());\r
1458                 hash = hashBuilder.ToString();                \r
1459             }\r
1460             return hash;\r
1461         }\r
1462         \r
1463         public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)\r
1464         {\r
1465             if (String.IsNullOrWhiteSpace(sourceContainer))\r
1466                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");\r
1467             if (String.IsNullOrWhiteSpace(oldObjectName))\r
1468                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");\r
1469             if (String.IsNullOrWhiteSpace(targetContainer))\r
1470                 throw new ArgumentNullException("targetContainer", "The container property can't be empty");\r
1471             if (String.IsNullOrWhiteSpace(newObjectName))\r
1472                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");\r
1473             Contract.EndContractBlock();\r
1474 \r
1475             var targetUrl = targetContainer + "/" + newObjectName;\r
1476             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);\r
1477 \r
1478             using (var client = new RestClient(_baseClient))\r
1479             {\r
1480                 if (!String.IsNullOrWhiteSpace(account))\r
1481                     client.BaseAddress = GetAccountUrl(account);\r
1482 \r
1483                 client.Headers.Add("X-Move-From", Uri.EscapeUriString(sourceUrl));\r
1484                 client.PutWithRetry(targetUrl, 3);\r
1485 \r
1486                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
1487                 if (!expectedCodes.Contains(client.StatusCode))\r
1488                     throw CreateWebException("MoveObject", client.StatusCode);\r
1489             }\r
1490         }\r
1491 \r
1492         public void DeleteObject(string account, string sourceContainer, string objectName, bool isDirectory)\r
1493         {            \r
1494             if (String.IsNullOrWhiteSpace(sourceContainer))\r
1495                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");\r
1496             if (String.IsNullOrWhiteSpace(objectName))\r
1497                 throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");\r
1498             Contract.EndContractBlock();\r
1499 \r
1500             var targetUrl = FolderConstants.TrashContainer + "/" + objectName;\r
1501 /*\r
1502             if (isDirectory)\r
1503                 targetUrl = targetUrl + "?delimiter=/";\r
1504 */\r
1505 \r
1506             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);\r
1507 \r
1508             using (var client = new RestClient(_baseClient))\r
1509             {\r
1510                 if (!String.IsNullOrWhiteSpace(account))\r
1511                     client.BaseAddress = GetAccountUrl(account);\r
1512 \r
1513                 client.Headers.Add("X-Move-From", Uri.EscapeUriString(sourceUrl));\r
1514                 client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);\r
1515                 Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);\r
1516                 client.PutWithRetry(targetUrl, 3);\r
1517 \r
1518                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};\r
1519                 if (!expectedCodes.Contains(client.StatusCode))\r
1520                     throw CreateWebException("DeleteObject", client.StatusCode);\r
1521             }\r
1522         }\r
1523 \r
1524       \r
1525         private static WebException CreateWebException(string operation, HttpStatusCode statusCode)\r
1526         {\r
1527             return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));\r
1528         }\r
1529 \r
1530 \r
1531 /*\r
1532         public IEnumerable<ObjectInfo> ListDirectories(ContainerInfo container)\r
1533         {\r
1534             var directories=this.ListObjects(container.Account, container.Name, "/");\r
1535         }\r
1536 */\r
1537 \r
1538         public bool CanUpload(string account, ObjectInfo cloudFile)\r
1539         {\r
1540             Contract.Requires(!String.IsNullOrWhiteSpace(account));\r
1541             Contract.Requires(cloudFile!=null);\r
1542 \r
1543             using (var client = new RestClient(_baseClient))\r
1544             {\r
1545                 if (!String.IsNullOrWhiteSpace(account))\r
1546                     client.BaseAddress = GetAccountUrl(account);\r
1547 \r
1548 \r
1549                 var parts = cloudFile.Name.Split('/');\r
1550                 var folder = String.Join("/", parts,0,parts.Length-1);\r
1551 \r
1552                 var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid());\r
1553 \r
1554                 client.Parameters.Clear();\r
1555                 try\r
1556                 {\r
1557                     client.PutWithRetry(fileUrl, 3, @"application/octet-stream");\r
1558 \r
1559                     var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
1560                     var result=(expectedCodes.Contains(client.StatusCode));\r
1561                     DeleteObject(account, cloudFile.Container, fileUrl, cloudFile.IsDirectory);\r
1562                     return result;\r
1563                 }\r
1564                 catch\r
1565                 {\r
1566                     return false;\r
1567                 }\r
1568             }\r
1569         }\r
1570     }\r
1571 \r
1572     public class ShareAccountInfo\r
1573     {\r
1574         public DateTime? last_modified { get; set; }\r
1575         public string name { get; set; }\r
1576     }\r
1577 }\r