code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="CloudFilesClient.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 \r
  43 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos\r
  44 //\r
  45 // The class provides methods to upload/download files, delete files, manage containers\r
  46 \r
  47 \r
  48 using System;\r
  49 using System.Collections.Generic;\r
  50 using System.Collections.Specialized;\r
  51 using System.ComponentModel.Composition;\r
  52 using System.Diagnostics;\r
  53 using System.Diagnostics.Contracts;\r
  54 using System.IO;\r
  55 using System.Linq;\r
  56 using System.Net;\r
  57 using System.Net.Http;\r
  58 using System.Net.Http.Headers;\r
  59 using System.Reflection;\r
  60 using System.Threading;\r
  61 using System.Threading.Tasks;\r
  62 using Newtonsoft.Json;\r
  63 using Pithos.Interfaces;\r
  64 using Pithos.Network;\r
  65 using log4net;\r
  66 \r
  67 namespace Pithos.Network\r
  68 {\r
  69 \r
  70     [Export(typeof(ICloudClient))]\r
  71     public class CloudFilesClient:ICloudClient,IDisposable\r
  72     {\r
  73         private const string TOKEN_HEADER = "X-Auth-Token";\r
  74         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  75 \r
  76         //CloudFilesClient uses *_baseClient* internally to communicate with the server\r
  77         //RestClient provides a REST-friendly interface over the standard WebClient.\r
  78         private RestClient _baseClient;\r
  79 \r
  80         private HttpClient _baseHttpClient;\r
  81         private HttpClient _baseHttpClientNoTimeout;\r
  82         \r
  83 \r
  84         //During authentication the client provides a UserName \r
  85         public string UserName { get; set; }\r
  86         \r
  87         //and and ApiKey to the server\r
  88         public string ApiKey { get; set; }\r
  89         \r
  90         //And receives an authentication Token. This token must be provided in ALL other operations,\r
  91         //in the X-Auth-Token header\r
  92         private string _token;\r
  93         private readonly string _emptyGuid = Guid.Empty.ToString();\r
  94         private readonly Uri _emptyUri = new Uri("",UriKind.Relative);\r
  95 \r
  96         private HttpClientHandler _httpClientHandler = new HttpClientHandler\r
  97                                                            {\r
  98                                                                AllowAutoRedirect = true,\r
  99                                                                AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,\r
 100                                                                UseCookies = true,                                \r
 101                                                            };\r
 102 \r
 103 \r
 104         public string Token\r
 105         {\r
 106             get { return _token; }\r
 107             set\r
 108             {\r
 109                 _token = value;\r
 110                 _baseClient.Headers[TOKEN_HEADER] = value;\r
 111             }\r
 112         }\r
 113 \r
 114         //The client also receives a StorageUrl after authentication. All subsequent operations must\r
 115         //use this url\r
 116         public Uri StorageUrl { get; set; }\r
 117 \r
 118 \r
 119         public Uri RootAddressUri { get; set; }\r
 120 \r
 121        /* private WebProxy _proxy;\r
 122         public WebProxy Proxy\r
 123         {\r
 124             get { return _proxy; }\r
 125             set\r
 126             {\r
 127                 _proxy = value;\r
 128                 if (_baseClient != null)\r
 129                     _baseClient.Proxy = value;                \r
 130             }\r
 131         }\r
 132 */\r
 133 \r
 134         /* private Uri _proxy;\r
 135         public Uri Proxy\r
 136         {\r
 137             get { return _proxy; }\r
 138             set\r
 139             {\r
 140                 _proxy = value;\r
 141                 if (_baseClient != null)\r
 142                     _baseClient.Proxy = new WebProxy(value);                \r
 143             }\r
 144         }*/\r
 145 \r
 146         public double DownloadPercentLimit { get; set; }\r
 147         public double UploadPercentLimit { get; set; }\r
 148 \r
 149         public string AuthenticationUrl { get; set; }\r
 150 \r
 151  \r
 152         public string VersionPath\r
 153         {\r
 154             get { return UsePithos ? "v1" : "v1.0"; }\r
 155         }\r
 156 \r
 157         public bool UsePithos { get; set; }\r
 158 \r
 159 \r
 160 \r
 161         public CloudFilesClient(string userName, string apiKey)\r
 162         {\r
 163             UserName = userName;\r
 164             ApiKey = apiKey;\r
 165         }\r
 166 \r
 167         public CloudFilesClient(AccountInfo accountInfo)\r
 168         {\r
 169             if (accountInfo==null)\r
 170                 throw new ArgumentNullException("accountInfo");\r
 171             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));\r
 172             Contract.Ensures(StorageUrl != null);\r
 173             Contract.Ensures(_baseClient != null);\r
 174             Contract.Ensures(RootAddressUri != null);\r
 175             Contract.EndContractBlock();          \r
 176 \r
 177             _baseClient = new RestClient\r
 178             {\r
 179                 BaseAddress = accountInfo.StorageUri.ToString(),\r
 180                 Timeout = 30000,\r
 181                 Retries = 3,\r
 182             };\r
 183             StorageUrl = accountInfo.StorageUri;\r
 184             Token = accountInfo.Token;\r
 185             UserName = accountInfo.UserName;\r
 186 \r
 187             //Get the root address (StorageUrl without the account)\r
 188             var storageUrl = StorageUrl.AbsoluteUri;\r
 189             var usernameIndex = storageUrl.LastIndexOf(UserName);\r
 190             var rootUrl = storageUrl.Substring(0, usernameIndex);\r
 191             RootAddressUri = new Uri(rootUrl);\r
 192 \r
 193             var httpClientHandler = new HttpClientHandler\r
 194             {\r
 195                 AllowAutoRedirect = true,\r
 196                 AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,\r
 197                 UseCookies = true,\r
 198             };\r
 199 \r
 200 \r
 201             _baseHttpClient = new HttpClient(httpClientHandler)\r
 202             {\r
 203                 BaseAddress = StorageUrl,\r
 204                 Timeout = TimeSpan.FromSeconds(30)\r
 205             };\r
 206             _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);\r
 207 \r
 208             _baseHttpClientNoTimeout = new HttpClient(httpClientHandler)\r
 209             {\r
 210                 BaseAddress = StorageUrl,\r
 211                 Timeout = TimeSpan.FromMilliseconds(-1)\r
 212             };\r
 213             _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);\r
 214 \r
 215 \r
 216         }\r
 217 \r
 218 \r
 219         private static void AssertStatusOK(HttpResponseMessage response, string message)\r
 220         {\r
 221             var statusCode = response.StatusCode;\r
 222             if (statusCode >= HttpStatusCode.BadRequest)\r
 223                 throw new WebException(String.Format("{0} with code {1} - {2}", message, statusCode, response.ReasonPhrase));\r
 224         }\r
 225 \r
 226         public async Task<AccountInfo> Authenticate()\r
 227         {\r
 228             if (String.IsNullOrWhiteSpace(UserName))\r
 229                 throw new InvalidOperationException("UserName is empty");\r
 230             if (String.IsNullOrWhiteSpace(ApiKey))\r
 231                 throw new InvalidOperationException("ApiKey is empty");\r
 232             if (String.IsNullOrWhiteSpace(AuthenticationUrl))\r
 233                 throw new InvalidOperationException("AuthenticationUrl is empty");\r
 234             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));\r
 235             Contract.Ensures(StorageUrl != null);\r
 236             Contract.Ensures(_baseClient != null);\r
 237             Contract.Ensures(RootAddressUri != null);\r
 238             Contract.EndContractBlock();\r
 239 \r
 240 \r
 241             Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);\r
 242 \r
 243             var groups = new List<Group>();\r
 244 \r
 245             using (var authClient = new HttpClient(_httpClientHandler,false){ BaseAddress = new Uri(AuthenticationUrl),Timeout=TimeSpan.FromSeconds(30) })\r
 246             //using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})\r
 247             {                \r
 248                /* if (Proxy != null)\r
 249                     authClient.Proxy = Proxy;*/\r
 250 \r
 251                 //Contract.Assume(authClient.DefaultRequestHeaders != null);\r
 252 \r
 253                 authClient.DefaultRequestHeaders.Add("X-Auth-User", UserName);\r
 254                 authClient.DefaultRequestHeaders.Add("X-Auth-Key", ApiKey);\r
 255 \r
 256                 //Func<Task<HttpResponseMessage>> call = () => ;\r
 257                 string storageUrl;\r
 258                 string token;\r
 259                 \r
 260                 using (var response = await authClient.GetAsyncWithRetries(new Uri(VersionPath, UriKind.Relative),3).ConfigureAwait(false)) // .DownloadStringWithRetryRelative(new Uri(VersionPath, UriKind.Relative), 3);                    \r
 261                 {\r
 262                     AssertStatusOK(response,"Authentication failed");\r
 263                 \r
 264                     storageUrl = response.Headers.GetValues("X-Storage-Url").First();\r
 265                     if (String.IsNullOrWhiteSpace(storageUrl))\r
 266                         throw new InvalidOperationException("Failed to obtain storage url");\r
 267 \r
 268                     token = response.Headers.GetValues(TOKEN_HEADER).First();\r
 269                     if (String.IsNullOrWhiteSpace(token))\r
 270                         throw new InvalidOperationException("Failed to obtain token url");\r
 271 \r
 272                 }\r
 273 \r
 274 \r
 275                 _baseClient = new RestClient\r
 276                 {\r
 277                     BaseAddress = storageUrl,\r
 278                     Timeout = 30000,\r
 279                     Retries = 3,                    \r
 280                     //Proxy=Proxy\r
 281                 };\r
 282 \r
 283                 StorageUrl = new Uri(storageUrl);\r
 284                 Token = token;\r
 285 \r
 286                 \r
 287 \r
 288                                                                \r
 289                 //Get the root address (StorageUrl without the account)\r
 290                 var usernameIndex=storageUrl.LastIndexOf(UserName);\r
 291                 var rootUrl = storageUrl.Substring(0, usernameIndex);\r
 292                 RootAddressUri = new Uri(rootUrl);\r
 293                 \r
 294 \r
 295                 _baseHttpClient = new HttpClient(_httpClientHandler,false)\r
 296                 {\r
 297                     BaseAddress = StorageUrl,\r
 298                     Timeout = TimeSpan.FromSeconds(30)\r
 299                 };\r
 300                 _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, token);\r
 301 \r
 302                 _baseHttpClientNoTimeout = new HttpClient(_httpClientHandler,false)\r
 303                 {\r
 304                     BaseAddress = StorageUrl,\r
 305                     Timeout = TimeSpan.FromMilliseconds(-1)\r
 306                 };\r
 307                 _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, token);\r
 308 \r
 309                 /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();\r
 310                 groups = (from key in keys\r
 311                             where key.StartsWith("X-Account-Group-")\r
 312                             let name = key.Substring(16)\r
 313                             select new Group(name, authClient.ResponseHeaders[key]))\r
 314                         .ToList();\r
 315                     \r
 316 */\r
 317             }\r
 318 \r
 319             Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);\r
 320             Debug.Assert(_baseClient!=null);\r
 321 \r
 322             return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};            \r
 323 \r
 324         }\r
 325 \r
 326         private static void TraceStart(string method, Uri actualAddress)\r
 327         {\r
 328             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);\r
 329         }\r
 330 \r
 331         private async Task<string> GetStringAsync(Uri targetUri, string errorMessage,DateTime? since=null)\r
 332         {\r
 333             TraceStart("GET",targetUri);\r
 334             var request = new HttpRequestMessage(HttpMethod.Get, targetUri);            \r
 335             if (since.HasValue)\r
 336             {\r
 337                 request.Headers.IfModifiedSince = since.Value;\r
 338             }\r
 339             //Func<Task<HttpResponseMessage>> call = () => _baseHttpClient.SendAsync(request);\r
 340             using (var response = await _baseHttpClient.SendAsyncWithRetries(request,3).ConfigureAwait(false))\r
 341             {\r
 342                 AssertStatusOK(response, errorMessage);\r
 343 \r
 344                 if (response.StatusCode == HttpStatusCode.NoContent)\r
 345                     return String.Empty;\r
 346 \r
 347                 var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);\r
 348                 return content;\r
 349             }\r
 350         }\r
 351 \r
 352         public async Task<IList<ContainerInfo>> ListContainers(string account)\r
 353         {\r
 354 \r
 355             var targetUrl = GetTargetUrl(account);\r
 356             var targetUri = new Uri(String.Format("{0}?format=json", targetUrl));\r
 357             var result = await GetStringAsync(targetUri, "List Containers failed").ConfigureAwait(false);\r
 358             if (String.IsNullOrWhiteSpace(result))\r
 359                 return new List<ContainerInfo>();\r
 360             var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(result);\r
 361             foreach (var info in infos)\r
 362             {\r
 363                 info.Account = account;\r
 364             }\r
 365             return infos;\r
 366         }\r
 367 \r
 368         //public IList<ContainerInfo> ListContainers(string account)\r
 369         //{\r
 370 \r
 371         //    var targetUrl = GetTargetUrl(account);\r
 372         //    var targetUri=new Uri(String.Format("{0}?format=json",targetUrl));\r
 373         //    var result = GetStringAsync(targetUri, "List Containers failed").Result;\r
 374         //    if (String.IsNullOrWhiteSpace(result))\r
 375         //        return new List<ContainerInfo>();\r
 376         //    var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(result);                \r
 377         //    foreach (var info in infos)\r
 378         //    {\r
 379         //        info.Account = account;\r
 380         //    }\r
 381         //    return infos;\r
 382         //   /* using (var client = new RestClient(_baseClient))\r
 383         //    {\r
 384         //        if (!String.IsNullOrWhiteSpace(account))\r
 385         //            client.BaseAddress = GetAccountUrl(account);\r
 386                 \r
 387         //        client.Parameters.Clear();\r
 388         //        client.Parameters.Add("format", "json");\r
 389         //        var content = client.DownloadStringWithRetryRelative(_emptyUri, 3);\r
 390         //        client.AssertStatusOK("List Containers failed");\r
 391 \r
 392         //        if (client.StatusCode == HttpStatusCode.NoContent)\r
 393         //            return new List<ContainerInfo>();\r
 394         //        var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);\r
 395                 \r
 396         //        foreach (var info in infos)\r
 397         //        {\r
 398         //            info.Account = account;\r
 399         //        }\r
 400         //        return infos;\r
 401         //    }   */       \r
 402 \r
 403         //}\r
 404 \r
 405         private string GetAccountUrl(string account)\r
 406         {\r
 407             return RootAddressUri.Combine(account).AbsoluteUri;\r
 408         }\r
 409 \r
 410         public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)\r
 411         {\r
 412             using (ThreadContext.Stacks["Share"].Push("List Accounts"))\r
 413             {\r
 414                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 415 \r
 416                 var targetUri = new Uri(String.Format("{0}?format=json", RootAddressUri), UriKind.Absolute);\r
 417                 var content=TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListSharingAccounts failed", since).ConfigureAwait(false)).Result;\r
 418 \r
 419                 //If the result is empty, return an empty list,\r
 420                 var infos = String.IsNullOrWhiteSpace(content)\r
 421                                 ? new List<ShareAccountInfo>()\r
 422                             //Otherwise deserialize the account list into a list of ShareAccountInfos\r
 423                                 : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);\r
 424 \r
 425                 Log.DebugFormat("END");\r
 426                 return infos;\r
 427 /*\r
 428                 using (var client = new RestClient(_baseClient))\r
 429                 {\r
 430                     client.Parameters.Clear();\r
 431                     client.Parameters.Add("format", "json");\r
 432                     client.IfModifiedSince = since;\r
 433 \r
 434                     //Extract the username from the base address\r
 435                     client.BaseAddress = RootAddressUri.AbsoluteUri; \r
 436 \r
 437                     var content = client.DownloadStringWithRetryRelative(_emptyUri, 3);\r
 438 \r
 439                     client.AssertStatusOK("ListSharingAccounts failed");\r
 440 \r
 441                     //If the result is empty, return an empty list,\r
 442                     var infos = String.IsNullOrWhiteSpace(content)\r
 443                                     ? new List<ShareAccountInfo>()\r
 444                                 //Otherwise deserialize the account list into a list of ShareAccountInfos\r
 445                                     : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);\r
 446 \r
 447                     Log.DebugFormat("END");\r
 448                     return infos;\r
 449                 }\r
 450 */\r
 451             }\r
 452         }\r
 453 \r
 454 \r
 455         /// <summary>\r
 456         /// Request listing of all objects in a container modified since a specific time.\r
 457         /// If the *since* value is missing, return all objects\r
 458         /// </summary>\r
 459         /// <param name="knownContainers">Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new\r
 460         /// and should be polled anyway\r
 461         /// </param>\r
 462         /// <param name="since"></param>\r
 463         /// <returns></returns>\r
 464         public IList<ObjectInfo> ListSharedObjects(HashSet<string> knownContainers,DateTime? since = null )\r
 465         {\r
 466 \r
 467             using (ThreadContext.Stacks["Share"].Push("List Objects"))\r
 468             {\r
 469                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 470                 //'since' is not used here because we need to have ListObjects return a NoChange result\r
 471                 //for all shared accounts,containers\r
 472 \r
 473                 Func<ContainerInfo, string> getKey = c => String.Format("{0}\\{1}", c.Account, c.Name);\r
 474 \r
 475                 var containers = (from account in ListSharingAccounts()\r
 476                                  let conts = ListContainers(account.name).Result\r
 477                                  from container in conts\r
 478                                  select container).ToList();                \r
 479                 var items = from container in containers \r
 480                             let actualSince=knownContainers.Contains(getKey(container))?since:null\r
 481                             select ListObjects(container.Account , container.Name,  actualSince);\r
 482                 var objects=items.SelectMany(r=> r).ToList();\r
 483 \r
 484                 //For each object\r
 485                 //Check parents recursively up to (but not including) the container.\r
 486                 //If parents are missing, add them to the list\r
 487                 //Need function to calculate all parent URLs\r
 488                 objects = AddMissingParents(objects);\r
 489                 \r
 490                 //Store any new containers\r
 491                 foreach (var container in containers)\r
 492                 {\r
 493                     knownContainers.Add(getKey(container));\r
 494                 }\r
 495 \r
 496 \r
 497 \r
 498                 if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 499                 return objects;\r
 500             }\r
 501         }\r
 502 \r
 503         private List<ObjectInfo> AddMissingParents(List<ObjectInfo> objects)\r
 504         {\r
 505             //TODO: Remove short-circuit when we decide to use Missing Parents functionality\r
 506             //return objects;\r
 507 \r
 508             var existingUris = objects.ToDictionary(o => o.Uri, o => o);\r
 509             foreach (var objectInfo in objects)\r
 510             {\r
 511                 //Can be null when retrieving objects to show in selective sync\r
 512                 if (objectInfo.Name == null)\r
 513                     continue;\r
 514 \r
 515                 //No need to unescape here, the parts will be used to create new ObjectInfos\r
 516                 var parts = objectInfo.Name.ToString().Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries);\r
 517                 //If there is no parent, skip\r
 518                 if (parts.Length == 1)\r
 519                     continue;\r
 520                 var baseParts = new[]\r
 521                                   {\r
 522                                       objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container.ToString()\r
 523                                   };\r
 524                 for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++)\r
 525                 {\r
 526                     var nameparts = parts.Range(0, partIdx).ToArray();\r
 527                     var parentName= String.Join("/", nameparts);\r
 528 \r
 529                     var parentParts = baseParts.Concat(nameparts);\r
 530                     var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts);\r
 531                     \r
 532                     var parentUri = new Uri(parentUrl, UriKind.Absolute);\r
 533 \r
 534                     ObjectInfo existingInfo;\r
 535                     if (!existingUris.TryGetValue(parentUri,out existingInfo))\r
 536                     {\r
 537                         var h = parentUrl.GetHashCode();\r
 538                         var reverse = new string(parentUrl.Reverse().ToArray());\r
 539                         var rh = reverse.GetHashCode();\r
 540                         var b1 = BitConverter.GetBytes(h);\r
 541                         var b2 = BitConverter.GetBytes(rh);\r
 542                         var g = new Guid(0,0,0,b1.Concat(b2).ToArray());\r
 543                         \r
 544 \r
 545                         existingUris[parentUri] = new ObjectInfo\r
 546                                                       {\r
 547                                                           Account = objectInfo.Account,\r
 548                                                           Container = objectInfo.Container,\r
 549                                                           Content_Type = ObjectInfo.CONTENT_TYPE_DIRECTORY,\r
 550                                                           ETag = Signature.MERKLE_EMPTY,\r
 551                                                           X_Object_Hash = Signature.MERKLE_EMPTY,\r
 552                                                           Name=new Uri(parentName,UriKind.Relative),\r
 553                                                           StorageUri=objectInfo.StorageUri,\r
 554                                                           Bytes = 0,\r
 555                                                           UUID=g.ToString(),                                                          \r
 556                                                       };\r
 557                     }\r
 558                 }\r
 559             }\r
 560             return existingUris.Values.ToList();\r
 561         }\r
 562 \r
 563         public void SetTags(ObjectInfo target,IDictionary<string,string> tags)\r
 564         {\r
 565             if (String.IsNullOrWhiteSpace(Token))\r
 566                 throw new InvalidOperationException("The Token is not set");\r
 567             if (StorageUrl == null)\r
 568                 throw new InvalidOperationException("The StorageUrl is not set");\r
 569             if (target == null)\r
 570                 throw new ArgumentNullException("target");\r
 571             Contract.EndContractBlock();\r
 572 \r
 573             using (ThreadContext.Stacks["Share"].Push("Share Object"))\r
 574             {\r
 575                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 576 \r
 577                 using (var client = new RestClient(_baseClient))\r
 578                 {\r
 579 \r
 580                     client.BaseAddress = GetAccountUrl(target.Account);\r
 581 \r
 582                     client.Parameters.Clear();\r
 583                     client.Parameters.Add("update", "");\r
 584 \r
 585                     foreach (var tag in tags)\r
 586                     {\r
 587                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);\r
 588                         client.Headers.Add(headerTag, tag.Value);\r
 589                     }\r
 590                     \r
 591                     client.DownloadStringWithRetryRelative(target.Container, 3);\r
 592 \r
 593                     \r
 594                     client.AssertStatusOK("SetTags failed");\r
 595                     //If the status is NOT ACCEPTED we have a problem\r
 596                     if (client.StatusCode != HttpStatusCode.Accepted)\r
 597                     {\r
 598                         Log.Error("Failed to set tags");\r
 599                         throw new Exception("Failed to set tags");\r
 600                     }\r
 601 \r
 602                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 603                 }\r
 604             }\r
 605 \r
 606 \r
 607         }\r
 608 \r
 609         public void ShareObject(string account, Uri container, Uri objectName, string shareTo, bool read, bool write)\r
 610         {\r
 611             if (String.IsNullOrWhiteSpace(Token))\r
 612                 throw new InvalidOperationException("The Token is not set");\r
 613             if (StorageUrl==null)\r
 614                 throw new InvalidOperationException("The StorageUrl is not set");\r
 615             if (container==null)\r
 616                 throw new ArgumentNullException("container");\r
 617             if (container.IsAbsoluteUri)\r
 618                 throw new ArgumentException("container");\r
 619             if (objectName==null)\r
 620                 throw new ArgumentNullException("objectName");\r
 621             if (objectName.IsAbsoluteUri)\r
 622                 throw new ArgumentException("objectName");\r
 623             if (String.IsNullOrWhiteSpace(account))\r
 624                 throw new ArgumentNullException("account");\r
 625             if (String.IsNullOrWhiteSpace(shareTo))\r
 626                 throw new ArgumentNullException("shareTo");\r
 627             Contract.EndContractBlock();\r
 628 \r
 629             using (ThreadContext.Stacks["Share"].Push("Share Object"))\r
 630             {\r
 631                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 632                 \r
 633                 using (var client = new RestClient(_baseClient))\r
 634                 {\r
 635 \r
 636                     client.BaseAddress = GetAccountUrl(account);\r
 637 \r
 638                     client.Parameters.Clear();\r
 639                     client.Parameters.Add("format", "json");\r
 640 \r
 641                     string permission = "";\r
 642                     if (write)\r
 643                         permission = String.Format("write={0}", shareTo);\r
 644                     else if (read)\r
 645                         permission = String.Format("read={0}", shareTo);\r
 646                     client.Headers.Add("X-Object-Sharing", permission);\r
 647 \r
 648                     var content = client.DownloadStringWithRetryRelative(container, 3);\r
 649 \r
 650                     client.AssertStatusOK("ShareObject failed");\r
 651 \r
 652                     //If the result is empty, return an empty list,\r
 653                     var infos = String.IsNullOrWhiteSpace(content)\r
 654                                     ? new List<ObjectInfo>()\r
 655                                 //Otherwise deserialize the object list into a list of ObjectInfos\r
 656                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 657 \r
 658                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 659                 }\r
 660             }\r
 661 \r
 662 \r
 663         }\r
 664 \r
 665         public AccountInfo GetAccountPolicies(AccountInfo accountInfo)\r
 666         {\r
 667             if (accountInfo==null)\r
 668                 throw new ArgumentNullException("accountInfo");\r
 669             Contract.EndContractBlock();\r
 670 \r
 671             using (ThreadContext.Stacks["Account"].Push("GetPolicies"))\r
 672             {\r
 673                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 674 \r
 675                 if (_baseClient == null)\r
 676                 {\r
 677                     _baseClient = new RestClient\r
 678                     {\r
 679                         BaseAddress = accountInfo.StorageUri.ToString(),\r
 680                         Timeout = 30000,\r
 681                         Retries = 3,\r
 682                     };\r
 683                 }\r
 684 \r
 685                 using (var client = new RestClient(_baseClient))\r
 686                 {\r
 687                     if (!String.IsNullOrWhiteSpace(accountInfo.UserName))\r
 688                         client.BaseAddress = GetAccountUrl(accountInfo.UserName);\r
 689 \r
 690                     client.Parameters.Clear();\r
 691                     client.Parameters.Add("format", "json");                    \r
 692                     client.Head(_emptyUri, 3);\r
 693 \r
 694                     var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];\r
 695                     var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];\r
 696 \r
 697                     long quota, bytes;\r
 698                     if (long.TryParse(quotaValue, out quota))\r
 699                         accountInfo.Quota = quota;\r
 700                     if (long.TryParse(bytesValue, out bytes))\r
 701                         accountInfo.BytesUsed = bytes;\r
 702                     \r
 703                     return accountInfo;\r
 704 \r
 705                 }\r
 706 \r
 707             }\r
 708         }\r
 709 \r
 710         public void UpdateMetadata(ObjectInfo objectInfo)\r
 711         {\r
 712             if (objectInfo == null)\r
 713                 throw new ArgumentNullException("objectInfo");\r
 714             Contract.EndContractBlock();\r
 715 \r
 716             using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))\r
 717             {\r
 718                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 719 \r
 720 \r
 721                 using(var client=new RestClient(_baseClient))\r
 722                 {\r
 723 \r
 724                     client.BaseAddress = GetAccountUrl(objectInfo.Account);\r
 725                     \r
 726                     client.Parameters.Clear();\r
 727                     \r
 728 \r
 729                     //Set Tags\r
 730                     foreach (var tag in objectInfo.Tags)\r
 731                     {\r
 732                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);\r
 733                         client.Headers.Add(headerTag, tag.Value);\r
 734                     }\r
 735 \r
 736                     //Set Permissions\r
 737 \r
 738                     var permissions=objectInfo.GetPermissionString();\r
 739                     client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);\r
 740 \r
 741                     client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);\r
 742                     client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);\r
 743                     client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);\r
 744                     var isPublic = objectInfo.IsPublic.ToString().ToLower();\r
 745                     client.Headers.Add("X-Object-Public", isPublic);\r
 746 \r
 747 \r
 748                     /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);\r
 749                     uriBuilder.Query = "update=";\r
 750                     var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/\r
 751                     var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);\r
 752                     client.PostWithRetry(new Uri(address,UriKind.Relative),"application/xml");\r
 753                     \r
 754                     //client.UploadValues(uri,new NameValueCollection());\r
 755 \r
 756 \r
 757                     client.AssertStatusOK("UpdateMetadata failed");\r
 758                     //If the status is NOT ACCEPTED or OK we have a problem\r
 759                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))\r
 760                     {\r
 761                         Log.Error("Failed to update metadata");\r
 762                         throw new Exception("Failed to update metadata");\r
 763                     }\r
 764 \r
 765                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 766                 }\r
 767             }\r
 768 \r
 769         }\r
 770 \r
 771         public void UpdateMetadata(ContainerInfo containerInfo)\r
 772         {\r
 773             if (containerInfo == null)\r
 774                 throw new ArgumentNullException("containerInfo");\r
 775             Contract.EndContractBlock();\r
 776 \r
 777             using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))\r
 778             {\r
 779                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 780 \r
 781 \r
 782                 using(var client=new RestClient(_baseClient))\r
 783                 {\r
 784 \r
 785                     client.BaseAddress = GetAccountUrl(containerInfo.Account);\r
 786                     \r
 787                     client.Parameters.Clear();\r
 788                     \r
 789 \r
 790                     //Set Tags\r
 791                     foreach (var tag in containerInfo.Tags)\r
 792                     {\r
 793                         var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);\r
 794                         client.Headers.Add(headerTag, tag.Value);\r
 795                     }\r
 796 \r
 797                     \r
 798                     //Set Policies\r
 799                     foreach (var policy in containerInfo.Policies)\r
 800                     {\r
 801                         var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);\r
 802                         client.Headers.Add(headerPolicy, policy.Value);\r
 803                     }\r
 804 \r
 805 \r
 806                     var uriBuilder = client.GetAddressBuilder(containerInfo.Name,_emptyUri);\r
 807                     var uri = uriBuilder.Uri;\r
 808 \r
 809                     client.UploadValues(uri,new NameValueCollection());\r
 810 \r
 811 \r
 812                     client.AssertStatusOK("UpdateMetadata failed");\r
 813                     //If the status is NOT ACCEPTED or OK we have a problem\r
 814                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))\r
 815                     {\r
 816                         Log.Error("Failed to update metadata");\r
 817                         throw new Exception("Failed to update metadata");\r
 818                     }\r
 819 \r
 820                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 821                 }\r
 822             }\r
 823 \r
 824         }\r
 825 \r
 826        \r
 827 \r
 828 \r
 829         public IList<ObjectInfo> ListObjects(string account, Uri container, DateTime? since = null)\r
 830         {\r
 831             if (container==null)\r
 832                 throw new ArgumentNullException("container");\r
 833             if (container.IsAbsoluteUri)\r
 834                 throw new ArgumentException("container");\r
 835             Contract.EndContractBlock();\r
 836 \r
 837             using (ThreadContext.Stacks["Objects"].Push("List"))\r
 838             {\r
 839                 /*if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 840 \r
 841                 var targetUrl = String.IsNullOrWhiteSpace(account)\r
 842                     ? _baseHttpClient.BaseAddress.ToString()\r
 843                     : GetAccountUrl(account);\r
 844                 var targetUri = new Uri(String.Format("{0}?format=json", targetUrl));\r
 845                 var result = GetString(targetUri, "ListObjects failed",since).Result;\r
 846 \r
 847                 if (String.IsNullOrWhiteSpace(result))\r
 848                     return new[]{new NoModificationInfo(account,container)};\r
 849                 //If the result is empty, return an empty list,\r
 850                 var infos = String.IsNullOrWhiteSpace(result)\r
 851                                 ? new List<ObjectInfo>()\r
 852                             //Otherwise deserialize the object list into a list of ObjectInfos\r
 853                                 : JsonConvert.DeserializeObject<IList<ObjectInfo>>(result);\r
 854 \r
 855                 foreach (var info in infos)\r
 856                 {\r
 857                     info.Container = container;\r
 858                     info.Account = account;\r
 859                     info.StorageUri = StorageUrl;\r
 860                 }\r
 861                 if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 862                 return infos;*/\r
 863 \r
 864                 var containerUri = GetTargetUri(account).Combine(container);\r
 865                 var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);\r
 866 \r
 867                 var content =TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result;\r
 868 \r
 869                 //304 will result in an empty string. Empty containers return an empty json array\r
 870                 if (String.IsNullOrWhiteSpace(content))\r
 871                      return new[] {new NoModificationInfo(account, container)};\r
 872 \r
 873                  //If the result is empty, return an empty list,\r
 874                  var infos = String.IsNullOrWhiteSpace(content)\r
 875                                  ? new List<ObjectInfo>()\r
 876                              //Otherwise deserialize the object list into a list of ObjectInfos\r
 877                                  : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 878 \r
 879                  foreach (var info in infos)\r
 880                  {\r
 881                      info.Container = container;\r
 882                      info.Account = account;\r
 883                      info.StorageUri = StorageUrl;\r
 884                  }\r
 885                  if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 886                  return infos;\r
 887 /*\r
 888              using (var client = new RestClient(_baseClient))\r
 889              {\r
 890                  if (!String.IsNullOrWhiteSpace(account))\r
 891                      client.BaseAddress = GetAccountUrl(account);\r
 892 \r
 893                  client.Parameters.Clear();\r
 894                  client.Parameters.Add("format", "json");\r
 895                  client.IfModifiedSince = since;\r
 896                  var content = client.DownloadStringWithRetryRelative(container, 3);\r
 897 \r
 898                  client.AssertStatusOK("ListObjects failed");\r
 899 \r
 900                  if (client.StatusCode == HttpStatusCode.NotModified)\r
 901                      return new[] {new NoModificationInfo(account, container)};\r
 902                  //If the result is empty, return an empty list,\r
 903                  var infos = String.IsNullOrWhiteSpace(content)\r
 904                                  ? new List<ObjectInfo>()\r
 905                              //Otherwise deserialize the object list into a list of ObjectInfos\r
 906                                  : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 907 \r
 908                  foreach (var info in infos)\r
 909                  {\r
 910                      info.Container = container;\r
 911                      info.Account = account;\r
 912                      info.StorageUri = StorageUrl;\r
 913                  }\r
 914                  if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 915                  return infos;\r
 916              }\r
 917 */\r
 918             }\r
 919         }\r
 920 \r
 921         public IList<ObjectInfo> ListObjects(string account, Uri container, Uri folder, DateTime? since = null)\r
 922         {\r
 923             if (container==null)\r
 924                 throw new ArgumentNullException("container");\r
 925             if (container.IsAbsoluteUri)\r
 926                 throw new ArgumentException("container");\r
 927 /*\r
 928             if (String.IsNullOrWhiteSpace(folder))\r
 929                 throw new ArgumentNullException("folder");\r
 930 */\r
 931             Contract.EndContractBlock();\r
 932 \r
 933             using (ThreadContext.Stacks["Objects"].Push("List"))\r
 934             {\r
 935                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 936 \r
 937                 var containerUri = GetTargetUri(account).Combine(container);\r
 938                 var targetUri = new Uri(String.Format("{0}?format=json&path={1}", containerUri,folder), UriKind.Absolute);\r
 939                 var content = TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result;                \r
 940 \r
 941                 //304 will result in an empty string. Empty containers return an empty json array\r
 942                 if (String.IsNullOrWhiteSpace(content))\r
 943                     return new[] { new NoModificationInfo(account, container) };\r
 944 \r
 945 \r
 946                 var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 947                 foreach (var info in infos)\r
 948                 {\r
 949                     info.Account = account;\r
 950                     if (info.Container == null)\r
 951                         info.Container = container;\r
 952                     info.StorageUri = StorageUrl;\r
 953                 }\r
 954                 if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 955                 return infos;\r
 956 /*\r
 957                 using (var client = new RestClient(_baseClient))\r
 958                 {\r
 959                     if (!String.IsNullOrWhiteSpace(account))\r
 960                         client.BaseAddress = GetAccountUrl(account);\r
 961 \r
 962                     client.Parameters.Clear();\r
 963                     client.Parameters.Add("format", "json");\r
 964                     client.Parameters.Add("path", folder.ToString());\r
 965                     client.IfModifiedSince = since;\r
 966                     var content = client.DownloadStringWithRetryRelative(container, 3);\r
 967                     client.AssertStatusOK("ListObjects failed");\r
 968 \r
 969                     if (client.StatusCode==HttpStatusCode.NotModified)\r
 970                         return new[]{new NoModificationInfo(account,container,folder)};\r
 971 \r
 972                     var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 973                     foreach (var info in infos)\r
 974                     {\r
 975                         info.Account = account;\r
 976                         if (info.Container == null)\r
 977                             info.Container = container;\r
 978                         info.StorageUri = StorageUrl;\r
 979                     }\r
 980                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 981                     return infos;\r
 982                 }\r
 983 */\r
 984             }\r
 985         }\r
 986 \r
 987  \r
 988         public bool ContainerExists(string account, Uri container)\r
 989         {\r
 990             if (container==null)\r
 991                 throw new ArgumentNullException("container", "The container property can't be empty");\r
 992             if (container.IsAbsoluteUri)\r
 993                 throw new ArgumentException( "The container must be relative","container");\r
 994             Contract.EndContractBlock();\r
 995 \r
 996             using (ThreadContext.Stacks["Containters"].Push("Exists"))\r
 997             {\r
 998                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 999 \r
1000                 var targetUri = GetTargetUri(account).Combine(container);\r
1001 \r
1002                 using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result)\r
1003                 {\r
1004 \r
1005                     bool result;\r
1006                     switch (response.StatusCode)\r
1007                     {\r
1008                         case HttpStatusCode.OK:\r
1009                         case HttpStatusCode.NoContent:\r
1010                             result = true;\r
1011                             break;\r
1012                         case HttpStatusCode.NotFound:\r
1013                             result = false;\r
1014                             break;\r
1015                         default:\r
1016                             throw CreateWebException("ContainerExists", response.StatusCode);\r
1017                     }\r
1018                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
1019 \r
1020                     return result;\r
1021                 }\r
1022 /*\r
1023                 using (var client = new RestClient(_baseClient))\r
1024                 {\r
1025                     if (!String.IsNullOrWhiteSpace(account))\r
1026                         client.BaseAddress = GetAccountUrl(account);\r
1027 \r
1028                     client.Parameters.Clear();\r
1029                     client.Head(container, 3);\r
1030                                         \r
1031                     bool result;\r
1032                     switch (client.StatusCode)\r
1033                     {\r
1034                         case HttpStatusCode.OK:\r
1035                         case HttpStatusCode.NoContent:\r
1036                             result=true;\r
1037                             break;\r
1038                         case HttpStatusCode.NotFound:\r
1039                             result=false;\r
1040                             break;\r
1041                         default:\r
1042                             throw CreateWebException("ContainerExists", client.StatusCode);\r
1043                     }\r
1044                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
1045 \r
1046                     return result;\r
1047                 }\r
1048 */\r
1049                 \r
1050             }\r
1051         }\r
1052 \r
1053         private Uri GetTargetUri(string account)\r
1054         {\r
1055             return new Uri(GetTargetUrl(account),UriKind.Absolute);\r
1056         }\r
1057 \r
1058         private string GetTargetUrl(string account)\r
1059         {\r
1060             return String.IsNullOrWhiteSpace(account)\r
1061                        ? _baseHttpClient.BaseAddress.ToString()\r
1062                        : GetAccountUrl(account);\r
1063         }\r
1064 \r
1065         public bool ObjectExists(string account, Uri container, Uri objectName)\r
1066         {\r
1067             if (container == null)\r
1068                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1069             if (container.IsAbsoluteUri)\r
1070                 throw new ArgumentException("The container must be relative","container");\r
1071             if (objectName == null)\r
1072                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1073             if (objectName.IsAbsoluteUri)\r
1074                 throw new ArgumentException("The objectName must be relative","objectName");\r
1075             Contract.EndContractBlock();\r
1076 \r
1077                 var targetUri=GetTargetUri(account).Combine(container).Combine(objectName);\r
1078 \r
1079             using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result)\r
1080             {\r
1081                 switch (response.StatusCode)\r
1082                 {\r
1083                     case HttpStatusCode.OK:\r
1084                     case HttpStatusCode.NoContent:\r
1085                         return true;\r
1086                     case HttpStatusCode.NotFound:\r
1087                         return false;\r
1088                     default:\r
1089                         throw CreateWebException("ObjectExists", response.StatusCode);\r
1090                 }\r
1091             }\r
1092 \r
1093 /*\r
1094             using (var client = new RestClient(_baseClient))\r
1095             {\r
1096                 if (!String.IsNullOrWhiteSpace(account))\r
1097                     client.BaseAddress = GetAccountUrl(account);\r
1098 \r
1099                 client.Parameters.Clear();\r
1100                 client.Head(container.Combine(objectName), 3);\r
1101 \r
1102                 switch (client.StatusCode)\r
1103                 {\r
1104                     case HttpStatusCode.OK:\r
1105                     case HttpStatusCode.NoContent:\r
1106                         return true;\r
1107                     case HttpStatusCode.NotFound:\r
1108                         return false;\r
1109                     default:\r
1110                         throw CreateWebException("ObjectExists", client.StatusCode);\r
1111                 }\r
1112             }\r
1113 */\r
1114 \r
1115         }\r
1116 \r
1117         public ObjectInfo GetObjectInfo(string account, Uri container, Uri objectName)\r
1118         {\r
1119             if (container == null)\r
1120                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1121             if (container.IsAbsoluteUri)\r
1122                 throw new ArgumentException("The container must be relative","container");\r
1123             if (objectName == null)\r
1124                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1125             if (objectName.IsAbsoluteUri)\r
1126                 throw new ArgumentException("The objectName must be relative","objectName");\r
1127             Contract.EndContractBlock();\r
1128 \r
1129             using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))\r
1130             {                \r
1131 \r
1132                 using (var client = new RestClient(_baseClient))\r
1133                 {\r
1134                     if (!String.IsNullOrWhiteSpace(account))\r
1135                         client.BaseAddress = GetAccountUrl(account);\r
1136                     try\r
1137                     {\r
1138                         client.Parameters.Clear();\r
1139 \r
1140                         client.Head(container.Combine(objectName), 3);\r
1141 \r
1142                         if (client.TimedOut)\r
1143                             return ObjectInfo.Empty;\r
1144 \r
1145                         switch (client.StatusCode)\r
1146                         {\r
1147                             case HttpStatusCode.OK:\r
1148                             case HttpStatusCode.NoContent:\r
1149                                 var keys = client.ResponseHeaders.AllKeys.AsQueryable();\r
1150                                 var tags = client.GetMeta("X-Object-Meta-");\r
1151                                 var extensions = (from key in keys\r
1152                                                   where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")\r
1153                                                   select new {Name = key, Value = client.ResponseHeaders[key]})\r
1154                                     .ToDictionary(t => t.Name, t => t.Value);\r
1155 \r
1156                                 var permissions=client.GetHeaderValue("X-Object-Sharing", true);\r
1157                                 \r
1158                                 \r
1159                                 var info = new ObjectInfo\r
1160                                                {\r
1161                                                    Account = account,\r
1162                                                    Container = container,\r
1163                                                    Name = objectName,\r
1164                                                    ETag = client.GetHeaderValue("ETag"),\r
1165                                                    UUID=client.GetHeaderValue("X-Object-UUID"),\r
1166                                                    X_Object_Hash = client.GetHeaderValue("X-Object-Hash"),\r
1167                                                    Content_Type = client.GetHeaderValue("Content-Type"),\r
1168                                                    Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),\r
1169                                                    Tags = tags,\r
1170                                                    Last_Modified = client.LastModified,\r
1171                                                    Extensions = extensions,\r
1172                                                    ContentEncoding=client.GetHeaderValue("Content-Encoding",true),\r
1173                                                    ContendDisposition = client.GetHeaderValue("Content-Disposition",true),\r
1174                                                    Manifest=client.GetHeaderValue("X-Object-Manifest",true),\r
1175                                                    PublicUrl=client.GetHeaderValue("X-Object-Public",true),  \r
1176                                                    StorageUri=StorageUrl,\r
1177                                                };\r
1178                                 info.SetPermissions(permissions);\r
1179                                 return info;\r
1180                             case HttpStatusCode.NotFound:\r
1181                                 return ObjectInfo.Empty;\r
1182                             default:\r
1183                                 throw new WebException(\r
1184                                     String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",\r
1185                                                   objectName, client.StatusCode));\r
1186                         }\r
1187 \r
1188                     }\r
1189                     catch (RetryException)\r
1190                     {\r
1191                         Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);\r
1192                         return ObjectInfo.Empty;\r
1193                     }\r
1194                     catch (WebException e)\r
1195                     {\r
1196                         Log.Error(\r
1197                             String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",\r
1198                                           objectName, client.StatusCode), e);\r
1199                         throw;\r
1200                     }\r
1201                 }                \r
1202             }\r
1203 \r
1204         }\r
1205 \r
1206         \r
1207 \r
1208         public void CreateFolder(string account, Uri container, Uri folder)\r
1209         {\r
1210             if (container == null)\r
1211                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1212             if (container.IsAbsoluteUri)\r
1213                 throw new ArgumentException("The container must be relative","container");\r
1214             if (folder == null)\r
1215                 throw new ArgumentNullException("folder", "The objectName property can't be empty");\r
1216             if (folder.IsAbsoluteUri)\r
1217                 throw new ArgumentException("The objectName must be relative","folder");\r
1218             Contract.EndContractBlock();\r
1219 \r
1220             var folderUri=container.Combine(folder);            \r
1221             var targetUri = GetTargetUri(account).Combine(folderUri);\r
1222             var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
1223             \r
1224             message.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY);\r
1225             message.Headers.Add("Content-Length", "0");\r
1226             using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)\r
1227             {\r
1228                 if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)\r
1229                     throw CreateWebException("CreateFolder", response.StatusCode);\r
1230             }\r
1231 /*\r
1232             using (var client = new RestClient(_baseClient))\r
1233             {\r
1234                 if (!String.IsNullOrWhiteSpace(account))\r
1235                     client.BaseAddress = GetAccountUrl(account);\r
1236 \r
1237                 client.Parameters.Clear();\r
1238                 client.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY);\r
1239                 client.Headers.Add("Content-Length", "0");\r
1240                 client.PutWithRetry(folderUri, 3);\r
1241 \r
1242                 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)\r
1243                     throw CreateWebException("CreateFolder", client.StatusCode);\r
1244             }\r
1245 */\r
1246         }\r
1247 \r
1248      \r
1249 \r
1250         public ContainerInfo GetContainerInfo(string account, Uri container)\r
1251         {\r
1252             if (container == null)\r
1253                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1254             if (container.IsAbsoluteUri)\r
1255                 throw new ArgumentException("The container must be relative","container");\r
1256             Contract.EndContractBlock();\r
1257 \r
1258             using (var client = new RestClient(_baseClient))\r
1259             {\r
1260                 if (!String.IsNullOrWhiteSpace(account))\r
1261                     client.BaseAddress = GetAccountUrl(account);                \r
1262 \r
1263                 client.Head(container);\r
1264                 switch (client.StatusCode)\r
1265                 {\r
1266                     case HttpStatusCode.OK:\r
1267                     case HttpStatusCode.NoContent:\r
1268                         var tags = client.GetMeta("X-Container-Meta-");\r
1269                         var policies = client.GetMeta("X-Container-Policy-");\r
1270 \r
1271                         var containerInfo = new ContainerInfo\r
1272                                                 {\r
1273                                                     Account=account,\r
1274                                                     Name = container,\r
1275                                                     StorageUrl=StorageUrl.ToString(),\r
1276                                                     Count =\r
1277                                                         long.Parse(client.GetHeaderValue("X-Container-Object-Count")),\r
1278                                                     Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),\r
1279                                                     BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),\r
1280                                                     BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),\r
1281                                                     Last_Modified=client.LastModified,\r
1282                                                     Tags=tags,\r
1283                                                     Policies=policies\r
1284                                                 };\r
1285                         \r
1286 \r
1287                         return containerInfo;\r
1288                     case HttpStatusCode.NotFound:\r
1289                         return ContainerInfo.Empty;\r
1290                     default:\r
1291                         throw CreateWebException("GetContainerInfo", client.StatusCode);\r
1292                 }\r
1293             }\r
1294         }\r
1295 \r
1296         public void CreateContainer(string account, Uri container)\r
1297         {\r
1298             if (container == null)\r
1299                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1300             if (container.IsAbsoluteUri)\r
1301                 throw new ArgumentException("The container must be relative","container");\r
1302             Contract.EndContractBlock();\r
1303 \r
1304             var targetUri=GetTargetUri(account).Combine(container);\r
1305             var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
1306             message.Headers.Add("Content-Length", "0");\r
1307             using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)\r
1308             {            \r
1309                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};\r
1310                 if (!expectedCodes.Contains(response.StatusCode))\r
1311                     throw CreateWebException("CreateContainer", response.StatusCode);\r
1312             }\r
1313 /*\r
1314             using (var client = new RestClient(_baseClient))\r
1315             {\r
1316                 if (!String.IsNullOrWhiteSpace(account))\r
1317                     client.BaseAddress = GetAccountUrl(account);\r
1318 \r
1319                 client.PutWithRetry(container, 3);\r
1320                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};\r
1321                 if (!expectedCodes.Contains(client.StatusCode))\r
1322                     throw CreateWebException("CreateContainer", client.StatusCode);\r
1323             }\r
1324 */\r
1325         }\r
1326 \r
1327         public void WipeContainer(string account, Uri container)\r
1328         {\r
1329             if (container == null)\r
1330                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1331             if (container.IsAbsoluteUri)\r
1332                 throw new ArgumentException("The container must be relative", "container");\r
1333             Contract.EndContractBlock();\r
1334 \r
1335             DeleteContainer(account, new Uri(String.Format("{0}&delimiter=//", container), UriKind.Relative));\r
1336         }\r
1337 \r
1338 \r
1339         public void DeleteContainer(string account, Uri container)\r
1340         {\r
1341             if (container == null)\r
1342                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1343             if (container.IsAbsoluteUri)\r
1344                 throw new ArgumentException("The container must be relative","container");\r
1345             Contract.EndContractBlock();\r
1346 \r
1347             var targetUri = GetTargetUri(account).Combine(container);\r
1348             var message = new HttpRequestMessage(HttpMethod.Delete, targetUri);\r
1349             using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)\r
1350             {\r
1351                 var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent };\r
1352                 if (!expectedCodes.Contains(response.StatusCode))\r
1353                     throw CreateWebException("DeleteContainer", response.StatusCode);\r
1354             }\r
1355 /*\r
1356             using (var client = new RestClient(_baseClient))\r
1357             {\r
1358                 if (!String.IsNullOrWhiteSpace(account))\r
1359                     client.BaseAddress = GetAccountUrl(account);\r
1360 \r
1361                 client.DeleteWithRetry(container, 3);\r
1362                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};\r
1363                 if (!expectedCodes.Contains(client.StatusCode))\r
1364                     throw CreateWebException("DeleteContainer", client.StatusCode);\r
1365             }\r
1366 */\r
1367 \r
1368         }\r
1369 \r
1370         /// <summary>\r
1371         /// \r
1372         /// </summary>\r
1373         /// <param name="account"></param>\r
1374         /// <param name="container"></param>\r
1375         /// <param name="objectName"></param>\r
1376         /// <param name="fileName"></param>\r
1377         /// <param name="cancellationToken"> </param>\r
1378         /// <returns></returns>\r
1379         /// <remarks>This method should have no timeout or a very long one</remarks>\r
1380         //Asynchronously download the object specified by *objectName* in a specific *container* to \r
1381         // a local file\r
1382         public async Task GetObject(string account, Uri container, Uri objectName, string fileName,CancellationToken cancellationToken)\r
1383         {\r
1384             if (container == null)\r
1385                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1386             if (container.IsAbsoluteUri)\r
1387                 throw new ArgumentException("The container must be relative","container");\r
1388             if (objectName == null)\r
1389                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1390             if (objectName.IsAbsoluteUri)\r
1391                 throw new ArgumentException("The objectName must be relative","objectName");\r
1392             Contract.EndContractBlock();\r
1393                         \r
1394 \r
1395             try\r
1396             {\r
1397                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient\r
1398                 //object to avoid concurrency errors.\r
1399                 //\r
1400                 //Download operations take a long time therefore they have no timeout.\r
1401                 using(var client = new RestClient(_baseClient) { Timeout = 0 })\r
1402                 {\r
1403                     if (!String.IsNullOrWhiteSpace(account))\r
1404                         client.BaseAddress = GetAccountUrl(account);\r
1405 \r
1406                     //The container and objectName are relative names. They are joined with the client's\r
1407                     //BaseAddress to create the object's absolute address\r
1408                     var builder = client.GetAddressBuilder(container, objectName);\r
1409                     var uri = builder.Uri;\r
1410 \r
1411                     //Download progress is reported to the Trace log\r
1412                     Log.InfoFormat("[GET] START {0}", objectName);\r
1413                     /*client.DownloadProgressChanged += (sender, args) =>\r
1414                                                       Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1415                                                                      fileName, args.ProgressPercentage,\r
1416                                                                      args.BytesReceived,\r
1417                                                                      args.TotalBytesToReceive);*/\r
1418                     var progress = new Progress<DownloadProgressChangedEventArgs>(args =>\r
1419                                 {\r
1420                                     Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1421                                                    fileName, args.ProgressPercentage,\r
1422                                                    args.BytesReceived,\r
1423                                                    args.TotalBytesToReceive);\r
1424                                     if (DownloadProgressChanged!=null)\r
1425                                         DownloadProgressChanged(this, new DownloadArgs(args));\r
1426                                 });\r
1427                     \r
1428                     //Start downloading the object asynchronously                    \r
1429                     await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false);\r
1430 \r
1431                     //Once the download completes\r
1432                     //Delete the local client object\r
1433                 }\r
1434                 //And report failure or completion\r
1435             }\r
1436             catch (Exception exc)\r
1437             {\r
1438                 Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);\r
1439                 throw;\r
1440             }\r
1441 \r
1442             Log.InfoFormat("[GET] END {0}", objectName);                                             \r
1443 \r
1444 \r
1445         }\r
1446 \r
1447         public async Task<IList<string>> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash)\r
1448         {\r
1449             if (container == null)\r
1450                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1451             if (container.IsAbsoluteUri)\r
1452                 throw new ArgumentException("The container must be relative","container");\r
1453             if (objectName == null)\r
1454                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1455             if (objectName.IsAbsoluteUri)\r
1456                 throw new ArgumentException("The objectName must be relative","objectName");\r
1457             if (hash == null)\r
1458                 throw new ArgumentNullException("hash");\r
1459             if (String.IsNullOrWhiteSpace(Token))\r
1460                 throw new InvalidOperationException("Invalid Token");\r
1461             if (StorageUrl == null)\r
1462                 throw new InvalidOperationException("Invalid Storage Url");\r
1463             Contract.EndContractBlock();\r
1464 \r
1465             \r
1466 \r
1467             //The container and objectName are relative names. They are joined with the client's\r
1468             //BaseAddress to create the object's absolute address\r
1469 \r
1470             var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);\r
1471   \r
1472 \r
1473             var uri = new Uri(String.Format("{0}?format=json&hashmap",targetUri),UriKind.Absolute);\r
1474 \r
1475             \r
1476             //Send the tree hash as Json to the server            \r
1477             var jsonHash = hash.ToJson();\r
1478             if (Log.IsDebugEnabled)\r
1479                 Log.DebugFormat("Hashes:\r\n{0}", jsonHash);\r
1480 \r
1481             var message = new HttpRequestMessage(HttpMethod.Put, uri)\r
1482             {\r
1483                 Content = new StringContent(jsonHash)\r
1484             };\r
1485             message.Headers.Add("ETag",hash.TopHash.ToHashString());\r
1486             \r
1487             //Don't use a timeout because putting the hashmap may be a long process\r
1488 \r
1489             using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false))\r
1490             {\r
1491                 var empty = (IList<string>)new List<string>();\r
1492                 \r
1493                 switch (response.StatusCode)\r
1494                 {\r
1495                     case HttpStatusCode.Created:\r
1496                         //The server will respond either with 201-created if all blocks were already on the server\r
1497                         return empty;\r
1498                     case HttpStatusCode.Conflict:\r
1499                         //or with a 409-conflict and return the list of missing parts\r
1500                         using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))\r
1501                         using(var reader=stream.GetLoggedReader(Log))\r
1502                         {                            \r
1503                             var serializer = new JsonSerializer();                            \r
1504                             serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);\r
1505                             var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));\r
1506                             return hashes;\r
1507                         }                        \r
1508                     default:\r
1509                         //All other cases are unexpected\r
1510                         //Ensure that failure codes raise exceptions\r
1511                         response.EnsureSuccessStatusCode();\r
1512                         //And log any other codes as warngings, but continute processing\r
1513                         Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",response.StatusCode,response.ReasonPhrase);\r
1514                         return empty;\r
1515                 }\r
1516             }\r
1517 \r
1518         }\r
1519 \r
1520 /*\r
1521         public Task<IList<string>> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash)\r
1522         {\r
1523             if (container == null)\r
1524                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1525             if (container.IsAbsoluteUri)\r
1526                 throw new ArgumentException("The container must be relative","container");\r
1527             if (objectName == null)\r
1528                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1529             if (objectName.IsAbsoluteUri)\r
1530                 throw new ArgumentException("The objectName must be relative","objectName");\r
1531             if (hash == null)\r
1532                 throw new ArgumentNullException("hash");\r
1533             if (String.IsNullOrWhiteSpace(Token))\r
1534                 throw new InvalidOperationException("Invalid Token");\r
1535             if (StorageUrl == null)\r
1536                 throw new InvalidOperationException("Invalid Storage Url");\r
1537             Contract.EndContractBlock();\r
1538 \r
1539             \r
1540 \r
1541             //Don't use a timeout because putting the hashmap may be a long process\r
1542             var client = new RestClient(_baseClient) { Timeout = 0 };           \r
1543             if (!String.IsNullOrWhiteSpace(account))\r
1544                 client.BaseAddress = GetAccountUrl(account);\r
1545 \r
1546             //The container and objectName are relative names. They are joined with the client's\r
1547             //BaseAddress to create the object's absolute address\r
1548             var builder = client.GetAddressBuilder(container, objectName);\r
1549             builder.Query = "format=json&hashmap";\r
1550             var uri = builder.Uri;\r
1551 \r
1552 \r
1553             //Send the tree hash as Json to the server            \r
1554             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";\r
1555             var jsonHash = hash.ToJson();\r
1556                         \r
1557             client.Headers.Add("ETag",hash.TopHash.ToHashString());\r
1558             var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);\r
1559             if (Log.IsDebugEnabled)\r
1560                 Log.DebugFormat("Hashes:\r\n{0}", jsonHash);\r
1561             return uploadTask.ContinueWith(t =>\r
1562             {\r
1563 \r
1564                 var empty = (IList<string>)new List<string>();\r
1565                 \r
1566 \r
1567                 //The server will respond either with 201-created if all blocks were already on the server\r
1568                 if (client.StatusCode == HttpStatusCode.Created)                    \r
1569                 {\r
1570                     //in which case we return an empty hash list\r
1571                     return empty;\r
1572                 }\r
1573                 //or with a 409-conflict and return the list of missing parts\r
1574                 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception                \r
1575                 if (t.IsFaulted)\r
1576                 {\r
1577                     var ex = t.Exception.InnerException;\r
1578                     var we = ex as WebException;\r
1579                     var response = we.Response as HttpWebResponse;\r
1580                     if (response!=null && response.StatusCode==HttpStatusCode.Conflict)\r
1581                     {\r
1582                         //In case of 409 the missing parts will be in the response content                        \r
1583                         using (var stream = response.GetResponseStream())\r
1584                         using(var reader=stream.GetLoggedReader(Log))\r
1585                         {\r
1586                             //We used to have to cleanup the content before returning it because it contains\r
1587                             //error content after the list of hashes\r
1588                             //\r
1589                             //As of 30/1/2012, the result is a proper Json array so we don't need to read the content\r
1590                             //line by line\r
1591                             \r
1592                             var serializer = new JsonSerializer();                            \r
1593                             serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);\r
1594                             var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));\r
1595                             return hashes;\r
1596                         }                        \r
1597                     }                    \r
1598                     //Any other status code is unexpected and the exception should be rethrown\r
1599                     Log.LogError(response);\r
1600                     throw ex;\r
1601                     \r
1602                 }\r
1603 \r
1604                 //Any other status code is unexpected but there was no exception. We can probably continue processing\r
1605                 Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);                    \r
1606                 \r
1607                 return empty;\r
1608             });\r
1609 \r
1610         }\r
1611 \r
1612 */\r
1613 \r
1614        \r
1615 \r
1616         public async Task<byte[]> GetBlock(string account, Uri container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)\r
1617         {\r
1618             if (String.IsNullOrWhiteSpace(Token))\r
1619                 throw new InvalidOperationException("Invalid Token");\r
1620             if (StorageUrl == null)\r
1621                 throw new InvalidOperationException("Invalid Storage Url");\r
1622             if (container == null)\r
1623                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1624             if (container.IsAbsoluteUri)\r
1625                 throw new ArgumentException("The container must be relative","container");\r
1626             if (relativeUrl == null)\r
1627                 throw new ArgumentNullException("relativeUrl");\r
1628             if (end.HasValue && end < 0)\r
1629                 throw new ArgumentOutOfRangeException("end");\r
1630             if (start < 0)\r
1631                 throw new ArgumentOutOfRangeException("start");\r
1632             Contract.EndContractBlock();\r
1633 \r
1634 \r
1635             var targetUri = GetTargetUri(account).Combine(container).Combine(relativeUrl);\r
1636             var message = new HttpRequestMessage(HttpMethod.Get, targetUri);\r
1637             message.Headers.Range.Ranges.Add(new RangeItemHeaderValue(start,end));\r
1638 \r
1639             //Don't use a timeout because putting the hashmap may be a long process\r
1640 \r
1641             IProgress<DownloadArgs> progress = new Progress<DownloadArgs>(args =>\r
1642                 {\r
1643                     Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1644                                     targetUri.Segments.Last(), args.ProgressPercentage,\r
1645                                     args.BytesReceived,\r
1646                                     args.TotalBytesToReceive);\r
1647 \r
1648                     if (DownloadProgressChanged!=null)\r
1649                         DownloadProgressChanged(this,  args);\r
1650                 });\r
1651 \r
1652 \r
1653             using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, HttpCompletionOption.ResponseHeadersRead,\r
1654                                                           cancellationToken).ConfigureAwait(false))\r
1655             using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))\r
1656             using(var targetStream=new MemoryStream())\r
1657             {\r
1658                 \r
1659                 long totalSize = response.Content.Headers.ContentLength ?? 0;\r
1660                 long total = 0;\r
1661                 var buffer = new byte[65536];\r
1662                 int read;\r
1663                 while ((read = await stream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) != 0)\r
1664                 {\r
1665                     total += read;\r
1666                     progress.Report(new DownloadArgs(total, totalSize));\r
1667                     await targetStream.WriteAsync(buffer, 0, read).ConfigureAwait(false);\r
1668                 }\r
1669 \r
1670                 var result = targetStream.ToArray();\r
1671                 return result;\r
1672             }\r
1673             /*using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end})\r
1674             {\r
1675                 if (!String.IsNullOrWhiteSpace(account))\r
1676                     client.BaseAddress = GetAccountUrl(account);\r
1677 \r
1678                 var builder = client.GetAddressBuilder(container, relativeUrl);\r
1679                 var uri = builder.Uri;\r
1680 \r
1681 /*                client.DownloadProgressChanged += (sender, args) =>\r
1682                                                       {\r
1683                                                           Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1684                                                                           uri.Segments.Last(), args.ProgressPercentage,\r
1685                                                                           args.BytesReceived,\r
1686                                                                           args.TotalBytesToReceive);\r
1687                                                           DownloadProgressChanged(sender, args);\r
1688                                                       };#1#\r
1689                 var progress = new Progress<DownloadProgressChangedEventArgs>(args =>\r
1690                 {\r
1691                     Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1692                                     uri.Segments.Last(), args.ProgressPercentage,\r
1693                                     args.BytesReceived,\r
1694                                     args.TotalBytesToReceive);\r
1695                     if (DownloadProgressChanged!=null)\r
1696                         DownloadProgressChanged(this, args);\r
1697                 });\r
1698 \r
1699 \r
1700                 var result = await client.DownloadDataTaskAsync(uri, cancellationToken,progress).ConfigureAwait(false);\r
1701                 return result;\r
1702             }*/\r
1703         }\r
1704 \r
1705         public event EventHandler<UploadArgs> UploadProgressChanged;\r
1706         public event EventHandler<DownloadArgs> DownloadProgressChanged;\r
1707         \r
1708 \r
1709         public async Task PostBlock(string account, Uri container, byte[] block, int offset, int count,CancellationToken token)\r
1710         {\r
1711             if (container == null)\r
1712                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1713             if (container.IsAbsoluteUri)\r
1714                 throw new ArgumentException("The container must be relative","container");\r
1715             if (block == null)\r
1716                 throw new ArgumentNullException("block");\r
1717             if (offset < 0 || offset >= block.Length)\r
1718                 throw new ArgumentOutOfRangeException("offset");\r
1719             if (count < 0 || count > block.Length)\r
1720                 throw new ArgumentOutOfRangeException("count");\r
1721             if (String.IsNullOrWhiteSpace(Token))\r
1722                 throw new InvalidOperationException("Invalid Token");\r
1723             if (StorageUrl == null)\r
1724                 throw new InvalidOperationException("Invalid Storage Url");                        \r
1725             Contract.EndContractBlock();\r
1726 \r
1727 \r
1728             try\r
1729             {\r
1730 /*\r
1731                 var containerUri = GetTargetUri(account).Combine(container);\r
1732                 var targetUri = new Uri(String.Format("{0}?update", containerUri));\r
1733 \r
1734 \r
1735                 //Don't use a timeout because putting the hashmap may be a long process\r
1736 \r
1737 \r
1738                 Log.InfoFormat("[BLOCK POST] START");\r
1739 \r
1740 \r
1741                 var progress = new Progress<UploadArgs>(args =>\r
1742                 {\r
1743                     Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",\r
1744                         args.ProgressPercentage,\r
1745                         args.BytesSent,\r
1746                         args.TotalBytesToSend);\r
1747                     if (UploadProgressChanged != null)\r
1748                         UploadProgressChanged(this,args);\r
1749                 });\r
1750 \r
1751                 var message = new HttpRequestMessage(HttpMethod.Post, targetUri);                \r
1752                 message.Content = new  ByteArrayContent(block, offset, count);\r
1753 \r
1754                 //Send the block\r
1755                 using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false))\r
1756                 {                    \r
1757                     Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");\r
1758                     response.EnsureSuccessStatusCode();\r
1759                 }\r
1760                 Log.InfoFormat("[BLOCK POST] END");*/\r
1761                 using (var client = new RestClient(_baseClient) { Timeout = 0 })\r
1762                 {\r
1763                     if (!String.IsNullOrWhiteSpace(account))\r
1764                         client.BaseAddress = GetAccountUrl(account);\r
1765 \r
1766                     var builder = client.GetAddressBuilder(container, _emptyUri);\r
1767                     //We are doing an update\r
1768                     builder.Query = "update";\r
1769                     var uri = builder.Uri;\r
1770 \r
1771                     client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";\r
1772 \r
1773                     Log.InfoFormat("[BLOCK POST] START");\r
1774 \r
1775                     client.UploadFileCompleted += (sender, args) =>\r
1776                                                   Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");\r
1777 \r
1778                     var progress=new Progress<UploadProgressChangedEventArgs>(args=>\r
1779                     {\r
1780                         Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",\r
1781                                         args.ProgressPercentage, args.BytesSent,\r
1782                                         args.TotalBytesToSend);\r
1783                         if (UploadProgressChanged!=null)\r
1784                             UploadProgressChanged(this, new UploadArgs(args));                                                                                      \r
1785                     });\r
1786                     var buffer = new byte[count];\r
1787                     Buffer.BlockCopy(block, offset, buffer, 0, count);\r
1788                     //Send the block\r
1789                     await client.UploadDataTaskAsync(uri, "POST", buffer,token,progress).ConfigureAwait(false);\r
1790                     Log.InfoFormat("[BLOCK POST] END");\r
1791                 }\r
1792             }\r
1793             catch (TaskCanceledException )\r
1794             {\r
1795                 Log.Info("Aborting block");\r
1796                 throw;\r
1797             }\r
1798             catch (Exception exc)\r
1799             {\r
1800                 Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);\r
1801                 throw;\r
1802             }\r
1803         }\r
1804 \r
1805 \r
1806         public async Task<TreeHash> GetHashMap(string account, Uri container, Uri objectName)\r
1807         {\r
1808             if (container == null)\r
1809                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1810             if (container.IsAbsoluteUri)\r
1811                 throw new ArgumentException("The container must be relative","container");\r
1812             if (objectName == null)\r
1813                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1814             if (objectName.IsAbsoluteUri)\r
1815                 throw new ArgumentException("The objectName must be relative","objectName");\r
1816             if (String.IsNullOrWhiteSpace(Token))\r
1817                 throw new InvalidOperationException("Invalid Token");\r
1818             if (StorageUrl == null)\r
1819                 throw new InvalidOperationException("Invalid Storage Url");\r
1820             Contract.EndContractBlock();\r
1821 \r
1822             try\r
1823             {\r
1824 \r
1825                 var objectUri = GetTargetUri(account).Combine(container).Combine(objectName);\r
1826                 var targetUri = new Uri(String.Format("{0}?format=json&hashmap", objectUri));\r
1827 \r
1828                 //Start downloading the object asynchronously\r
1829                 var json = await GetStringAsync(targetUri, "").ConfigureAwait(false);\r
1830                 var treeHash = TreeHash.Parse(json);\r
1831                 Log.InfoFormat("[GET HASH] END {0}", objectName);\r
1832                 return treeHash;\r
1833                 /*\r
1834                                 using (var client = new RestClient(_baseClient) { Timeout = 0 })\r
1835                                 {\r
1836                                     if (!String.IsNullOrWhiteSpace(account))\r
1837                                         client.BaseAddress = GetAccountUrl(account);\r
1838 \r
1839                                     //The container and objectName are relative names. They are joined with the client's\r
1840                                     //BaseAddress to create the object's absolute address\r
1841                     \r
1842                                     var builder = client.GetAddressBuilder(container, objectName);\r
1843                                     builder.Query = "format=json&hashmap";\r
1844                                     var uri = builder.Uri;\r
1845 \r
1846 \r
1847                                     var json = await client.DownloadStringTaskAsync(uri).ConfigureAwait(false);\r
1848                                     var treeHash = TreeHash.Parse(json);\r
1849                                     Log.InfoFormat("[GET HASH] END {0}", objectName);\r
1850                                     return treeHash;\r
1851                                 }\r
1852                 */\r
1853             }\r
1854             catch (Exception exc)\r
1855             {\r
1856                 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);\r
1857                 throw;\r
1858             }\r
1859 \r
1860         }\r
1861 \r
1862 \r
1863         /// <summary>\r
1864         /// \r
1865         /// </summary>\r
1866         /// <param name="account"></param>\r
1867         /// <param name="container"></param>\r
1868         /// <param name="objectName"></param>\r
1869         /// <param name="fileName"></param>\r
1870         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>\r
1871         /// <param name="contentType"> </param>\r
1872         /// <remarks>>This method should have no timeout or a very long one</remarks>\r
1873         public async Task PutObject(string account, Uri container, Uri objectName, string fileName, string hash = Signature.MERKLE_EMPTY, string contentType = "application/octet-stream")\r
1874         {\r
1875             if (container == null)\r
1876                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1877             if (container.IsAbsoluteUri)\r
1878                 throw new ArgumentException("The container must be relative","container");\r
1879             if (objectName == null)\r
1880                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1881             if (objectName.IsAbsoluteUri)\r
1882                 throw new ArgumentException("The objectName must be relative","objectName");\r
1883             if (String.IsNullOrWhiteSpace(fileName))\r
1884                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");\r
1885 /*\r
1886             if (!File.Exists(fileName) && !Directory.Exists(fileName))\r
1887                 throw new FileNotFoundException("The file or directory does not exist",fileName);\r
1888 */\r
1889             \r
1890             try\r
1891             {\r
1892 \r
1893                 using (var client = new RestClient(_baseClient) { Timeout = 0 })\r
1894                 {\r
1895                     if (!String.IsNullOrWhiteSpace(account))\r
1896                         client.BaseAddress = GetAccountUrl(account);\r
1897 \r
1898                     var builder = client.GetAddressBuilder(container, objectName);\r
1899                     var uri = builder.Uri;\r
1900 \r
1901                     string etag = hash ;\r
1902 \r
1903                     client.Headers.Add("Content-Type", contentType);\r
1904                     if (contentType!=ObjectInfo.CONTENT_TYPE_DIRECTORY)\r
1905                         client.Headers.Add("ETag", etag);\r
1906 \r
1907 \r
1908                     Log.InfoFormat("[PUT] START {0}", objectName);\r
1909                     client.UploadProgressChanged += (sender, args) =>\r
1910                                                         {\r
1911                                                             using (ThreadContext.Stacks["PUT"].Push("Progress"))\r
1912                                                             {\r
1913                                                                 Log.InfoFormat("{0} {1}% {2} of {3}", fileName,\r
1914                                                                                args.ProgressPercentage,\r
1915                                                                                args.BytesSent, args.TotalBytesToSend);\r
1916                                                             }\r
1917                                                         };\r
1918 \r
1919                     client.UploadFileCompleted += (sender, args) =>\r
1920                                                       {\r
1921                                                           using (ThreadContext.Stacks["PUT"].Push("Progress"))\r
1922                                                           {\r
1923                                                               Log.InfoFormat("Completed {0}", fileName);\r
1924                                                           }\r
1925                                                       }; \r
1926                     \r
1927                     if (contentType==ObjectInfo.CONTENT_TYPE_DIRECTORY)\r
1928                         await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false);\r
1929                     else\r
1930                         await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false);\r
1931                 }\r
1932 \r
1933                 Log.InfoFormat("[PUT] END {0}", objectName);\r
1934             }\r
1935             catch (Exception exc)\r
1936             {\r
1937                 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);\r
1938                 throw;\r
1939             }                \r
1940 \r
1941         }\r
1942        \r
1943         \r
1944 /*\r
1945         private static string CalculateHash(string fileName)\r
1946         {\r
1947             Contract.Requires(!String.IsNullOrWhiteSpace(fileName));\r
1948             Contract.EndContractBlock();\r
1949 \r
1950             string hash;\r
1951             using (var hasher = MD5.Create())\r
1952             using(var stream=File.OpenRead(fileName))\r
1953             {\r
1954                 var hashBuilder=new StringBuilder();\r
1955                 foreach (byte b in hasher.ComputeHash(stream))\r
1956                     hashBuilder.Append(b.ToString("x2").ToLower());\r
1957                 hash = hashBuilder.ToString();                \r
1958             }\r
1959             return hash;\r
1960         }\r
1961 */\r
1962 \r
1963         \r
1964         public void MoveObject(string account, Uri sourceContainer, Uri oldObjectName, Uri targetContainer, Uri newObjectName)\r
1965         {\r
1966             if (sourceContainer == null)\r
1967                 throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");\r
1968             if (sourceContainer.IsAbsoluteUri)\r
1969                 throw new ArgumentException("The sourceContainer must be relative","sourceContainer");\r
1970             if (oldObjectName == null)\r
1971                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");\r
1972             if (oldObjectName.IsAbsoluteUri)\r
1973                 throw new ArgumentException("The oldObjectName must be relative","oldObjectName");\r
1974             if (targetContainer == null)\r
1975                 throw new ArgumentNullException("targetContainer", "The targetContainer property can't be empty");\r
1976             if (targetContainer.IsAbsoluteUri)\r
1977                 throw new ArgumentException("The targetContainer must be relative","targetContainer");\r
1978             if (newObjectName == null)\r
1979                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");\r
1980             if (newObjectName.IsAbsoluteUri)\r
1981                 throw new ArgumentException("The newObjectName must be relative","newObjectName");\r
1982             Contract.EndContractBlock();\r
1983 \r
1984             var baseUri = GetTargetUri(account);\r
1985             var targetUri = baseUri.Combine(targetContainer).Combine(newObjectName);\r
1986             var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, oldObjectName),UriKind.Relative);\r
1987 \r
1988             var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
1989             message.Headers.Add("X-Move-From", sourceUri.ToString());\r
1990             using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)\r
1991             {\r
1992                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
1993                 if (!expectedCodes.Contains(response.StatusCode))\r
1994                     throw CreateWebException("MoveObject", response.StatusCode);\r
1995             }\r
1996 /*\r
1997             using (var client = new RestClient(_baseClient))\r
1998             {\r
1999                 if (!String.IsNullOrWhiteSpace(account))\r
2000                     client.BaseAddress = GetAccountUrl(account);\r
2001 \r
2002                 client.Headers.Add("X-Move-From", sourceUri.ToString());\r
2003                 client.PutWithRetry(targetUri, 3);\r
2004 \r
2005                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
2006                 if (!expectedCodes.Contains(client.StatusCode))\r
2007                     throw CreateWebException("MoveObject", client.StatusCode);\r
2008             }\r
2009 */\r
2010         }\r
2011 \r
2012         public void DeleteObject(string account, Uri sourceContainer, Uri objectName, bool isDirectory)\r
2013         {\r
2014             if (sourceContainer == null)\r
2015                 throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");\r
2016             if (sourceContainer.IsAbsoluteUri)\r
2017                 throw new ArgumentException("The sourceContainer must be relative","sourceContainer");\r
2018             if (objectName == null)\r
2019                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
2020             if (objectName.IsAbsoluteUri)\r
2021                 throw new ArgumentException("The objectName must be relative","objectName");\r
2022             Contract.EndContractBlock();\r
2023 \r
2024             var targetUrl = FolderConstants.TrashContainer + "/" + objectName;\r
2025 /*\r
2026             if (isDirectory)\r
2027                 targetUrl = targetUrl + "?delimiter=/";\r
2028 */\r
2029 \r
2030             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);\r
2031 \r
2032             using (var client = new RestClient(_baseClient))\r
2033             {\r
2034                 if (!String.IsNullOrWhiteSpace(account))\r
2035                     client.BaseAddress = GetAccountUrl(account);\r
2036 \r
2037                 client.Headers.Add("X-Move-From", sourceUrl);\r
2038                 client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);\r
2039                 Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);\r
2040                 client.PutWithRetry(new Uri(targetUrl,UriKind.Relative), 3);\r
2041 \r
2042                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};\r
2043                 if (!expectedCodes.Contains(client.StatusCode))\r
2044                     throw CreateWebException("DeleteObject", client.StatusCode);\r
2045             }\r
2046         }\r
2047 \r
2048       \r
2049         private static WebException CreateWebException(string operation, HttpStatusCode statusCode)\r
2050         {\r
2051             return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));\r
2052         }\r
2053 \r
2054 \r
2055 /*\r
2056         public IEnumerable<ObjectInfo> ListDirectories(ContainerInfo container)\r
2057         {\r
2058             var directories=this.ListObjects(container.Account, container.Name, "/");\r
2059         }\r
2060 */\r
2061 \r
2062         public bool CanUpload(string account, ObjectInfo cloudFile)\r
2063         {\r
2064             Contract.Requires(!String.IsNullOrWhiteSpace(account));\r
2065             Contract.Requires(cloudFile!=null);\r
2066 \r
2067             using (var client = new RestClient(_baseClient))\r
2068             {\r
2069                 if (!String.IsNullOrWhiteSpace(account))\r
2070                     client.BaseAddress = GetAccountUrl(account);\r
2071 \r
2072 \r
2073                 var parts = cloudFile.Name.ToString().Split('/');\r
2074                 var folder = String.Join("/", parts,0,parts.Length-1);\r
2075 \r
2076                 var fileName = String.Format("{0}/{1}.pithos.ignore", folder, Guid.NewGuid());\r
2077                 var fileUri=fileName.ToEscapedUri();                                            \r
2078 \r
2079                 client.Parameters.Clear();\r
2080                 try\r
2081                 {\r
2082                     var relativeUri = cloudFile.Container.Combine(fileUri);\r
2083                     client.PutWithRetry(relativeUri, 3, @"application/octet-stream");\r
2084 \r
2085                     var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
2086                     var result=(expectedCodes.Contains(client.StatusCode));\r
2087                     DeleteObject(account, cloudFile.Container, fileUri, cloudFile.IsDirectory);\r
2088                     return result;\r
2089                 }\r
2090                 catch\r
2091                 {\r
2092                     return false;\r
2093                 }\r
2094             }\r
2095         }\r
2096 \r
2097         ~CloudFilesClient()\r
2098         {\r
2099             Dispose(false);\r
2100         }\r
2101 \r
2102         public void Dispose()\r
2103         {\r
2104             Dispose(true);\r
2105             GC.SuppressFinalize(this);\r
2106         }\r
2107 \r
2108         protected virtual void Dispose(bool disposing)\r
2109         {\r
2110             if (disposing)\r
2111             {\r
2112                 if (_httpClientHandler!=null)\r
2113                     _httpClientHandler.Dispose();\r
2114                 if (_baseClient!=null)\r
2115                     _baseClient.Dispose();\r
2116                 if(_baseHttpClient!=null)\r
2117                     _baseHttpClient.Dispose();\r
2118                 if (_baseHttpClientNoTimeout!=null)\r
2119                     _baseHttpClientNoTimeout.Dispose();\r
2120             }\r
2121             _httpClientHandler = null;\r
2122             _baseClient = null;\r
2123             _baseHttpClient = null;\r
2124             _baseHttpClientNoTimeout = null;\r
2125         }\r
2126     }\r
2127 \r
2128     public class ShareAccountInfo\r
2129     {\r
2130         public DateTime? last_modified { get; set; }\r
2131         public string name { get; set; }\r
2132     }\r
2133 }\r