code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="CloudFilesClient.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 \r
  43 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos\r
  44 //\r
  45 // The class provides methods to upload/download files, delete files, manage containers\r
  46 \r
  47 \r
  48 using System;\r
  49 using System.Collections.Generic;\r
  50 using System.Collections.Specialized;\r
  51 using System.ComponentModel.Composition;\r
  52 using System.Diagnostics;\r
  53 using System.Diagnostics.Contracts;\r
  54 using System.IO;\r
  55 using System.Linq;\r
  56 using System.Net;\r
  57 using System.Net.Http;\r
  58 using System.Net.Http.Headers;\r
  59 using System.Reflection;\r
  60 using System.Threading;\r
  61 using System.Threading.Tasks;\r
  62 using Newtonsoft.Json;\r
  63 using Pithos.Interfaces;\r
  64 using Pithos.Network;\r
  65 using log4net;\r
  66 \r
  67 namespace Pithos.Network\r
  68 {\r
  69 \r
  70     [Export(typeof(ICloudClient))]\r
  71     public class CloudFilesClient:ICloudClient\r
  72     {\r
  73         private const string TOKEN_HEADER = "X-Auth-Token";\r
  74         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  75 \r
  76         //CloudFilesClient uses *_baseClient* internally to communicate with the server\r
  77         //RestClient provides a REST-friendly interface over the standard WebClient.\r
  78         private RestClient _baseClient;\r
  79 \r
  80         private HttpClient _baseHttpClient;\r
  81         private HttpClient _baseHttpClientNoTimeout;\r
  82         \r
  83 \r
  84         //During authentication the client provides a UserName \r
  85         public string UserName { get; set; }\r
  86         \r
  87         //and and ApiKey to the server\r
  88         public string ApiKey { get; set; }\r
  89         \r
  90         //And receives an authentication Token. This token must be provided in ALL other operations,\r
  91         //in the X-Auth-Token header\r
  92         private string _token;\r
  93         private readonly string _emptyGuid = Guid.Empty.ToString();\r
  94         private readonly Uri _emptyUri = new Uri("",UriKind.Relative);\r
  95 \r
  96 \r
  97         public string Token\r
  98         {\r
  99             get { return _token; }\r
 100             set\r
 101             {\r
 102                 _token = value;\r
 103                 _baseClient.Headers[TOKEN_HEADER] = value;\r
 104             }\r
 105         }\r
 106 \r
 107         //The client also receives a StorageUrl after authentication. All subsequent operations must\r
 108         //use this url\r
 109         public Uri StorageUrl { get; set; }\r
 110 \r
 111 \r
 112         public Uri RootAddressUri { get; set; }\r
 113 \r
 114        /* private WebProxy _proxy;\r
 115         public WebProxy Proxy\r
 116         {\r
 117             get { return _proxy; }\r
 118             set\r
 119             {\r
 120                 _proxy = value;\r
 121                 if (_baseClient != null)\r
 122                     _baseClient.Proxy = value;                \r
 123             }\r
 124         }\r
 125 */\r
 126 \r
 127         /* private Uri _proxy;\r
 128         public Uri Proxy\r
 129         {\r
 130             get { return _proxy; }\r
 131             set\r
 132             {\r
 133                 _proxy = value;\r
 134                 if (_baseClient != null)\r
 135                     _baseClient.Proxy = new WebProxy(value);                \r
 136             }\r
 137         }*/\r
 138 \r
 139         public double DownloadPercentLimit { get; set; }\r
 140         public double UploadPercentLimit { get; set; }\r
 141 \r
 142         public string AuthenticationUrl { get; set; }\r
 143 \r
 144  \r
 145         public string VersionPath\r
 146         {\r
 147             get { return UsePithos ? "v1" : "v1.0"; }\r
 148         }\r
 149 \r
 150         public bool UsePithos { get; set; }\r
 151 \r
 152 \r
 153 \r
 154         public CloudFilesClient(string userName, string apiKey)\r
 155         {\r
 156             UserName = userName;\r
 157             ApiKey = apiKey;\r
 158         }\r
 159 \r
 160         public CloudFilesClient(AccountInfo accountInfo)\r
 161         {\r
 162             if (accountInfo==null)\r
 163                 throw new ArgumentNullException("accountInfo");\r
 164             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));\r
 165             Contract.Ensures(StorageUrl != null);\r
 166             Contract.Ensures(_baseClient != null);\r
 167             Contract.Ensures(RootAddressUri != null);\r
 168             Contract.EndContractBlock();          \r
 169 \r
 170             _baseClient = new RestClient\r
 171             {\r
 172                 BaseAddress = accountInfo.StorageUri.ToString(),\r
 173                 Timeout = 30000,\r
 174                 Retries = 3,\r
 175             };\r
 176             StorageUrl = accountInfo.StorageUri;\r
 177             Token = accountInfo.Token;\r
 178             UserName = accountInfo.UserName;\r
 179 \r
 180             //Get the root address (StorageUrl without the account)\r
 181             var storageUrl = StorageUrl.AbsoluteUri;\r
 182             var usernameIndex = storageUrl.LastIndexOf(UserName);\r
 183             var rootUrl = storageUrl.Substring(0, usernameIndex);\r
 184             RootAddressUri = new Uri(rootUrl);\r
 185 \r
 186             var httpClientHandler = new HttpClientHandler\r
 187             {\r
 188                 AllowAutoRedirect = true,\r
 189                 AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,\r
 190                 UseCookies = true,\r
 191             };\r
 192 \r
 193 \r
 194             _baseHttpClient = new HttpClient(httpClientHandler)\r
 195             {\r
 196                 BaseAddress = StorageUrl,\r
 197                 Timeout = TimeSpan.FromSeconds(30)\r
 198             };\r
 199             _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);\r
 200 \r
 201             _baseHttpClientNoTimeout = new HttpClient(httpClientHandler)\r
 202             {\r
 203                 BaseAddress = StorageUrl,\r
 204                 Timeout = TimeSpan.FromMilliseconds(-1)\r
 205             };\r
 206             _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);\r
 207 \r
 208 \r
 209         }\r
 210 \r
 211 \r
 212         private static void AssertStatusOK(HttpResponseMessage response, string message)\r
 213         {\r
 214             var statusCode = response.StatusCode;\r
 215             if (statusCode >= HttpStatusCode.BadRequest)\r
 216                 throw new WebException(String.Format("{0} with code {1} - {2}", message, statusCode, response.ReasonPhrase));\r
 217         }\r
 218 \r
 219         public AccountInfo Authenticate()\r
 220         {\r
 221             if (String.IsNullOrWhiteSpace(UserName))\r
 222                 throw new InvalidOperationException("UserName is empty");\r
 223             if (String.IsNullOrWhiteSpace(ApiKey))\r
 224                 throw new InvalidOperationException("ApiKey is empty");\r
 225             if (String.IsNullOrWhiteSpace(AuthenticationUrl))\r
 226                 throw new InvalidOperationException("AuthenticationUrl is empty");\r
 227             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));\r
 228             Contract.Ensures(StorageUrl != null);\r
 229             Contract.Ensures(_baseClient != null);\r
 230             Contract.Ensures(RootAddressUri != null);\r
 231             Contract.EndContractBlock();\r
 232 \r
 233 \r
 234             Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);\r
 235 \r
 236             var groups = new List<Group>();\r
 237 \r
 238             var httpClientHandler = new HttpClientHandler\r
 239             {\r
 240                 AllowAutoRedirect = true,\r
 241                 AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,\r
 242                 UseCookies = true,                                \r
 243             };\r
 244 \r
 245             using (var authClient = new HttpClient(httpClientHandler){ BaseAddress = new Uri(AuthenticationUrl),Timeout=TimeSpan.FromSeconds(30) })\r
 246             //using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})\r
 247             {                \r
 248                /* if (Proxy != null)\r
 249                     authClient.Proxy = Proxy;*/\r
 250 \r
 251                 //Contract.Assume(authClient.DefaultRequestHeaders != null);\r
 252 \r
 253                 authClient.DefaultRequestHeaders.Add("X-Auth-User", UserName);\r
 254                 authClient.DefaultRequestHeaders.Add("X-Auth-Key", ApiKey);\r
 255 \r
 256                 //Func<Task<HttpResponseMessage>> call = () => ;\r
 257                 string storageUrl;\r
 258                 string token;\r
 259                 \r
 260                 using (var response = authClient.GetAsyncWithRetries(new Uri(VersionPath, UriKind.Relative),3).Result) // .DownloadStringWithRetryRelative(new Uri(VersionPath, UriKind.Relative), 3);                    \r
 261                 {\r
 262                     AssertStatusOK(response,"Authentication failed");\r
 263                 \r
 264                     storageUrl = response.Headers.GetValues("X-Storage-Url").First();\r
 265                     if (String.IsNullOrWhiteSpace(storageUrl))\r
 266                         throw new InvalidOperationException("Failed to obtain storage url");\r
 267 \r
 268                     token = response.Headers.GetValues(TOKEN_HEADER).First();\r
 269                     if (String.IsNullOrWhiteSpace(token))\r
 270                         throw new InvalidOperationException("Failed to obtain token url");\r
 271 \r
 272                 }\r
 273 \r
 274 \r
 275                 _baseClient = new RestClient\r
 276                 {\r
 277                     BaseAddress = storageUrl,\r
 278                     Timeout = 30000,\r
 279                     Retries = 3,                    \r
 280                     //Proxy=Proxy\r
 281                 };\r
 282 \r
 283                 StorageUrl = new Uri(storageUrl);\r
 284                 Token = token;\r
 285 \r
 286                 \r
 287 \r
 288                                                                \r
 289                 //Get the root address (StorageUrl without the account)\r
 290                 var usernameIndex=storageUrl.LastIndexOf(UserName);\r
 291                 var rootUrl = storageUrl.Substring(0, usernameIndex);\r
 292                 RootAddressUri = new Uri(rootUrl);\r
 293                 \r
 294 \r
 295                 _baseHttpClient = new HttpClient(httpClientHandler)\r
 296                 {\r
 297                     BaseAddress = StorageUrl,\r
 298                     Timeout = TimeSpan.FromSeconds(30)\r
 299                 };\r
 300                 _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, token);\r
 301 \r
 302                 _baseHttpClientNoTimeout = new HttpClient(httpClientHandler)\r
 303                 {\r
 304                     BaseAddress = StorageUrl,\r
 305                     Timeout = TimeSpan.FromMilliseconds(-1)\r
 306                 };\r
 307                 _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, token);\r
 308 \r
 309                 /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();\r
 310                 groups = (from key in keys\r
 311                             where key.StartsWith("X-Account-Group-")\r
 312                             let name = key.Substring(16)\r
 313                             select new Group(name, authClient.ResponseHeaders[key]))\r
 314                         .ToList();\r
 315                     \r
 316 */\r
 317             }\r
 318 \r
 319             Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);\r
 320             Debug.Assert(_baseClient!=null);\r
 321 \r
 322             return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};            \r
 323 \r
 324         }\r
 325 \r
 326         private static void TraceStart(string method, Uri actualAddress)\r
 327         {\r
 328             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);\r
 329         }\r
 330 \r
 331         private async Task<string> GetStringAsync(Uri targetUri, string errorMessage,DateTime? since=null)\r
 332         {\r
 333             TraceStart("GET",targetUri);\r
 334             var request = new HttpRequestMessage(HttpMethod.Get, targetUri);            \r
 335             if (since.HasValue)\r
 336             {\r
 337                 request.Headers.IfModifiedSince = since.Value;\r
 338             }\r
 339             //Func<Task<HttpResponseMessage>> call = () => _baseHttpClient.SendAsync(request);\r
 340             using (var response = await _baseHttpClient.SendAsyncWithRetries(request,3).ConfigureAwait(false))\r
 341             {\r
 342                 AssertStatusOK(response, errorMessage);\r
 343 \r
 344                 if (response.StatusCode == HttpStatusCode.NoContent)\r
 345                     return String.Empty;\r
 346 \r
 347                 var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);\r
 348                 return content;\r
 349             }\r
 350         }\r
 351 \r
 352         public IList<ContainerInfo> ListContainers(string account)\r
 353         {\r
 354 \r
 355             var targetUrl = GetTargetUrl(account);\r
 356             var targetUri=new Uri(String.Format("{0}?format=json",targetUrl));\r
 357             var result = GetStringAsync(targetUri, "List Containers failed").Result;\r
 358             if (String.IsNullOrWhiteSpace(result))\r
 359                 return new List<ContainerInfo>();\r
 360             var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(result);                \r
 361             foreach (var info in infos)\r
 362             {\r
 363                 info.Account = account;\r
 364             }\r
 365             return infos;\r
 366            /* using (var client = new RestClient(_baseClient))\r
 367             {\r
 368                 if (!String.IsNullOrWhiteSpace(account))\r
 369                     client.BaseAddress = GetAccountUrl(account);\r
 370                 \r
 371                 client.Parameters.Clear();\r
 372                 client.Parameters.Add("format", "json");\r
 373                 var content = client.DownloadStringWithRetryRelative(_emptyUri, 3);\r
 374                 client.AssertStatusOK("List Containers failed");\r
 375 \r
 376                 if (client.StatusCode == HttpStatusCode.NoContent)\r
 377                     return new List<ContainerInfo>();\r
 378                 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);\r
 379                 \r
 380                 foreach (var info in infos)\r
 381                 {\r
 382                     info.Account = account;\r
 383                 }\r
 384                 return infos;\r
 385             }   */       \r
 386 \r
 387         }\r
 388 \r
 389         private string GetAccountUrl(string account)\r
 390         {\r
 391             return RootAddressUri.Combine(account).AbsoluteUri;\r
 392         }\r
 393 \r
 394         public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)\r
 395         {\r
 396             using (ThreadContext.Stacks["Share"].Push("List Accounts"))\r
 397             {\r
 398                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 399 \r
 400                 var targetUri = new Uri(String.Format("{0}?format=json", RootAddressUri), UriKind.Absolute);\r
 401                 var content=GetStringAsync(targetUri, "ListSharingAccounts failed", since).Result;\r
 402 \r
 403                 //If the result is empty, return an empty list,\r
 404                 var infos = String.IsNullOrWhiteSpace(content)\r
 405                                 ? new List<ShareAccountInfo>()\r
 406                             //Otherwise deserialize the account list into a list of ShareAccountInfos\r
 407                                 : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);\r
 408 \r
 409                 Log.DebugFormat("END");\r
 410                 return infos;\r
 411 /*\r
 412                 using (var client = new RestClient(_baseClient))\r
 413                 {\r
 414                     client.Parameters.Clear();\r
 415                     client.Parameters.Add("format", "json");\r
 416                     client.IfModifiedSince = since;\r
 417 \r
 418                     //Extract the username from the base address\r
 419                     client.BaseAddress = RootAddressUri.AbsoluteUri; \r
 420 \r
 421                     var content = client.DownloadStringWithRetryRelative(_emptyUri, 3);\r
 422 \r
 423                     client.AssertStatusOK("ListSharingAccounts failed");\r
 424 \r
 425                     //If the result is empty, return an empty list,\r
 426                     var infos = String.IsNullOrWhiteSpace(content)\r
 427                                     ? new List<ShareAccountInfo>()\r
 428                                 //Otherwise deserialize the account list into a list of ShareAccountInfos\r
 429                                     : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);\r
 430 \r
 431                     Log.DebugFormat("END");\r
 432                     return infos;\r
 433                 }\r
 434 */\r
 435             }\r
 436         }\r
 437 \r
 438 \r
 439         /// <summary>\r
 440         /// Request listing of all objects in a container modified since a specific time.\r
 441         /// If the *since* value is missing, return all objects\r
 442         /// </summary>\r
 443         /// <param name="knownContainers">Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new\r
 444         /// and should be polled anyway\r
 445         /// </param>\r
 446         /// <param name="since"></param>\r
 447         /// <returns></returns>\r
 448         public IList<ObjectInfo> ListSharedObjects(HashSet<string> knownContainers,DateTime? since = null )\r
 449         {\r
 450 \r
 451             using (ThreadContext.Stacks["Share"].Push("List Objects"))\r
 452             {\r
 453                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 454                 //'since' is not used here because we need to have ListObjects return a NoChange result\r
 455                 //for all shared accounts,containers\r
 456 \r
 457                 Func<ContainerInfo, string> getKey = c => String.Format("{0}\\{1}", c.Account, c.Name);\r
 458 \r
 459                 var accounts = ListSharingAccounts();\r
 460                 var containers = (from account in accounts\r
 461                                  let conts = ListContainers(account.name)\r
 462                                  from container in conts\r
 463                                  select container).ToList();                \r
 464                 var items = from container in containers \r
 465                             let actualSince=knownContainers.Contains(getKey(container))?since:null\r
 466                             select ListObjects(container.Account , container.Name,  actualSince);\r
 467                 var objects=items.SelectMany(r=> r).ToList();\r
 468 \r
 469                 //For each object\r
 470                 //Check parents recursively up to (but not including) the container.\r
 471                 //If parents are missing, add them to the list\r
 472                 //Need function to calculate all parent URLs\r
 473                 objects = AddMissingParents(objects);\r
 474                 \r
 475                 //Store any new containers\r
 476                 foreach (var container in containers)\r
 477                 {\r
 478                     knownContainers.Add(getKey(container));\r
 479                 }\r
 480 \r
 481 \r
 482 \r
 483                 if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 484                 return objects;\r
 485             }\r
 486         }\r
 487 \r
 488         private List<ObjectInfo> AddMissingParents(List<ObjectInfo> objects)\r
 489         {\r
 490             //TODO: Remove short-circuit when we decide to use Missing Parents functionality\r
 491             //return objects;\r
 492 \r
 493             var existingUris = objects.ToDictionary(o => o.Uri, o => o);\r
 494             foreach (var objectInfo in objects)\r
 495             {\r
 496                 //Can be null when retrieving objects to show in selective sync\r
 497                 if (objectInfo.Name == null)\r
 498                     continue;\r
 499 \r
 500                 //No need to unescape here, the parts will be used to create new ObjectInfos\r
 501                 var parts = objectInfo.Name.ToString().Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries);\r
 502                 //If there is no parent, skip\r
 503                 if (parts.Length == 1)\r
 504                     continue;\r
 505                 var baseParts = new[]\r
 506                                   {\r
 507                                       objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container.ToString()\r
 508                                   };\r
 509                 for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++)\r
 510                 {\r
 511                     var nameparts = parts.Range(0, partIdx).ToArray();\r
 512                     var parentName= String.Join("/", nameparts);\r
 513 \r
 514                     var parentParts = baseParts.Concat(nameparts);\r
 515                     var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts);\r
 516                     \r
 517                     var parentUri = new Uri(parentUrl, UriKind.Absolute);\r
 518 \r
 519                     ObjectInfo existingInfo;\r
 520                     if (!existingUris.TryGetValue(parentUri,out existingInfo))\r
 521                     {\r
 522                         var h = parentUrl.GetHashCode();\r
 523                         var reverse = new string(parentUrl.Reverse().ToArray());\r
 524                         var rh = reverse.GetHashCode();\r
 525                         var b1 = BitConverter.GetBytes(h);\r
 526                         var b2 = BitConverter.GetBytes(rh);\r
 527                         var g = new Guid(0,0,0,b1.Concat(b2).ToArray());\r
 528                         \r
 529 \r
 530                         existingUris[parentUri] = new ObjectInfo\r
 531                                                       {\r
 532                                                           Account = objectInfo.Account,\r
 533                                                           Container = objectInfo.Container,\r
 534                                                           Content_Type = ObjectInfo.CONTENT_TYPE_DIRECTORY,\r
 535                                                           ETag = Signature.MERKLE_EMPTY,\r
 536                                                           X_Object_Hash = Signature.MERKLE_EMPTY,\r
 537                                                           Name=new Uri(parentName,UriKind.Relative),\r
 538                                                           StorageUri=objectInfo.StorageUri,\r
 539                                                           Bytes = 0,\r
 540                                                           UUID=g.ToString(),                                                          \r
 541                                                       };\r
 542                     }\r
 543                 }\r
 544             }\r
 545             return existingUris.Values.ToList();\r
 546         }\r
 547 \r
 548         public void SetTags(ObjectInfo target,IDictionary<string,string> tags)\r
 549         {\r
 550             if (String.IsNullOrWhiteSpace(Token))\r
 551                 throw new InvalidOperationException("The Token is not set");\r
 552             if (StorageUrl == null)\r
 553                 throw new InvalidOperationException("The StorageUrl is not set");\r
 554             if (target == null)\r
 555                 throw new ArgumentNullException("target");\r
 556             Contract.EndContractBlock();\r
 557 \r
 558             using (ThreadContext.Stacks["Share"].Push("Share Object"))\r
 559             {\r
 560                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 561 \r
 562                 using (var client = new RestClient(_baseClient))\r
 563                 {\r
 564 \r
 565                     client.BaseAddress = GetAccountUrl(target.Account);\r
 566 \r
 567                     client.Parameters.Clear();\r
 568                     client.Parameters.Add("update", "");\r
 569 \r
 570                     foreach (var tag in tags)\r
 571                     {\r
 572                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);\r
 573                         client.Headers.Add(headerTag, tag.Value);\r
 574                     }\r
 575                     \r
 576                     client.DownloadStringWithRetryRelative(target.Container, 3);\r
 577 \r
 578                     \r
 579                     client.AssertStatusOK("SetTags failed");\r
 580                     //If the status is NOT ACCEPTED we have a problem\r
 581                     if (client.StatusCode != HttpStatusCode.Accepted)\r
 582                     {\r
 583                         Log.Error("Failed to set tags");\r
 584                         throw new Exception("Failed to set tags");\r
 585                     }\r
 586 \r
 587                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 588                 }\r
 589             }\r
 590 \r
 591 \r
 592         }\r
 593 \r
 594         public void ShareObject(string account, Uri container, Uri objectName, string shareTo, bool read, bool write)\r
 595         {\r
 596             if (String.IsNullOrWhiteSpace(Token))\r
 597                 throw new InvalidOperationException("The Token is not set");\r
 598             if (StorageUrl==null)\r
 599                 throw new InvalidOperationException("The StorageUrl is not set");\r
 600             if (container==null)\r
 601                 throw new ArgumentNullException("container");\r
 602             if (container.IsAbsoluteUri)\r
 603                 throw new ArgumentException("container");\r
 604             if (objectName==null)\r
 605                 throw new ArgumentNullException("objectName");\r
 606             if (objectName.IsAbsoluteUri)\r
 607                 throw new ArgumentException("objectName");\r
 608             if (String.IsNullOrWhiteSpace(account))\r
 609                 throw new ArgumentNullException("account");\r
 610             if (String.IsNullOrWhiteSpace(shareTo))\r
 611                 throw new ArgumentNullException("shareTo");\r
 612             Contract.EndContractBlock();\r
 613 \r
 614             using (ThreadContext.Stacks["Share"].Push("Share Object"))\r
 615             {\r
 616                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 617                 \r
 618                 using (var client = new RestClient(_baseClient))\r
 619                 {\r
 620 \r
 621                     client.BaseAddress = GetAccountUrl(account);\r
 622 \r
 623                     client.Parameters.Clear();\r
 624                     client.Parameters.Add("format", "json");\r
 625 \r
 626                     string permission = "";\r
 627                     if (write)\r
 628                         permission = String.Format("write={0}", shareTo);\r
 629                     else if (read)\r
 630                         permission = String.Format("read={0}", shareTo);\r
 631                     client.Headers.Add("X-Object-Sharing", permission);\r
 632 \r
 633                     var content = client.DownloadStringWithRetryRelative(container, 3);\r
 634 \r
 635                     client.AssertStatusOK("ShareObject failed");\r
 636 \r
 637                     //If the result is empty, return an empty list,\r
 638                     var infos = String.IsNullOrWhiteSpace(content)\r
 639                                     ? new List<ObjectInfo>()\r
 640                                 //Otherwise deserialize the object list into a list of ObjectInfos\r
 641                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 642 \r
 643                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 644                 }\r
 645             }\r
 646 \r
 647 \r
 648         }\r
 649 \r
 650         public AccountInfo GetAccountPolicies(AccountInfo accountInfo)\r
 651         {\r
 652             if (accountInfo==null)\r
 653                 throw new ArgumentNullException("accountInfo");\r
 654             Contract.EndContractBlock();\r
 655 \r
 656             using (ThreadContext.Stacks["Account"].Push("GetPolicies"))\r
 657             {\r
 658                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 659 \r
 660                 if (_baseClient == null)\r
 661                 {\r
 662                     _baseClient = new RestClient\r
 663                     {\r
 664                         BaseAddress = accountInfo.StorageUri.ToString(),\r
 665                         Timeout = 30000,\r
 666                         Retries = 3,\r
 667                     };\r
 668                 }\r
 669 \r
 670                 using (var client = new RestClient(_baseClient))\r
 671                 {\r
 672                     if (!String.IsNullOrWhiteSpace(accountInfo.UserName))\r
 673                         client.BaseAddress = GetAccountUrl(accountInfo.UserName);\r
 674 \r
 675                     client.Parameters.Clear();\r
 676                     client.Parameters.Add("format", "json");                    \r
 677                     client.Head(_emptyUri, 3);\r
 678 \r
 679                     var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];\r
 680                     var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];\r
 681 \r
 682                     long quota, bytes;\r
 683                     if (long.TryParse(quotaValue, out quota))\r
 684                         accountInfo.Quota = quota;\r
 685                     if (long.TryParse(bytesValue, out bytes))\r
 686                         accountInfo.BytesUsed = bytes;\r
 687                     \r
 688                     return accountInfo;\r
 689 \r
 690                 }\r
 691 \r
 692             }\r
 693         }\r
 694 \r
 695         public void UpdateMetadata(ObjectInfo objectInfo)\r
 696         {\r
 697             if (objectInfo == null)\r
 698                 throw new ArgumentNullException("objectInfo");\r
 699             Contract.EndContractBlock();\r
 700 \r
 701             using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))\r
 702             {\r
 703                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 704 \r
 705 \r
 706                 using(var client=new RestClient(_baseClient))\r
 707                 {\r
 708 \r
 709                     client.BaseAddress = GetAccountUrl(objectInfo.Account);\r
 710                     \r
 711                     client.Parameters.Clear();\r
 712                     \r
 713 \r
 714                     //Set Tags\r
 715                     foreach (var tag in objectInfo.Tags)\r
 716                     {\r
 717                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);\r
 718                         client.Headers.Add(headerTag, tag.Value);\r
 719                     }\r
 720 \r
 721                     //Set Permissions\r
 722 \r
 723                     var permissions=objectInfo.GetPermissionString();\r
 724                     client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);\r
 725 \r
 726                     client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);\r
 727                     client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);\r
 728                     client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);\r
 729                     var isPublic = objectInfo.IsPublic.ToString().ToLower();\r
 730                     client.Headers.Add("X-Object-Public", isPublic);\r
 731 \r
 732 \r
 733                     /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);\r
 734                     uriBuilder.Query = "update=";\r
 735                     var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/\r
 736                     var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);\r
 737                     client.PostWithRetry(new Uri(address,UriKind.Relative),"application/xml");\r
 738                     \r
 739                     //client.UploadValues(uri,new NameValueCollection());\r
 740 \r
 741 \r
 742                     client.AssertStatusOK("UpdateMetadata failed");\r
 743                     //If the status is NOT ACCEPTED or OK we have a problem\r
 744                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))\r
 745                     {\r
 746                         Log.Error("Failed to update metadata");\r
 747                         throw new Exception("Failed to update metadata");\r
 748                     }\r
 749 \r
 750                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 751                 }\r
 752             }\r
 753 \r
 754         }\r
 755 \r
 756         public void UpdateMetadata(ContainerInfo containerInfo)\r
 757         {\r
 758             if (containerInfo == null)\r
 759                 throw new ArgumentNullException("containerInfo");\r
 760             Contract.EndContractBlock();\r
 761 \r
 762             using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))\r
 763             {\r
 764                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 765 \r
 766 \r
 767                 using(var client=new RestClient(_baseClient))\r
 768                 {\r
 769 \r
 770                     client.BaseAddress = GetAccountUrl(containerInfo.Account);\r
 771                     \r
 772                     client.Parameters.Clear();\r
 773                     \r
 774 \r
 775                     //Set Tags\r
 776                     foreach (var tag in containerInfo.Tags)\r
 777                     {\r
 778                         var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);\r
 779                         client.Headers.Add(headerTag, tag.Value);\r
 780                     }\r
 781 \r
 782                     \r
 783                     //Set Policies\r
 784                     foreach (var policy in containerInfo.Policies)\r
 785                     {\r
 786                         var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);\r
 787                         client.Headers.Add(headerPolicy, policy.Value);\r
 788                     }\r
 789 \r
 790 \r
 791                     var uriBuilder = client.GetAddressBuilder(containerInfo.Name,_emptyUri);\r
 792                     var uri = uriBuilder.Uri;\r
 793 \r
 794                     client.UploadValues(uri,new NameValueCollection());\r
 795 \r
 796 \r
 797                     client.AssertStatusOK("UpdateMetadata failed");\r
 798                     //If the status is NOT ACCEPTED or OK we have a problem\r
 799                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))\r
 800                     {\r
 801                         Log.Error("Failed to update metadata");\r
 802                         throw new Exception("Failed to update metadata");\r
 803                     }\r
 804 \r
 805                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 806                 }\r
 807             }\r
 808 \r
 809         }\r
 810 \r
 811        \r
 812 \r
 813 \r
 814         public IList<ObjectInfo> ListObjects(string account, Uri container, DateTime? since = null)\r
 815         {\r
 816             if (container==null)\r
 817                 throw new ArgumentNullException("container");\r
 818             if (container.IsAbsoluteUri)\r
 819                 throw new ArgumentException("container");\r
 820             Contract.EndContractBlock();\r
 821 \r
 822             using (ThreadContext.Stacks["Objects"].Push("List"))\r
 823             {\r
 824                 /*if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 825 \r
 826                 var targetUrl = String.IsNullOrWhiteSpace(account)\r
 827                     ? _baseHttpClient.BaseAddress.ToString()\r
 828                     : GetAccountUrl(account);\r
 829                 var targetUri = new Uri(String.Format("{0}?format=json", targetUrl));\r
 830                 var result = GetString(targetUri, "ListObjects failed",since).Result;\r
 831 \r
 832                 if (String.IsNullOrWhiteSpace(result))\r
 833                     return new[]{new NoModificationInfo(account,container)};\r
 834                 //If the result is empty, return an empty list,\r
 835                 var infos = String.IsNullOrWhiteSpace(result)\r
 836                                 ? new List<ObjectInfo>()\r
 837                             //Otherwise deserialize the object list into a list of ObjectInfos\r
 838                                 : JsonConvert.DeserializeObject<IList<ObjectInfo>>(result);\r
 839 \r
 840                 foreach (var info in infos)\r
 841                 {\r
 842                     info.Container = container;\r
 843                     info.Account = account;\r
 844                     info.StorageUri = StorageUrl;\r
 845                 }\r
 846                 if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 847                 return infos;*/\r
 848 \r
 849                 var containerUri = GetTargetUri(account).Combine(container);\r
 850                 var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);\r
 851                 var content = GetStringAsync(targetUri, "ListObjects failed", since).Result;\r
 852 \r
 853                 //304 will result in an empty string. Empty containers return an empty json array\r
 854                 if (String.IsNullOrWhiteSpace(content))\r
 855                      return new[] {new NoModificationInfo(account, container)};\r
 856 \r
 857                  //If the result is empty, return an empty list,\r
 858                  var infos = String.IsNullOrWhiteSpace(content)\r
 859                                  ? new List<ObjectInfo>()\r
 860                              //Otherwise deserialize the object list into a list of ObjectInfos\r
 861                                  : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 862 \r
 863                  foreach (var info in infos)\r
 864                  {\r
 865                      info.Container = container;\r
 866                      info.Account = account;\r
 867                      info.StorageUri = StorageUrl;\r
 868                  }\r
 869                  if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 870                  return infos;\r
 871 /*\r
 872              using (var client = new RestClient(_baseClient))\r
 873              {\r
 874                  if (!String.IsNullOrWhiteSpace(account))\r
 875                      client.BaseAddress = GetAccountUrl(account);\r
 876 \r
 877                  client.Parameters.Clear();\r
 878                  client.Parameters.Add("format", "json");\r
 879                  client.IfModifiedSince = since;\r
 880                  var content = client.DownloadStringWithRetryRelative(container, 3);\r
 881 \r
 882                  client.AssertStatusOK("ListObjects failed");\r
 883 \r
 884                  if (client.StatusCode == HttpStatusCode.NotModified)\r
 885                      return new[] {new NoModificationInfo(account, container)};\r
 886                  //If the result is empty, return an empty list,\r
 887                  var infos = String.IsNullOrWhiteSpace(content)\r
 888                                  ? new List<ObjectInfo>()\r
 889                              //Otherwise deserialize the object list into a list of ObjectInfos\r
 890                                  : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 891 \r
 892                  foreach (var info in infos)\r
 893                  {\r
 894                      info.Container = container;\r
 895                      info.Account = account;\r
 896                      info.StorageUri = StorageUrl;\r
 897                  }\r
 898                  if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 899                  return infos;\r
 900              }\r
 901 */\r
 902             }\r
 903         }\r
 904 \r
 905         public IList<ObjectInfo> ListObjects(string account, Uri container, Uri folder, DateTime? since = null)\r
 906         {\r
 907             if (container==null)\r
 908                 throw new ArgumentNullException("container");\r
 909             if (container.IsAbsoluteUri)\r
 910                 throw new ArgumentException("container");\r
 911 /*\r
 912             if (String.IsNullOrWhiteSpace(folder))\r
 913                 throw new ArgumentNullException("folder");\r
 914 */\r
 915             Contract.EndContractBlock();\r
 916 \r
 917             using (ThreadContext.Stacks["Objects"].Push("List"))\r
 918             {\r
 919                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 920 \r
 921                 var containerUri = GetTargetUri(account).Combine(container);\r
 922                 var targetUri = new Uri(String.Format("{0}?format=json&path={1}", containerUri,folder), UriKind.Absolute);\r
 923                 var content = GetStringAsync(targetUri, "ListObjects failed", since).Result;                \r
 924 \r
 925                 //304 will result in an empty string. Empty containers return an empty json array\r
 926                 if (String.IsNullOrWhiteSpace(content))\r
 927                     return new[] { new NoModificationInfo(account, container) };\r
 928 \r
 929 \r
 930                 var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 931                 foreach (var info in infos)\r
 932                 {\r
 933                     info.Account = account;\r
 934                     if (info.Container == null)\r
 935                         info.Container = container;\r
 936                     info.StorageUri = StorageUrl;\r
 937                 }\r
 938                 if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 939                 return infos;\r
 940 /*\r
 941                 using (var client = new RestClient(_baseClient))\r
 942                 {\r
 943                     if (!String.IsNullOrWhiteSpace(account))\r
 944                         client.BaseAddress = GetAccountUrl(account);\r
 945 \r
 946                     client.Parameters.Clear();\r
 947                     client.Parameters.Add("format", "json");\r
 948                     client.Parameters.Add("path", folder.ToString());\r
 949                     client.IfModifiedSince = since;\r
 950                     var content = client.DownloadStringWithRetryRelative(container, 3);\r
 951                     client.AssertStatusOK("ListObjects failed");\r
 952 \r
 953                     if (client.StatusCode==HttpStatusCode.NotModified)\r
 954                         return new[]{new NoModificationInfo(account,container,folder)};\r
 955 \r
 956                     var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 957                     foreach (var info in infos)\r
 958                     {\r
 959                         info.Account = account;\r
 960                         if (info.Container == null)\r
 961                             info.Container = container;\r
 962                         info.StorageUri = StorageUrl;\r
 963                     }\r
 964                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 965                     return infos;\r
 966                 }\r
 967 */\r
 968             }\r
 969         }\r
 970 \r
 971  \r
 972         public bool ContainerExists(string account, Uri container)\r
 973         {\r
 974             if (container==null)\r
 975                 throw new ArgumentNullException("container", "The container property can't be empty");\r
 976             if (container.IsAbsoluteUri)\r
 977                 throw new ArgumentException( "The container must be relative","container");\r
 978             Contract.EndContractBlock();\r
 979 \r
 980             using (ThreadContext.Stacks["Containters"].Push("Exists"))\r
 981             {\r
 982                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 983 \r
 984                 var targetUri = GetTargetUri(account).Combine(container);\r
 985 \r
 986                 using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result)\r
 987                 {\r
 988 \r
 989                     bool result;\r
 990                     switch (response.StatusCode)\r
 991                     {\r
 992                         case HttpStatusCode.OK:\r
 993                         case HttpStatusCode.NoContent:\r
 994                             result = true;\r
 995                             break;\r
 996                         case HttpStatusCode.NotFound:\r
 997                             result = false;\r
 998                             break;\r
 999                         default:\r
1000                             throw CreateWebException("ContainerExists", response.StatusCode);\r
1001                     }\r
1002                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
1003 \r
1004                     return result;\r
1005                 }\r
1006 /*\r
1007                 using (var client = new RestClient(_baseClient))\r
1008                 {\r
1009                     if (!String.IsNullOrWhiteSpace(account))\r
1010                         client.BaseAddress = GetAccountUrl(account);\r
1011 \r
1012                     client.Parameters.Clear();\r
1013                     client.Head(container, 3);\r
1014                                         \r
1015                     bool result;\r
1016                     switch (client.StatusCode)\r
1017                     {\r
1018                         case HttpStatusCode.OK:\r
1019                         case HttpStatusCode.NoContent:\r
1020                             result=true;\r
1021                             break;\r
1022                         case HttpStatusCode.NotFound:\r
1023                             result=false;\r
1024                             break;\r
1025                         default:\r
1026                             throw CreateWebException("ContainerExists", client.StatusCode);\r
1027                     }\r
1028                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
1029 \r
1030                     return result;\r
1031                 }\r
1032 */\r
1033                 \r
1034             }\r
1035         }\r
1036 \r
1037         private Uri GetTargetUri(string account)\r
1038         {\r
1039             return new Uri(GetTargetUrl(account),UriKind.Absolute);\r
1040         }\r
1041 \r
1042         private string GetTargetUrl(string account)\r
1043         {\r
1044             return String.IsNullOrWhiteSpace(account)\r
1045                        ? _baseHttpClient.BaseAddress.ToString()\r
1046                        : GetAccountUrl(account);\r
1047         }\r
1048 \r
1049         public bool ObjectExists(string account, Uri container, Uri objectName)\r
1050         {\r
1051             if (container == null)\r
1052                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1053             if (container.IsAbsoluteUri)\r
1054                 throw new ArgumentException("The container must be relative","container");\r
1055             if (objectName == null)\r
1056                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1057             if (objectName.IsAbsoluteUri)\r
1058                 throw new ArgumentException("The objectName must be relative","objectName");\r
1059             Contract.EndContractBlock();\r
1060 \r
1061                 var targetUri=GetTargetUri(account).Combine(container).Combine(objectName);\r
1062 \r
1063             using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result)\r
1064             {\r
1065                 switch (response.StatusCode)\r
1066                 {\r
1067                     case HttpStatusCode.OK:\r
1068                     case HttpStatusCode.NoContent:\r
1069                         return true;\r
1070                     case HttpStatusCode.NotFound:\r
1071                         return false;\r
1072                     default:\r
1073                         throw CreateWebException("ObjectExists", response.StatusCode);\r
1074                 }\r
1075             }\r
1076 \r
1077 /*\r
1078             using (var client = new RestClient(_baseClient))\r
1079             {\r
1080                 if (!String.IsNullOrWhiteSpace(account))\r
1081                     client.BaseAddress = GetAccountUrl(account);\r
1082 \r
1083                 client.Parameters.Clear();\r
1084                 client.Head(container.Combine(objectName), 3);\r
1085 \r
1086                 switch (client.StatusCode)\r
1087                 {\r
1088                     case HttpStatusCode.OK:\r
1089                     case HttpStatusCode.NoContent:\r
1090                         return true;\r
1091                     case HttpStatusCode.NotFound:\r
1092                         return false;\r
1093                     default:\r
1094                         throw CreateWebException("ObjectExists", client.StatusCode);\r
1095                 }\r
1096             }\r
1097 */\r
1098 \r
1099         }\r
1100 \r
1101         public ObjectInfo GetObjectInfo(string account, Uri container, Uri objectName)\r
1102         {\r
1103             if (container == null)\r
1104                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1105             if (container.IsAbsoluteUri)\r
1106                 throw new ArgumentException("The container must be relative","container");\r
1107             if (objectName == null)\r
1108                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1109             if (objectName.IsAbsoluteUri)\r
1110                 throw new ArgumentException("The objectName must be relative","objectName");\r
1111             Contract.EndContractBlock();\r
1112 \r
1113             using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))\r
1114             {                \r
1115 \r
1116                 using (var client = new RestClient(_baseClient))\r
1117                 {\r
1118                     if (!String.IsNullOrWhiteSpace(account))\r
1119                         client.BaseAddress = GetAccountUrl(account);\r
1120                     try\r
1121                     {\r
1122                         client.Parameters.Clear();\r
1123 \r
1124                         client.Head(container.Combine(objectName), 3);\r
1125 \r
1126                         if (client.TimedOut)\r
1127                             return ObjectInfo.Empty;\r
1128 \r
1129                         switch (client.StatusCode)\r
1130                         {\r
1131                             case HttpStatusCode.OK:\r
1132                             case HttpStatusCode.NoContent:\r
1133                                 var keys = client.ResponseHeaders.AllKeys.AsQueryable();\r
1134                                 var tags = client.GetMeta("X-Object-Meta-");\r
1135                                 var extensions = (from key in keys\r
1136                                                   where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")\r
1137                                                   select new {Name = key, Value = client.ResponseHeaders[key]})\r
1138                                     .ToDictionary(t => t.Name, t => t.Value);\r
1139 \r
1140                                 var permissions=client.GetHeaderValue("X-Object-Sharing", true);\r
1141                                 \r
1142                                 \r
1143                                 var info = new ObjectInfo\r
1144                                                {\r
1145                                                    Account = account,\r
1146                                                    Container = container,\r
1147                                                    Name = objectName,\r
1148                                                    ETag = client.GetHeaderValue("ETag"),\r
1149                                                    UUID=client.GetHeaderValue("X-Object-UUID"),\r
1150                                                    X_Object_Hash = client.GetHeaderValue("X-Object-Hash"),\r
1151                                                    Content_Type = client.GetHeaderValue("Content-Type"),\r
1152                                                    Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),\r
1153                                                    Tags = tags,\r
1154                                                    Last_Modified = client.LastModified,\r
1155                                                    Extensions = extensions,\r
1156                                                    ContentEncoding=client.GetHeaderValue("Content-Encoding",true),\r
1157                                                    ContendDisposition = client.GetHeaderValue("Content-Disposition",true),\r
1158                                                    Manifest=client.GetHeaderValue("X-Object-Manifest",true),\r
1159                                                    PublicUrl=client.GetHeaderValue("X-Object-Public",true),  \r
1160                                                    StorageUri=StorageUrl,\r
1161                                                };\r
1162                                 info.SetPermissions(permissions);\r
1163                                 return info;\r
1164                             case HttpStatusCode.NotFound:\r
1165                                 return ObjectInfo.Empty;\r
1166                             default:\r
1167                                 throw new WebException(\r
1168                                     String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",\r
1169                                                   objectName, client.StatusCode));\r
1170                         }\r
1171 \r
1172                     }\r
1173                     catch (RetryException)\r
1174                     {\r
1175                         Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);\r
1176                         return ObjectInfo.Empty;\r
1177                     }\r
1178                     catch (WebException e)\r
1179                     {\r
1180                         Log.Error(\r
1181                             String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",\r
1182                                           objectName, client.StatusCode), e);\r
1183                         throw;\r
1184                     }\r
1185                 }                \r
1186             }\r
1187 \r
1188         }\r
1189 \r
1190         \r
1191 \r
1192         public void CreateFolder(string account, Uri container, Uri folder)\r
1193         {\r
1194             if (container == null)\r
1195                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1196             if (container.IsAbsoluteUri)\r
1197                 throw new ArgumentException("The container must be relative","container");\r
1198             if (folder == null)\r
1199                 throw new ArgumentNullException("folder", "The objectName property can't be empty");\r
1200             if (folder.IsAbsoluteUri)\r
1201                 throw new ArgumentException("The objectName must be relative","folder");\r
1202             Contract.EndContractBlock();\r
1203 \r
1204             var folderUri=container.Combine(folder);            \r
1205             var targetUri = GetTargetUri(account).Combine(folderUri);\r
1206             var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
1207             \r
1208             message.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY);\r
1209             message.Headers.Add("Content-Length", "0");\r
1210             using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)\r
1211             {\r
1212                 if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)\r
1213                     throw CreateWebException("CreateFolder", response.StatusCode);\r
1214             }\r
1215 /*\r
1216             using (var client = new RestClient(_baseClient))\r
1217             {\r
1218                 if (!String.IsNullOrWhiteSpace(account))\r
1219                     client.BaseAddress = GetAccountUrl(account);\r
1220 \r
1221                 client.Parameters.Clear();\r
1222                 client.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY);\r
1223                 client.Headers.Add("Content-Length", "0");\r
1224                 client.PutWithRetry(folderUri, 3);\r
1225 \r
1226                 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)\r
1227                     throw CreateWebException("CreateFolder", client.StatusCode);\r
1228             }\r
1229 */\r
1230         }\r
1231 \r
1232      \r
1233 \r
1234         public ContainerInfo GetContainerInfo(string account, Uri container)\r
1235         {\r
1236             if (container == null)\r
1237                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1238             if (container.IsAbsoluteUri)\r
1239                 throw new ArgumentException("The container must be relative","container");\r
1240             Contract.EndContractBlock();\r
1241 \r
1242             using (var client = new RestClient(_baseClient))\r
1243             {\r
1244                 if (!String.IsNullOrWhiteSpace(account))\r
1245                     client.BaseAddress = GetAccountUrl(account);                \r
1246 \r
1247                 client.Head(container);\r
1248                 switch (client.StatusCode)\r
1249                 {\r
1250                     case HttpStatusCode.OK:\r
1251                     case HttpStatusCode.NoContent:\r
1252                         var tags = client.GetMeta("X-Container-Meta-");\r
1253                         var policies = client.GetMeta("X-Container-Policy-");\r
1254 \r
1255                         var containerInfo = new ContainerInfo\r
1256                                                 {\r
1257                                                     Account=account,\r
1258                                                     Name = container,\r
1259                                                     StorageUrl=StorageUrl.ToString(),\r
1260                                                     Count =\r
1261                                                         long.Parse(client.GetHeaderValue("X-Container-Object-Count")),\r
1262                                                     Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),\r
1263                                                     BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),\r
1264                                                     BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),\r
1265                                                     Last_Modified=client.LastModified,\r
1266                                                     Tags=tags,\r
1267                                                     Policies=policies\r
1268                                                 };\r
1269                         \r
1270 \r
1271                         return containerInfo;\r
1272                     case HttpStatusCode.NotFound:\r
1273                         return ContainerInfo.Empty;\r
1274                     default:\r
1275                         throw CreateWebException("GetContainerInfo", client.StatusCode);\r
1276                 }\r
1277             }\r
1278         }\r
1279 \r
1280         public void CreateContainer(string account, Uri container)\r
1281         {\r
1282             if (container == null)\r
1283                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1284             if (container.IsAbsoluteUri)\r
1285                 throw new ArgumentException("The container must be relative","container");\r
1286             Contract.EndContractBlock();\r
1287 \r
1288             var targetUri=GetTargetUri(account).Combine(container);\r
1289             var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
1290             message.Headers.Add("Content-Length", "0");\r
1291             using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)\r
1292             {            \r
1293                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};\r
1294                 if (!expectedCodes.Contains(response.StatusCode))\r
1295                     throw CreateWebException("CreateContainer", response.StatusCode);\r
1296             }\r
1297 /*\r
1298             using (var client = new RestClient(_baseClient))\r
1299             {\r
1300                 if (!String.IsNullOrWhiteSpace(account))\r
1301                     client.BaseAddress = GetAccountUrl(account);\r
1302 \r
1303                 client.PutWithRetry(container, 3);\r
1304                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};\r
1305                 if (!expectedCodes.Contains(client.StatusCode))\r
1306                     throw CreateWebException("CreateContainer", client.StatusCode);\r
1307             }\r
1308 */\r
1309         }\r
1310 \r
1311         public void WipeContainer(string account, Uri container)\r
1312         {\r
1313             if (container == null)\r
1314                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1315             if (container.IsAbsoluteUri)\r
1316                 throw new ArgumentException("The container must be relative", "container");\r
1317             Contract.EndContractBlock();\r
1318 \r
1319             DeleteContainer(account, new Uri(String.Format("{0}&delimiter=//", container), UriKind.Relative));\r
1320         }\r
1321 \r
1322 \r
1323         public void DeleteContainer(string account, Uri container)\r
1324         {\r
1325             if (container == null)\r
1326                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1327             if (container.IsAbsoluteUri)\r
1328                 throw new ArgumentException("The container must be relative","container");\r
1329             Contract.EndContractBlock();\r
1330 \r
1331             var targetUri = GetTargetUri(account).Combine(container);\r
1332             var message = new HttpRequestMessage(HttpMethod.Delete, targetUri);\r
1333             using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)\r
1334             {\r
1335                 var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent };\r
1336                 if (!expectedCodes.Contains(response.StatusCode))\r
1337                     throw CreateWebException("DeleteContainer", response.StatusCode);\r
1338             }\r
1339 /*\r
1340             using (var client = new RestClient(_baseClient))\r
1341             {\r
1342                 if (!String.IsNullOrWhiteSpace(account))\r
1343                     client.BaseAddress = GetAccountUrl(account);\r
1344 \r
1345                 client.DeleteWithRetry(container, 3);\r
1346                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};\r
1347                 if (!expectedCodes.Contains(client.StatusCode))\r
1348                     throw CreateWebException("DeleteContainer", client.StatusCode);\r
1349             }\r
1350 */\r
1351 \r
1352         }\r
1353 \r
1354         /// <summary>\r
1355         /// \r
1356         /// </summary>\r
1357         /// <param name="account"></param>\r
1358         /// <param name="container"></param>\r
1359         /// <param name="objectName"></param>\r
1360         /// <param name="fileName"></param>\r
1361         /// <param name="cancellationToken"> </param>\r
1362         /// <returns></returns>\r
1363         /// <remarks>This method should have no timeout or a very long one</remarks>\r
1364         //Asynchronously download the object specified by *objectName* in a specific *container* to \r
1365         // a local file\r
1366         public async Task GetObject(string account, Uri container, Uri objectName, string fileName,CancellationToken cancellationToken)\r
1367         {\r
1368             if (container == null)\r
1369                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1370             if (container.IsAbsoluteUri)\r
1371                 throw new ArgumentException("The container must be relative","container");\r
1372             if (objectName == null)\r
1373                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1374             if (objectName.IsAbsoluteUri)\r
1375                 throw new ArgumentException("The objectName must be relative","objectName");\r
1376             Contract.EndContractBlock();\r
1377                         \r
1378 \r
1379             try\r
1380             {\r
1381                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient\r
1382                 //object to avoid concurrency errors.\r
1383                 //\r
1384                 //Download operations take a long time therefore they have no timeout.\r
1385                 using(var client = new RestClient(_baseClient) { Timeout = 0 })\r
1386                 {\r
1387                     if (!String.IsNullOrWhiteSpace(account))\r
1388                         client.BaseAddress = GetAccountUrl(account);\r
1389 \r
1390                     //The container and objectName are relative names. They are joined with the client's\r
1391                     //BaseAddress to create the object's absolute address\r
1392                     var builder = client.GetAddressBuilder(container, objectName);\r
1393                     var uri = builder.Uri;\r
1394 \r
1395                     //Download progress is reported to the Trace log\r
1396                     Log.InfoFormat("[GET] START {0}", objectName);\r
1397                     /*client.DownloadProgressChanged += (sender, args) =>\r
1398                                                       Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1399                                                                      fileName, args.ProgressPercentage,\r
1400                                                                      args.BytesReceived,\r
1401                                                                      args.TotalBytesToReceive);*/\r
1402                     var progress = new Progress<DownloadProgressChangedEventArgs>(args =>\r
1403                                 {\r
1404                                     Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1405                                                    fileName, args.ProgressPercentage,\r
1406                                                    args.BytesReceived,\r
1407                                                    args.TotalBytesToReceive);\r
1408                                     if (DownloadProgressChanged!=null)\r
1409                                         DownloadProgressChanged(this, new DownloadArgs(args));\r
1410                                 });\r
1411                     \r
1412                     //Start downloading the object asynchronously                    \r
1413                     await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false);\r
1414 \r
1415                     //Once the download completes\r
1416                     //Delete the local client object\r
1417                 }\r
1418                 //And report failure or completion\r
1419             }\r
1420             catch (Exception exc)\r
1421             {\r
1422                 Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);\r
1423                 throw;\r
1424             }\r
1425 \r
1426             Log.InfoFormat("[GET] END {0}", objectName);                                             \r
1427 \r
1428 \r
1429         }\r
1430 \r
1431         public async Task<IList<string>> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash)\r
1432         {\r
1433             if (container == null)\r
1434                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1435             if (container.IsAbsoluteUri)\r
1436                 throw new ArgumentException("The container must be relative","container");\r
1437             if (objectName == null)\r
1438                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1439             if (objectName.IsAbsoluteUri)\r
1440                 throw new ArgumentException("The objectName must be relative","objectName");\r
1441             if (hash == null)\r
1442                 throw new ArgumentNullException("hash");\r
1443             if (String.IsNullOrWhiteSpace(Token))\r
1444                 throw new InvalidOperationException("Invalid Token");\r
1445             if (StorageUrl == null)\r
1446                 throw new InvalidOperationException("Invalid Storage Url");\r
1447             Contract.EndContractBlock();\r
1448 \r
1449             \r
1450 \r
1451             //The container and objectName are relative names. They are joined with the client's\r
1452             //BaseAddress to create the object's absolute address\r
1453 \r
1454             var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);\r
1455   \r
1456 \r
1457             var uri = new Uri(String.Format("{0}?format=json&hashmap",targetUri),UriKind.Absolute);\r
1458 \r
1459             \r
1460             //Send the tree hash as Json to the server            \r
1461             var jsonHash = hash.ToJson();\r
1462             if (Log.IsDebugEnabled)\r
1463                 Log.DebugFormat("Hashes:\r\n{0}", jsonHash);\r
1464 \r
1465             var message = new HttpRequestMessage(HttpMethod.Put, uri)\r
1466             {\r
1467                 Content = new StringContent(jsonHash)\r
1468             };\r
1469             message.Headers.Add("ETag",hash.TopHash.ToHashString());\r
1470             \r
1471             //Don't use a timeout because putting the hashmap may be a long process\r
1472 \r
1473             using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false))\r
1474             {\r
1475                 var empty = (IList<string>)new List<string>();\r
1476                 \r
1477                 switch (response.StatusCode)\r
1478                 {\r
1479                     case HttpStatusCode.Created:\r
1480                         //The server will respond either with 201-created if all blocks were already on the server\r
1481                         return empty;\r
1482                     case HttpStatusCode.Conflict:\r
1483                         //or with a 409-conflict and return the list of missing parts\r
1484                         using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))\r
1485                         using(var reader=stream.GetLoggedReader(Log))\r
1486                         {                            \r
1487                             var serializer = new JsonSerializer();                            \r
1488                             serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);\r
1489                             var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));\r
1490                             return hashes;\r
1491                         }                        \r
1492                     default:\r
1493                         //All other cases are unexpected\r
1494                         //Ensure that failure codes raise exceptions\r
1495                         response.EnsureSuccessStatusCode();\r
1496                         //And log any other codes as warngings, but continute processing\r
1497                         Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",response.StatusCode,response.ReasonPhrase);\r
1498                         return empty;\r
1499                 }\r
1500             }\r
1501 \r
1502         }\r
1503 \r
1504 /*\r
1505         public Task<IList<string>> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash)\r
1506         {\r
1507             if (container == null)\r
1508                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1509             if (container.IsAbsoluteUri)\r
1510                 throw new ArgumentException("The container must be relative","container");\r
1511             if (objectName == null)\r
1512                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1513             if (objectName.IsAbsoluteUri)\r
1514                 throw new ArgumentException("The objectName must be relative","objectName");\r
1515             if (hash == null)\r
1516                 throw new ArgumentNullException("hash");\r
1517             if (String.IsNullOrWhiteSpace(Token))\r
1518                 throw new InvalidOperationException("Invalid Token");\r
1519             if (StorageUrl == null)\r
1520                 throw new InvalidOperationException("Invalid Storage Url");\r
1521             Contract.EndContractBlock();\r
1522 \r
1523             \r
1524 \r
1525             //Don't use a timeout because putting the hashmap may be a long process\r
1526             var client = new RestClient(_baseClient) { Timeout = 0 };           \r
1527             if (!String.IsNullOrWhiteSpace(account))\r
1528                 client.BaseAddress = GetAccountUrl(account);\r
1529 \r
1530             //The container and objectName are relative names. They are joined with the client's\r
1531             //BaseAddress to create the object's absolute address\r
1532             var builder = client.GetAddressBuilder(container, objectName);\r
1533             builder.Query = "format=json&hashmap";\r
1534             var uri = builder.Uri;\r
1535 \r
1536 \r
1537             //Send the tree hash as Json to the server            \r
1538             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";\r
1539             var jsonHash = hash.ToJson();\r
1540                         \r
1541             client.Headers.Add("ETag",hash.TopHash.ToHashString());\r
1542             var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);\r
1543             if (Log.IsDebugEnabled)\r
1544                 Log.DebugFormat("Hashes:\r\n{0}", jsonHash);\r
1545             return uploadTask.ContinueWith(t =>\r
1546             {\r
1547 \r
1548                 var empty = (IList<string>)new List<string>();\r
1549                 \r
1550 \r
1551                 //The server will respond either with 201-created if all blocks were already on the server\r
1552                 if (client.StatusCode == HttpStatusCode.Created)                    \r
1553                 {\r
1554                     //in which case we return an empty hash list\r
1555                     return empty;\r
1556                 }\r
1557                 //or with a 409-conflict and return the list of missing parts\r
1558                 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception                \r
1559                 if (t.IsFaulted)\r
1560                 {\r
1561                     var ex = t.Exception.InnerException;\r
1562                     var we = ex as WebException;\r
1563                     var response = we.Response as HttpWebResponse;\r
1564                     if (response!=null && response.StatusCode==HttpStatusCode.Conflict)\r
1565                     {\r
1566                         //In case of 409 the missing parts will be in the response content                        \r
1567                         using (var stream = response.GetResponseStream())\r
1568                         using(var reader=stream.GetLoggedReader(Log))\r
1569                         {\r
1570                             //We used to have to cleanup the content before returning it because it contains\r
1571                             //error content after the list of hashes\r
1572                             //\r
1573                             //As of 30/1/2012, the result is a proper Json array so we don't need to read the content\r
1574                             //line by line\r
1575                             \r
1576                             var serializer = new JsonSerializer();                            \r
1577                             serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);\r
1578                             var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));\r
1579                             return hashes;\r
1580                         }                        \r
1581                     }                    \r
1582                     //Any other status code is unexpected and the exception should be rethrown\r
1583                     Log.LogError(response);\r
1584                     throw ex;\r
1585                     \r
1586                 }\r
1587 \r
1588                 //Any other status code is unexpected but there was no exception. We can probably continue processing\r
1589                 Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);                    \r
1590                 \r
1591                 return empty;\r
1592             });\r
1593 \r
1594         }\r
1595 \r
1596 */\r
1597 \r
1598        \r
1599 \r
1600         public async Task<byte[]> GetBlock(string account, Uri container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)\r
1601         {\r
1602             if (String.IsNullOrWhiteSpace(Token))\r
1603                 throw new InvalidOperationException("Invalid Token");\r
1604             if (StorageUrl == null)\r
1605                 throw new InvalidOperationException("Invalid Storage Url");\r
1606             if (container == null)\r
1607                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1608             if (container.IsAbsoluteUri)\r
1609                 throw new ArgumentException("The container must be relative","container");\r
1610             if (relativeUrl == null)\r
1611                 throw new ArgumentNullException("relativeUrl");\r
1612             if (end.HasValue && end < 0)\r
1613                 throw new ArgumentOutOfRangeException("end");\r
1614             if (start < 0)\r
1615                 throw new ArgumentOutOfRangeException("start");\r
1616             Contract.EndContractBlock();\r
1617 \r
1618 \r
1619             var targetUri = GetTargetUri(account).Combine(container).Combine(relativeUrl);\r
1620             var message = new HttpRequestMessage(HttpMethod.Get, targetUri);\r
1621             message.Headers.Range.Ranges.Add(new RangeItemHeaderValue(start,end));\r
1622 \r
1623             //Don't use a timeout because putting the hashmap may be a long process\r
1624 \r
1625             IProgress<DownloadArgs> progress = new Progress<DownloadArgs>(args =>\r
1626                 {\r
1627                     Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1628                                     targetUri.Segments.Last(), args.ProgressPercentage,\r
1629                                     args.BytesReceived,\r
1630                                     args.TotalBytesToReceive);\r
1631 \r
1632                     if (DownloadProgressChanged!=null)\r
1633                         DownloadProgressChanged(this,  args);\r
1634                 });\r
1635 \r
1636 \r
1637             using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, HttpCompletionOption.ResponseHeadersRead,\r
1638                                                           cancellationToken).ConfigureAwait(false))\r
1639             using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))\r
1640             using(var targetStream=new MemoryStream())\r
1641             {\r
1642                 \r
1643                 long totalSize = response.Content.Headers.ContentLength ?? 0;\r
1644                 long total = 0;\r
1645                 var buffer = new byte[65536];\r
1646                 int read;\r
1647                 while ((read = await stream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) != 0)\r
1648                 {\r
1649                     total += read;\r
1650                     progress.Report(new DownloadArgs(total, totalSize));\r
1651                     await targetStream.WriteAsync(buffer, 0, read).ConfigureAwait(false);\r
1652                 }\r
1653 \r
1654                 var result = targetStream.ToArray();\r
1655                 return result;\r
1656             }\r
1657             /*using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end})\r
1658             {\r
1659                 if (!String.IsNullOrWhiteSpace(account))\r
1660                     client.BaseAddress = GetAccountUrl(account);\r
1661 \r
1662                 var builder = client.GetAddressBuilder(container, relativeUrl);\r
1663                 var uri = builder.Uri;\r
1664 \r
1665 /*                client.DownloadProgressChanged += (sender, args) =>\r
1666                                                       {\r
1667                                                           Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1668                                                                           uri.Segments.Last(), args.ProgressPercentage,\r
1669                                                                           args.BytesReceived,\r
1670                                                                           args.TotalBytesToReceive);\r
1671                                                           DownloadProgressChanged(sender, args);\r
1672                                                       };#1#\r
1673                 var progress = new Progress<DownloadProgressChangedEventArgs>(args =>\r
1674                 {\r
1675                     Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1676                                     uri.Segments.Last(), args.ProgressPercentage,\r
1677                                     args.BytesReceived,\r
1678                                     args.TotalBytesToReceive);\r
1679                     if (DownloadProgressChanged!=null)\r
1680                         DownloadProgressChanged(this, args);\r
1681                 });\r
1682 \r
1683 \r
1684                 var result = await client.DownloadDataTaskAsync(uri, cancellationToken,progress).ConfigureAwait(false);\r
1685                 return result;\r
1686             }*/\r
1687         }\r
1688 \r
1689         public event EventHandler<UploadArgs> UploadProgressChanged;\r
1690         public event EventHandler<DownloadArgs> DownloadProgressChanged;\r
1691         \r
1692 \r
1693         public async Task PostBlock(string account, Uri container, byte[] block, int offset, int count,CancellationToken token)\r
1694         {\r
1695             if (container == null)\r
1696                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1697             if (container.IsAbsoluteUri)\r
1698                 throw new ArgumentException("The container must be relative","container");\r
1699             if (block == null)\r
1700                 throw new ArgumentNullException("block");\r
1701             if (offset < 0 || offset >= block.Length)\r
1702                 throw new ArgumentOutOfRangeException("offset");\r
1703             if (count < 0 || count > block.Length)\r
1704                 throw new ArgumentOutOfRangeException("count");\r
1705             if (String.IsNullOrWhiteSpace(Token))\r
1706                 throw new InvalidOperationException("Invalid Token");\r
1707             if (StorageUrl == null)\r
1708                 throw new InvalidOperationException("Invalid Storage Url");                        \r
1709             Contract.EndContractBlock();\r
1710 \r
1711 \r
1712             try\r
1713             {\r
1714 /*\r
1715                 var containerUri = GetTargetUri(account).Combine(container);\r
1716                 var targetUri = new Uri(String.Format("{0}?update", containerUri));\r
1717 \r
1718 \r
1719                 //Don't use a timeout because putting the hashmap may be a long process\r
1720 \r
1721 \r
1722                 Log.InfoFormat("[BLOCK POST] START");\r
1723 \r
1724 \r
1725                 var progress = new Progress<UploadArgs>(args =>\r
1726                 {\r
1727                     Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",\r
1728                         args.ProgressPercentage,\r
1729                         args.BytesSent,\r
1730                         args.TotalBytesToSend);\r
1731                     if (UploadProgressChanged != null)\r
1732                         UploadProgressChanged(this,args);\r
1733                 });\r
1734 \r
1735                 var message = new HttpRequestMessage(HttpMethod.Post, targetUri);                \r
1736                 message.Content = new  ByteArrayContent(block, offset, count);\r
1737 \r
1738                 //Send the block\r
1739                 using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false))\r
1740                 {                    \r
1741                     Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");\r
1742                     response.EnsureSuccessStatusCode();\r
1743                 }\r
1744                 Log.InfoFormat("[BLOCK POST] END");*/\r
1745                 using (var client = new RestClient(_baseClient) { Timeout = 0 })\r
1746                 {\r
1747                     if (!String.IsNullOrWhiteSpace(account))\r
1748                         client.BaseAddress = GetAccountUrl(account);\r
1749 \r
1750                     var builder = client.GetAddressBuilder(container, _emptyUri);\r
1751                     //We are doing an update\r
1752                     builder.Query = "update";\r
1753                     var uri = builder.Uri;\r
1754 \r
1755                     client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";\r
1756 \r
1757                     Log.InfoFormat("[BLOCK POST] START");\r
1758 \r
1759                     client.UploadFileCompleted += (sender, args) =>\r
1760                                                   Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");\r
1761 \r
1762                     var progress=new Progress<UploadProgressChangedEventArgs>(args=>\r
1763                     {\r
1764                         Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",\r
1765                                         args.ProgressPercentage, args.BytesSent,\r
1766                                         args.TotalBytesToSend);\r
1767                         if (UploadProgressChanged!=null)\r
1768                             UploadProgressChanged(this, new UploadArgs(args));                                                                                      \r
1769                     });\r
1770                     var buffer = new byte[count];\r
1771                     Buffer.BlockCopy(block, offset, buffer, 0, count);\r
1772                     //Send the block\r
1773                     await client.UploadDataTaskAsync(uri, "POST", buffer,token,progress).ConfigureAwait(false);\r
1774                     Log.InfoFormat("[BLOCK POST] END");\r
1775                 }\r
1776             }\r
1777             catch (TaskCanceledException )\r
1778             {\r
1779                 Log.Info("Aborting block");\r
1780                 throw;\r
1781             }\r
1782             catch (Exception exc)\r
1783             {\r
1784                 Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);\r
1785                 throw;\r
1786             }\r
1787         }\r
1788 \r
1789 \r
1790         public async Task<TreeHash> GetHashMap(string account, Uri container, Uri objectName)\r
1791         {\r
1792             if (container == null)\r
1793                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1794             if (container.IsAbsoluteUri)\r
1795                 throw new ArgumentException("The container must be relative","container");\r
1796             if (objectName == null)\r
1797                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1798             if (objectName.IsAbsoluteUri)\r
1799                 throw new ArgumentException("The objectName must be relative","objectName");\r
1800             if (String.IsNullOrWhiteSpace(Token))\r
1801                 throw new InvalidOperationException("Invalid Token");\r
1802             if (StorageUrl == null)\r
1803                 throw new InvalidOperationException("Invalid Storage Url");\r
1804             Contract.EndContractBlock();\r
1805 \r
1806             try\r
1807             {\r
1808 \r
1809                 var objectUri = GetTargetUri(account).Combine(container).Combine(objectName);\r
1810                 var targetUri = new Uri(String.Format("{0}?format=json&hashmap", objectUri));\r
1811 \r
1812                 //Start downloading the object asynchronously\r
1813                 var json = await GetStringAsync(targetUri, "").ConfigureAwait(false);\r
1814                 var treeHash = TreeHash.Parse(json);\r
1815                 Log.InfoFormat("[GET HASH] END {0}", objectName);\r
1816                 return treeHash;\r
1817                 /*\r
1818                                 using (var client = new RestClient(_baseClient) { Timeout = 0 })\r
1819                                 {\r
1820                                     if (!String.IsNullOrWhiteSpace(account))\r
1821                                         client.BaseAddress = GetAccountUrl(account);\r
1822 \r
1823                                     //The container and objectName are relative names. They are joined with the client's\r
1824                                     //BaseAddress to create the object's absolute address\r
1825                     \r
1826                                     var builder = client.GetAddressBuilder(container, objectName);\r
1827                                     builder.Query = "format=json&hashmap";\r
1828                                     var uri = builder.Uri;\r
1829 \r
1830 \r
1831                                     var json = await client.DownloadStringTaskAsync(uri).ConfigureAwait(false);\r
1832                                     var treeHash = TreeHash.Parse(json);\r
1833                                     Log.InfoFormat("[GET HASH] END {0}", objectName);\r
1834                                     return treeHash;\r
1835                                 }\r
1836                 */\r
1837             }\r
1838             catch (Exception exc)\r
1839             {\r
1840                 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);\r
1841                 throw;\r
1842             }\r
1843 \r
1844         }\r
1845 \r
1846 \r
1847         /// <summary>\r
1848         /// \r
1849         /// </summary>\r
1850         /// <param name="account"></param>\r
1851         /// <param name="container"></param>\r
1852         /// <param name="objectName"></param>\r
1853         /// <param name="fileName"></param>\r
1854         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>\r
1855         /// <param name="contentType"> </param>\r
1856         /// <remarks>>This method should have no timeout or a very long one</remarks>\r
1857         public async Task PutObject(string account, Uri container, Uri objectName, string fileName, string hash = Signature.MERKLE_EMPTY, string contentType = "application/octet-stream")\r
1858         {\r
1859             if (container == null)\r
1860                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1861             if (container.IsAbsoluteUri)\r
1862                 throw new ArgumentException("The container must be relative","container");\r
1863             if (objectName == null)\r
1864                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1865             if (objectName.IsAbsoluteUri)\r
1866                 throw new ArgumentException("The objectName must be relative","objectName");\r
1867             if (String.IsNullOrWhiteSpace(fileName))\r
1868                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");\r
1869 /*\r
1870             if (!File.Exists(fileName) && !Directory.Exists(fileName))\r
1871                 throw new FileNotFoundException("The file or directory does not exist",fileName);\r
1872 */\r
1873             \r
1874             try\r
1875             {\r
1876 \r
1877                 using (var client = new RestClient(_baseClient) { Timeout = 0 })\r
1878                 {\r
1879                     if (!String.IsNullOrWhiteSpace(account))\r
1880                         client.BaseAddress = GetAccountUrl(account);\r
1881 \r
1882                     var builder = client.GetAddressBuilder(container, objectName);\r
1883                     var uri = builder.Uri;\r
1884 \r
1885                     string etag = hash ;\r
1886 \r
1887                     client.Headers.Add("Content-Type", contentType);\r
1888                     if (contentType!=ObjectInfo.CONTENT_TYPE_DIRECTORY)\r
1889                         client.Headers.Add("ETag", etag);\r
1890 \r
1891 \r
1892                     Log.InfoFormat("[PUT] START {0}", objectName);\r
1893                     client.UploadProgressChanged += (sender, args) =>\r
1894                                                         {\r
1895                                                             using (ThreadContext.Stacks["PUT"].Push("Progress"))\r
1896                                                             {\r
1897                                                                 Log.InfoFormat("{0} {1}% {2} of {3}", fileName,\r
1898                                                                                args.ProgressPercentage,\r
1899                                                                                args.BytesSent, args.TotalBytesToSend);\r
1900                                                             }\r
1901                                                         };\r
1902 \r
1903                     client.UploadFileCompleted += (sender, args) =>\r
1904                                                       {\r
1905                                                           using (ThreadContext.Stacks["PUT"].Push("Progress"))\r
1906                                                           {\r
1907                                                               Log.InfoFormat("Completed {0}", fileName);\r
1908                                                           }\r
1909                                                       }; \r
1910                     \r
1911                     if (contentType==ObjectInfo.CONTENT_TYPE_DIRECTORY)\r
1912                         await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false);\r
1913                     else\r
1914                         await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false);\r
1915                 }\r
1916 \r
1917                 Log.InfoFormat("[PUT] END {0}", objectName);\r
1918             }\r
1919             catch (Exception exc)\r
1920             {\r
1921                 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);\r
1922                 throw;\r
1923             }                \r
1924 \r
1925         }\r
1926        \r
1927         \r
1928 /*\r
1929         private static string CalculateHash(string fileName)\r
1930         {\r
1931             Contract.Requires(!String.IsNullOrWhiteSpace(fileName));\r
1932             Contract.EndContractBlock();\r
1933 \r
1934             string hash;\r
1935             using (var hasher = MD5.Create())\r
1936             using(var stream=File.OpenRead(fileName))\r
1937             {\r
1938                 var hashBuilder=new StringBuilder();\r
1939                 foreach (byte b in hasher.ComputeHash(stream))\r
1940                     hashBuilder.Append(b.ToString("x2").ToLower());\r
1941                 hash = hashBuilder.ToString();                \r
1942             }\r
1943             return hash;\r
1944         }\r
1945 */\r
1946 \r
1947         \r
1948         public void MoveObject(string account, Uri sourceContainer, Uri oldObjectName, Uri targetContainer, Uri newObjectName)\r
1949         {\r
1950             if (sourceContainer == null)\r
1951                 throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");\r
1952             if (sourceContainer.IsAbsoluteUri)\r
1953                 throw new ArgumentException("The sourceContainer must be relative","sourceContainer");\r
1954             if (oldObjectName == null)\r
1955                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");\r
1956             if (oldObjectName.IsAbsoluteUri)\r
1957                 throw new ArgumentException("The oldObjectName must be relative","oldObjectName");\r
1958             if (targetContainer == null)\r
1959                 throw new ArgumentNullException("targetContainer", "The targetContainer property can't be empty");\r
1960             if (targetContainer.IsAbsoluteUri)\r
1961                 throw new ArgumentException("The targetContainer must be relative","targetContainer");\r
1962             if (newObjectName == null)\r
1963                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");\r
1964             if (newObjectName.IsAbsoluteUri)\r
1965                 throw new ArgumentException("The newObjectName must be relative","newObjectName");\r
1966             Contract.EndContractBlock();\r
1967 \r
1968             var baseUri = GetTargetUri(account);\r
1969             var targetUri = baseUri.Combine(targetContainer).Combine(newObjectName);\r
1970             var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, oldObjectName),UriKind.Relative);\r
1971 \r
1972             var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
1973             message.Headers.Add("X-Move-From", sourceUri.ToString());\r
1974             using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)\r
1975             {\r
1976                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
1977                 if (!expectedCodes.Contains(response.StatusCode))\r
1978                     throw CreateWebException("MoveObject", response.StatusCode);\r
1979             }\r
1980 /*\r
1981             using (var client = new RestClient(_baseClient))\r
1982             {\r
1983                 if (!String.IsNullOrWhiteSpace(account))\r
1984                     client.BaseAddress = GetAccountUrl(account);\r
1985 \r
1986                 client.Headers.Add("X-Move-From", sourceUri.ToString());\r
1987                 client.PutWithRetry(targetUri, 3);\r
1988 \r
1989                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
1990                 if (!expectedCodes.Contains(client.StatusCode))\r
1991                     throw CreateWebException("MoveObject", client.StatusCode);\r
1992             }\r
1993 */\r
1994         }\r
1995 \r
1996         public void DeleteObject(string account, Uri sourceContainer, Uri objectName, bool isDirectory)\r
1997         {\r
1998             if (sourceContainer == null)\r
1999                 throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");\r
2000             if (sourceContainer.IsAbsoluteUri)\r
2001                 throw new ArgumentException("The sourceContainer must be relative","sourceContainer");\r
2002             if (objectName == null)\r
2003                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
2004             if (objectName.IsAbsoluteUri)\r
2005                 throw new ArgumentException("The objectName must be relative","objectName");\r
2006             Contract.EndContractBlock();\r
2007 \r
2008             var targetUrl = FolderConstants.TrashContainer + "/" + objectName;\r
2009 /*\r
2010             if (isDirectory)\r
2011                 targetUrl = targetUrl + "?delimiter=/";\r
2012 */\r
2013 \r
2014             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);\r
2015 \r
2016             using (var client = new RestClient(_baseClient))\r
2017             {\r
2018                 if (!String.IsNullOrWhiteSpace(account))\r
2019                     client.BaseAddress = GetAccountUrl(account);\r
2020 \r
2021                 client.Headers.Add("X-Move-From", sourceUrl);\r
2022                 client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);\r
2023                 Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);\r
2024                 client.PutWithRetry(new Uri(targetUrl,UriKind.Relative), 3);\r
2025 \r
2026                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};\r
2027                 if (!expectedCodes.Contains(client.StatusCode))\r
2028                     throw CreateWebException("DeleteObject", client.StatusCode);\r
2029             }\r
2030         }\r
2031 \r
2032       \r
2033         private static WebException CreateWebException(string operation, HttpStatusCode statusCode)\r
2034         {\r
2035             return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));\r
2036         }\r
2037 \r
2038 \r
2039 /*\r
2040         public IEnumerable<ObjectInfo> ListDirectories(ContainerInfo container)\r
2041         {\r
2042             var directories=this.ListObjects(container.Account, container.Name, "/");\r
2043         }\r
2044 */\r
2045 \r
2046         public bool CanUpload(string account, ObjectInfo cloudFile)\r
2047         {\r
2048             Contract.Requires(!String.IsNullOrWhiteSpace(account));\r
2049             Contract.Requires(cloudFile!=null);\r
2050 \r
2051             using (var client = new RestClient(_baseClient))\r
2052             {\r
2053                 if (!String.IsNullOrWhiteSpace(account))\r
2054                     client.BaseAddress = GetAccountUrl(account);\r
2055 \r
2056 \r
2057                 var parts = cloudFile.Name.ToString().Split('/');\r
2058                 var folder = String.Join("/", parts,0,parts.Length-1);\r
2059 \r
2060                 var fileName = String.Format("{0}/{1}.pithos.ignore", folder, Guid.NewGuid());\r
2061                 var fileUri=fileName.ToEscapedUri();                                            \r
2062 \r
2063                 client.Parameters.Clear();\r
2064                 try\r
2065                 {\r
2066                     var relativeUri = cloudFile.Container.Combine(fileUri);\r
2067                     client.PutWithRetry(relativeUri, 3, @"application/octet-stream");\r
2068 \r
2069                     var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
2070                     var result=(expectedCodes.Contains(client.StatusCode));\r
2071                     DeleteObject(account, cloudFile.Container, fileUri, cloudFile.IsDirectory);\r
2072                     return result;\r
2073                 }\r
2074                 catch\r
2075                 {\r
2076                     return false;\r
2077                 }\r
2078             }\r
2079         }\r
2080     }\r
2081 \r
2082     public class ShareAccountInfo\r
2083     {\r
2084         public DateTime? last_modified { get; set; }\r
2085         public string name { get; set; }\r
2086     }\r
2087 }\r