code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/CloudFilesClient.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="CloudFilesClient.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 \r
  43 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos\r
  44 //\r
  45 // The class provides methods to upload/download files, delete files, manage containers\r
  46 \r
  47 \r
  48 using System;\r
  49 using System.Collections.Generic;\r
  50 using System.Collections.Specialized;\r
  51 using System.ComponentModel.Composition;\r
  52 using System.Diagnostics;\r
  53 using System.Diagnostics.Contracts;\r
  54 using System.IO;\r
  55 using System.Linq;\r
  56 using System.Net;\r
  57 using System.Net.Http;\r
  58 using System.Net.Http.Headers;\r
  59 using System.Reflection;\r
  60 using System.ServiceModel.Channels;\r
  61 using System.Text;\r
  62 using System.Threading;\r
  63 using System.Threading.Tasks;\r
  64 using Newtonsoft.Json;\r
  65 using Newtonsoft.Json.Linq;\r
  66 using Pithos.Interfaces;\r
  67 using Pithos.Network;\r
  68 using log4net;\r
  69 \r
  70 namespace Pithos.Network\r
  71 {\r
  72 \r
  73     [Export(typeof(ICloudClient))]\r
  74     public class CloudFilesClient:ICloudClient,IDisposable\r
  75     {\r
  76         private const string TOKEN_HEADER = "X-Auth-Token";\r
  77         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  78 \r
  79         //CloudFilesClient uses *_baseClient* internally to communicate with the server\r
  80         //RestClient provides a REST-friendly interface over the standard WebClient.\r
  81         private RestClient _baseClient;\r
  82 \r
  83         private HttpClient _baseHttpClient;\r
  84         private HttpClient _baseHttpClientNoTimeout;\r
  85         \r
  86 \r
  87         //During authentication the client provides a UserName \r
  88         public string UserName { get; set; }\r
  89         \r
  90         //and and ApiKey to the server\r
  91         public string ApiKey { get; set; }\r
  92         \r
  93         //And receives an authentication Token. This token must be provided in ALL other operations,\r
  94         //in the X-Auth-Token header\r
  95         private string _token;\r
  96         private readonly string _emptyGuid = Guid.Empty.ToString();\r
  97         private readonly Uri _emptyUri = new Uri("",UriKind.Relative);\r
  98 \r
  99         private HttpClientHandler _httpClientHandler = new HttpClientHandler\r
 100                                                            {\r
 101                                                                AllowAutoRedirect = true,\r
 102                                                                AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,\r
 103                                                                UseCookies = true,   \r
 104                                                            };\r
 105 \r
 106 \r
 107         public string Token\r
 108         {\r
 109             get { return _token; }\r
 110             set\r
 111             {\r
 112                 _token = value;\r
 113                 _baseClient.Headers[TOKEN_HEADER] = value;\r
 114             }\r
 115         }\r
 116 \r
 117         //The client also receives a StorageUrl after authentication. All subsequent operations must\r
 118         //use this url\r
 119         public Uri StorageUrl { get; set; }\r
 120 \r
 121 \r
 122         public Uri RootAddressUri { get; set; }\r
 123 \r
 124 \r
 125         public double DownloadPercentLimit { get; set; }\r
 126         public double UploadPercentLimit { get; set; }\r
 127 \r
 128         public string AuthenticationUrl { get; set; }\r
 129 \r
 130  \r
 131         public string VersionPath\r
 132         {\r
 133             get { return UsePithos ? "v1" : "v1.0"; }\r
 134         }\r
 135 \r
 136         public bool UsePithos { get; set; }\r
 137 \r
 138 \r
 139         BufferManager _bufferManager=BufferManager.CreateBufferManager(TreeHash.DEFAULT_BLOCK_SIZE*4,(int)TreeHash.DEFAULT_BLOCK_SIZE);\r
 140         private string _userCatalogUrl;\r
 141 \r
 142         public CloudFilesClient(string userName, string apiKey)\r
 143         {\r
 144             UserName = userName;\r
 145             ApiKey = apiKey;\r
 146             _userCatalogUrl = "https://pithos.okeanos.io/user_catalogs";\r
 147         }\r
 148 \r
 149         public CloudFilesClient(AccountInfo accountInfo)\r
 150         {\r
 151             Contract.Requires<ArgumentNullException>(accountInfo!=null,"accountInfo is null");\r
 152             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));\r
 153             Contract.Ensures(StorageUrl != null);\r
 154             Contract.Ensures(_baseClient != null);\r
 155             Contract.Ensures(RootAddressUri != null);\r
 156             Contract.EndContractBlock();          \r
 157 \r
 158             _baseClient = new RestClient\r
 159             {\r
 160                 BaseAddress = accountInfo.StorageUri.ToString(),\r
 161                 Timeout = 30000,\r
 162                 Retries = 3,\r
 163             };\r
 164             StorageUrl = accountInfo.StorageUri;\r
 165             Token = accountInfo.Token;\r
 166             UserName = accountInfo.UserName;\r
 167             _userCatalogUrl = "https://pithos.okeanos.io/user_catalogs";\r
 168             //Get the root address (StorageUrl without the account)\r
 169             var storageUrl = StorageUrl.AbsoluteUri;\r
 170             var usernameIndex = storageUrl.LastIndexOf(UserName);\r
 171             var rootUrl = storageUrl.Substring(0, usernameIndex);\r
 172             RootAddressUri = new Uri(rootUrl);\r
 173 \r
 174             var httpClientHandler = new HttpClientHandler\r
 175             {\r
 176                 AllowAutoRedirect = true,\r
 177                 AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,\r
 178                 UseCookies = true,\r
 179             };\r
 180 \r
 181 \r
 182             _baseHttpClient = new HttpClient(httpClientHandler)\r
 183             {\r
 184                 BaseAddress = StorageUrl,\r
 185                 Timeout = TimeSpan.FromSeconds(30)\r
 186             };\r
 187             _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);\r
 188 \r
 189             _baseHttpClientNoTimeout = new HttpClient(httpClientHandler)\r
 190             {\r
 191                 BaseAddress = StorageUrl,\r
 192                 Timeout = TimeSpan.FromMilliseconds(-1)\r
 193             };\r
 194             _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);\r
 195 \r
 196 \r
 197         }\r
 198 \r
 199 \r
 200         private static void AssertStatusOK(HttpResponseMessage response, string message)\r
 201         {\r
 202             var statusCode = response.StatusCode;\r
 203             if (statusCode >= HttpStatusCode.BadRequest)\r
 204                 throw new WebException(String.Format("{0} with code {1} - {2}", message, statusCode, response.ReasonPhrase));\r
 205         }\r
 206 \r
 207         public async Task<AccountInfo> Authenticate()\r
 208         {\r
 209             if (String.IsNullOrWhiteSpace(UserName))\r
 210                 throw new InvalidOperationException("UserName is empty");\r
 211             if (String.IsNullOrWhiteSpace(ApiKey))\r
 212                 throw new InvalidOperationException("ApiKey is empty");\r
 213             if (String.IsNullOrWhiteSpace(AuthenticationUrl))\r
 214                 throw new InvalidOperationException("AuthenticationUrl is empty");\r
 215             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));\r
 216             Contract.Ensures(StorageUrl != null);\r
 217             Contract.Ensures(_baseClient != null);\r
 218             Contract.Ensures(RootAddressUri != null);\r
 219             Contract.EndContractBlock();\r
 220 \r
 221 \r
 222             Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);\r
 223 \r
 224             var groups = new List<Group>();\r
 225 \r
 226             using (var authClient = new HttpClient(_httpClientHandler,false){ BaseAddress = new Uri(AuthenticationUrl),Timeout=TimeSpan.FromSeconds(30) })\r
 227             {                \r
 228 \r
 229                 authClient.DefaultRequestHeaders.Add("X-Auth-User", UserName);\r
 230                 authClient.DefaultRequestHeaders.Add("X-Auth-Key", ApiKey);\r
 231 \r
 232                 string storageUrl;\r
 233                 string token;\r
 234                 \r
 235                 using (var response = await authClient.GetAsyncWithRetries(new Uri(VersionPath, UriKind.Relative),3).ConfigureAwait(false)) // .DownloadStringWithRetryRelative(new Uri(VersionPath, UriKind.Relative), 3);                    \r
 236                 {\r
 237                     AssertStatusOK(response,"Authentication failed");\r
 238                 \r
 239                     storageUrl = response.Headers.GetFirstValue("X-Storage-Url");\r
 240                     if (String.IsNullOrWhiteSpace(storageUrl))\r
 241                         throw new InvalidOperationException("Failed to obtain storage url");\r
 242 \r
 243                     token = response.Headers.GetFirstValue(TOKEN_HEADER);\r
 244                     if (String.IsNullOrWhiteSpace(token))\r
 245                         throw new InvalidOperationException("Failed to obtain token url");\r
 246 \r
 247                 }\r
 248 \r
 249 \r
 250                 _baseClient = new RestClient\r
 251                 {\r
 252                     BaseAddress = storageUrl,\r
 253                     Timeout = 30000,\r
 254                     Retries = 3,                    \r
 255                 };\r
 256 \r
 257                 StorageUrl = new Uri(storageUrl);\r
 258                 Token = token;\r
 259 \r
 260                 \r
 261 \r
 262                                                                \r
 263                 //Get the root address (StorageUrl without the account)\r
 264                 var usernameIndex=storageUrl.LastIndexOf(UserName);\r
 265                 var rootUrl = storageUrl.Substring(0, usernameIndex);\r
 266                 RootAddressUri = new Uri(rootUrl);\r
 267                 \r
 268 \r
 269                 _baseHttpClient = new HttpClient(_httpClientHandler,false)\r
 270                 {\r
 271                     BaseAddress = StorageUrl,\r
 272                     Timeout = TimeSpan.FromSeconds(30)\r
 273                 };\r
 274                 _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, token);\r
 275 \r
 276                 _baseHttpClientNoTimeout = new HttpClient(_httpClientHandler,false)\r
 277                 {\r
 278                     BaseAddress = StorageUrl,\r
 279                     Timeout = TimeSpan.FromMilliseconds(-1)\r
 280                 };\r
 281                 _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, token);\r
 282 \r
 283                 /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();\r
 284                 groups = (from key in keys\r
 285                             where key.StartsWith("X-Account-Group-")\r
 286                             let name = key.Substring(16)\r
 287                             select new Group(name, authClient.ResponseHeaders[key]))\r
 288                         .ToList();\r
 289                     \r
 290 */\r
 291             }\r
 292 \r
 293             Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);\r
 294             Debug.Assert(_baseClient!=null);\r
 295             var displayName = UserName;\r
 296             Guid uuid;\r
 297             if (Guid.TryParse(UserName, out uuid))\r
 298             {\r
 299                 displayName = await ResolveName(uuid);\r
 300             }\r
 301             return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,DisplayName=displayName,Groups=groups};            \r
 302 \r
 303         }\r
 304 \r
 305         private static void TraceStart(string method, Uri actualAddress)\r
 306         {\r
 307             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);\r
 308         }\r
 309 \r
 310         private async Task<string> GetStringAsync(Uri targetUri, string errorMessage,DateTimeOffset? since=null)\r
 311         {\r
 312             TraceStart("GET",targetUri);\r
 313             var request = new HttpRequestMessage(HttpMethod.Get, targetUri);\r
 314             //request.Headers.Add("User-Agent", "Pithos+ Custom Header");\r
 315             if (since.HasValue)\r
 316             {\r
 317                 request.Headers.IfModifiedSince = since.Value;\r
 318             }\r
 319             using (var response = await _baseHttpClient.SendAsyncWithRetries(request,3).ConfigureAwait(false))\r
 320             {\r
 321                 AssertStatusOK(response, errorMessage);\r
 322 \r
 323                 if (response.StatusCode == HttpStatusCode.NoContent)\r
 324                     return String.Empty;\r
 325 \r
 326                 var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);\r
 327                 return content;\r
 328             }\r
 329         }\r
 330 \r
 331         public async Task<IList<ContainerInfo>> ListContainers(string account)\r
 332         {\r
 333 \r
 334             var targetUrl = GetTargetUrl(account);\r
 335             var targetUri = new Uri(String.Format("{0}?format=json", targetUrl));\r
 336             var result = await GetStringAsync(targetUri, "List Containers failed").ConfigureAwait(false);\r
 337             if (String.IsNullOrWhiteSpace(result))\r
 338                 return new List<ContainerInfo>();\r
 339             var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(result);\r
 340             foreach (var info in infos)\r
 341             {\r
 342                 info.Account = account;\r
 343             }\r
 344             return infos;\r
 345         }\r
 346 \r
 347         \r
 348         private string GetAccountUrl(string account)\r
 349         {\r
 350             return RootAddressUri.Combine(account).AbsoluteUri;\r
 351         }\r
 352 \r
 353         public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)\r
 354         {\r
 355             using (ThreadContext.Stacks["Share"].Push("List Accounts"))\r
 356             {\r
 357                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 358 \r
 359                 var targetUri = new Uri(String.Format("{0}?format=json", RootAddressUri), UriKind.Absolute);\r
 360                 var content=TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListSharingAccounts failed", since).ConfigureAwait(false)).Result;\r
 361 \r
 362                 //If the result is empty, return an empty list,\r
 363                 var infos = String.IsNullOrWhiteSpace(content)\r
 364                                 ? new List<ShareAccountInfo>()\r
 365                             //Otherwise deserialize the account list into a list of ShareAccountInfos\r
 366                                 : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);\r
 367 \r
 368                 Log.DebugFormat("END");\r
 369                 return infos;\r
 370             }\r
 371         }\r
 372 \r
 373 \r
 374         /// <summary>\r
 375         /// Request listing of all objects in a container modified since a specific time.\r
 376         /// If the *since* value is missing, return all objects\r
 377         /// </summary>\r
 378         /// <param name="knownContainers">Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new\r
 379         /// and should be polled anyway\r
 380         /// </param>\r
 381         /// <param name="since"></param>\r
 382         /// <returns></returns>\r
 383         public IList<ObjectInfo> ListSharedObjects(HashSet<string> knownContainers, DateTimeOffset? since)\r
 384         {\r
 385 \r
 386             using (ThreadContext.Stacks["Share"].Push("List Objects"))\r
 387             {\r
 388                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 389                 //'since' is not used here because we need to have ListObjects return a NoChange result\r
 390                 //for all shared accounts,containers\r
 391 \r
 392                 Func<ContainerInfo, string> getKey = c => String.Format("{0}\\{1}", c.Account, c.Name);\r
 393                 \r
 394                 var containers = (from account in ListSharingAccounts()\r
 395                                  let conts = TaskEx.Run(async ()=>await ListContainers(account.name).ConfigureAwait(false)).Result\r
 396                                  from container in conts\r
 397                                  select container).ToList();                \r
 398                 var items = from container in containers \r
 399                             let actualSince=knownContainers.Contains(getKey(container))?since:null\r
 400                             select ListObjects(container.Account , container.Name,  actualSince);\r
 401                 var objects=items.SelectMany(r=> r).ToList();\r
 402 \r
 403                 //For each object\r
 404                 //Check parents recursively up to (but not including) the container.\r
 405                 //If parents are missing, add them to the list\r
 406                 //Need function to calculate all parent URLs\r
 407                 objects = AddMissingParents(objects);\r
 408                 \r
 409                 //Store any new containers\r
 410                 foreach (var container in containers)\r
 411                 {\r
 412                     knownContainers.Add(getKey(container));\r
 413                 }\r
 414 \r
 415 \r
 416 \r
 417                 if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 418                 return objects;\r
 419             }\r
 420         }\r
 421 \r
 422         private List<ObjectInfo> AddMissingParents(List<ObjectInfo> objects)\r
 423         {\r
 424             //TODO: Remove short-circuit when we decide to use Missing Parents functionality\r
 425             //return objects;\r
 426 \r
 427             var existingUris = objects.ToDictionary(o => o.Uri, o => o);\r
 428             foreach (var objectInfo in objects)\r
 429             {\r
 430                 //Can be null when retrieving objects to show in selective sync\r
 431                 if (objectInfo.Name == null)\r
 432                     continue;\r
 433 \r
 434                 //No need to unescape here, the parts will be used to create new ObjectInfos\r
 435                 var parts = objectInfo.Name.ToString().Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries);\r
 436                 //If there is no parent, skip\r
 437                 if (parts.Length == 1)\r
 438                     continue;\r
 439                 var baseParts = new[]\r
 440                                   {\r
 441                                       objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container.ToString()\r
 442                                   };\r
 443                 for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++)\r
 444                 {\r
 445                     var nameparts = parts.Range(0, partIdx).ToArray();\r
 446                     var parentName= String.Join("/", nameparts);\r
 447 \r
 448                     var parentParts = baseParts.Concat(nameparts);\r
 449                     var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts);\r
 450                     \r
 451                     var parentUri = new Uri(parentUrl, UriKind.Absolute);\r
 452 \r
 453                     ObjectInfo existingInfo;\r
 454                     if (!existingUris.TryGetValue(parentUri,out existingInfo))\r
 455                     {\r
 456                         var h = parentUrl.GetHashCode();\r
 457                         var reverse = new string(parentUrl.Reverse().ToArray());\r
 458                         var rh = reverse.GetHashCode();\r
 459                         var b1 = BitConverter.GetBytes(h);\r
 460                         var b2 = BitConverter.GetBytes(rh);\r
 461                         var g = new Guid(0,0,0,b1.Concat(b2).ToArray());\r
 462                         \r
 463 \r
 464                         existingUris[parentUri] = new ObjectInfo\r
 465                                                       {\r
 466                                                           Account = objectInfo.Account,\r
 467                                                           Container = objectInfo.Container,\r
 468                                                           Content_Type = ObjectInfo.CONTENT_TYPE_DIRECTORY,\r
 469                                                           ETag = Signature.MERKLE_EMPTY,\r
 470                                                           X_Object_Hash = Signature.MERKLE_EMPTY,\r
 471                                                           Name=new Uri(parentName,UriKind.Relative),\r
 472                                                           StorageUri=objectInfo.StorageUri,\r
 473                                                           Bytes = 0,\r
 474                                                           UUID=g.ToString(),                                                          \r
 475                                                       };\r
 476                     }\r
 477                 }\r
 478             }\r
 479             return existingUris.Values.ToList();\r
 480         }\r
 481 \r
 482         public void SetTags(ObjectInfo target,IDictionary<string,string> tags)\r
 483         {\r
 484             Contract.Requires<InvalidOperationException>(!String.IsNullOrWhiteSpace(Token),"The Token is not set");\r
 485             Contract.Requires<InvalidOperationException>(StorageUrl != null,"The StorageUrl is not set");\r
 486             Contract.Requires<ArgumentNullException>(target != null,"target is null");\r
 487             Contract.EndContractBlock();\r
 488 \r
 489             using (ThreadContext.Stacks["Share"].Push("Share Object"))\r
 490             {\r
 491                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 492 \r
 493                 using (var client = new RestClient(_baseClient))\r
 494                 {\r
 495 \r
 496                     client.BaseAddress = GetAccountUrl(target.Account);\r
 497 \r
 498                     client.Parameters.Clear();\r
 499                     client.Parameters.Add("update", "");\r
 500 \r
 501                     foreach (var tag in tags)\r
 502                     {\r
 503                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);\r
 504                         client.Headers.Add(headerTag, tag.Value);\r
 505                     }\r
 506                     \r
 507                     client.DownloadStringWithRetryRelative(target.Container, 3);\r
 508 \r
 509                     \r
 510                     client.AssertStatusOK("SetTags failed");\r
 511                     //If the status is NOT ACCEPTED we have a problem\r
 512                     if (client.StatusCode != HttpStatusCode.Accepted)\r
 513                     {\r
 514                         Log.Error("Failed to set tags");\r
 515                         throw new Exception("Failed to set tags");\r
 516                     }\r
 517 \r
 518                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 519                 }\r
 520             }\r
 521 \r
 522 \r
 523         }\r
 524 \r
 525         public void ShareObject(string account, Uri container, Uri objectName, string shareTo, bool read, bool write)\r
 526         {\r
 527 \r
 528             Contract.Requires<InvalidOperationException>(!String.IsNullOrWhiteSpace(Token), "The Token is not set");\r
 529             Contract.Requires<InvalidOperationException>(StorageUrl != null, "The StorageUrl is not set");\r
 530             Contract.Requires<ArgumentNullException>(container != null, "container is null");\r
 531             Contract.Requires<ArgumentException>(!container.IsAbsoluteUri, "container is absolute");\r
 532             Contract.Requires<ArgumentNullException>(objectName != null, "objectName is null");\r
 533             Contract.Requires<ArgumentException>(!objectName.IsAbsoluteUri, "objectName  is absolute");\r
 534             Contract.Requires<ArgumentNullException>(!String.IsNullOrWhiteSpace(account), "account is not set");\r
 535             Contract.Requires<ArgumentNullException>(!String.IsNullOrWhiteSpace(shareTo), "shareTo is not set");\r
 536             Contract.EndContractBlock();\r
 537 \r
 538             using (ThreadContext.Stacks["Share"].Push("Share Object"))\r
 539             {\r
 540                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 541                 \r
 542                 using (var client = new RestClient(_baseClient))\r
 543                 {\r
 544 \r
 545                     client.BaseAddress = GetAccountUrl(account);\r
 546 \r
 547                     client.Parameters.Clear();\r
 548                     client.Parameters.Add("format", "json");\r
 549 \r
 550                     string permission = "";\r
 551                     if (write)\r
 552                         permission = String.Format("write={0}", shareTo);\r
 553                     else if (read)\r
 554                         permission = String.Format("read={0}", shareTo);\r
 555                     client.Headers.Add("X-Object-Sharing", permission);\r
 556 \r
 557                     var content = client.DownloadStringWithRetryRelative(container, 3);\r
 558 \r
 559                     client.AssertStatusOK("ShareObject failed");\r
 560 \r
 561                     //If the result is empty, return an empty list,\r
 562                     var infos = String.IsNullOrWhiteSpace(content)\r
 563                                     ? new List<ObjectInfo>()\r
 564                                 //Otherwise deserialize the object list into a list of ObjectInfos\r
 565                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 566 \r
 567                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 568                 }\r
 569             }\r
 570 \r
 571 \r
 572         }\r
 573 \r
 574         public async Task<AccountInfo> GetAccountPolicies(AccountInfo accountInfo)\r
 575         {\r
 576             if (accountInfo==null)\r
 577                 throw new ArgumentNullException("accountInfo");\r
 578             Contract.EndContractBlock();\r
 579 \r
 580             using (ThreadContext.Stacks["Account"].Push("GetPolicies"))\r
 581             {\r
 582                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 583 \r
 584 /*\r
 585                 if (_baseClient == null)\r
 586                 {\r
 587                     _baseClient = new RestClient\r
 588                     {\r
 589                         BaseAddress = accountInfo.StorageUri.ToString(),\r
 590                         Timeout = 30000,\r
 591                         Retries = 3,\r
 592                     };\r
 593                 }\r
 594 \r
 595 */                \r
 596                 var containerUri = GetTargetUri(accountInfo.UserName);\r
 597                 var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);\r
 598                 using(var response=await _baseHttpClient.HeadAsyncWithRetries(targetUri,3).ConfigureAwait(false))\r
 599                 {\r
 600                     \r
 601                     var quotaValue=response.Headers.GetFirstValue("X-Account-Policy-Quota");\r
 602                     var bytesValue = response.Headers.GetFirstValue("X-Account-Bytes-Used");\r
 603                     long quota, bytes;\r
 604                     if (long.TryParse(quotaValue, out quota))\r
 605                         accountInfo.Quota = quota;\r
 606                     if (long.TryParse(bytesValue, out bytes))\r
 607                         accountInfo.BytesUsed = bytes;\r
 608 \r
 609                     return accountInfo;   \r
 610                 }\r
 611 \r
 612 \r
 613                 //using (var client = new RestClient(_baseClient))\r
 614                 //{\r
 615                 //    if (!String.IsNullOrWhiteSpace(accountInfo.UserName))\r
 616                 //        client.BaseAddress = GetAccountUrl(accountInfo.UserName);\r
 617 \r
 618                 //    client.Parameters.Clear();\r
 619                 //    client.Parameters.Add("format", "json");                    \r
 620                 //    client.Head(_emptyUri, 3);\r
 621 \r
 622                 //    var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];\r
 623                 //    var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];\r
 624 \r
 625                 //    long quota, bytes;\r
 626                 //    if (long.TryParse(quotaValue, out quota))\r
 627                 //        accountInfo.Quota = quota;\r
 628                 //    if (long.TryParse(bytesValue, out bytes))\r
 629                 //        accountInfo.BytesUsed = bytes;\r
 630                     \r
 631                 //    return accountInfo;\r
 632 \r
 633                 //}\r
 634 \r
 635             }\r
 636         }\r
 637 \r
 638         public void UpdateMetadata(ObjectInfo objectInfo)\r
 639         {\r
 640             Contract.Requires<ArgumentNullException>(objectInfo != null,"objectInfo is null");\r
 641             Contract.EndContractBlock();\r
 642 \r
 643             using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))\r
 644             {\r
 645                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 646 \r
 647 \r
 648                 using(var client=new RestClient(_baseClient))\r
 649                 {\r
 650 \r
 651                     client.BaseAddress = GetAccountUrl(objectInfo.Account);\r
 652                     \r
 653                     client.Parameters.Clear();\r
 654                     \r
 655 \r
 656                     //Set Tags\r
 657                     foreach (var tag in objectInfo.Tags)\r
 658                     {\r
 659                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);\r
 660                         client.Headers.Add(headerTag, tag.Value);\r
 661                     }\r
 662 \r
 663                     //Set Permissions\r
 664 \r
 665                     var permissions=objectInfo.GetPermissionString();\r
 666                     client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);\r
 667 \r
 668                     client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);\r
 669                     client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);\r
 670                     client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);\r
 671                     var isPublic = objectInfo.IsPublic.ToString().ToLower();\r
 672                     client.Headers.Add("X-Object-Public", isPublic);\r
 673 \r
 674 \r
 675                     var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);\r
 676                     client.PostWithRetry(new Uri(address,UriKind.Relative),"application/xml");\r
 677                     \r
 678                     client.AssertStatusOK("UpdateMetadata failed");\r
 679                     //If the status is NOT ACCEPTED or OK we have a problem\r
 680                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))\r
 681                     {\r
 682                         Log.Error("Failed to update metadata");\r
 683                         throw new Exception("Failed to update metadata");\r
 684                     }\r
 685 \r
 686                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 687                 }\r
 688             }\r
 689 \r
 690         }\r
 691 \r
 692         public void UpdateMetadata(ContainerInfo containerInfo)\r
 693         {\r
 694             if (containerInfo == null)\r
 695                 throw new ArgumentNullException("containerInfo");\r
 696             Contract.EndContractBlock();\r
 697 \r
 698             using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))\r
 699             {\r
 700                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 701 \r
 702 \r
 703                 using(var client=new RestClient(_baseClient))\r
 704                 {\r
 705 \r
 706                     client.BaseAddress = GetAccountUrl(containerInfo.Account);\r
 707                     \r
 708                     client.Parameters.Clear();\r
 709                     \r
 710 \r
 711                     //Set Tags\r
 712                     foreach (var tag in containerInfo.Tags)\r
 713                     {\r
 714                         var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);\r
 715                         client.Headers.Add(headerTag, tag.Value);\r
 716                     }\r
 717 \r
 718                     \r
 719                     //Set Policies\r
 720                     foreach (var policy in containerInfo.Policies)\r
 721                     {\r
 722                         var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);\r
 723                         client.Headers.Add(headerPolicy, policy.Value);\r
 724                     }\r
 725 \r
 726 \r
 727                     var uriBuilder = client.GetAddressBuilder(containerInfo.Name,_emptyUri);\r
 728                     var uri = uriBuilder.Uri;\r
 729 \r
 730                     client.UploadValues(uri,new NameValueCollection());\r
 731 \r
 732 \r
 733                     client.AssertStatusOK("UpdateMetadata failed");\r
 734                     //If the status is NOT ACCEPTED or OK we have a problem\r
 735                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))\r
 736                     {\r
 737                         Log.Error("Failed to update metadata");\r
 738                         throw new Exception("Failed to update metadata");\r
 739                     }\r
 740 \r
 741                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 742                 }\r
 743             }\r
 744 \r
 745         }\r
 746 \r
 747        \r
 748 \r
 749 \r
 750         public IList<ObjectInfo> ListObjects(string account, Uri container, DateTimeOffset? since = null)\r
 751         {\r
 752 /*\r
 753             if (container==null)\r
 754                 throw new ArgumentNullException("container");\r
 755             if (container.IsAbsoluteUri)\r
 756                 throw new ArgumentException("container");\r
 757             Contract.EndContractBlock();\r
 758 */\r
 759 \r
 760             using (ThreadContext.Stacks["Objects"].Push("List"))\r
 761             {\r
 762 \r
 763                 var containerUri = GetTargetUri(account).Combine(container);\r
 764                 var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);\r
 765 \r
 766                 var content =TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result;\r
 767 \r
 768                 //304 will result in an empty string. Empty containers return an empty json array\r
 769                 if (String.IsNullOrWhiteSpace(content))\r
 770                      return new[] {new NoModificationInfo(account, container)};\r
 771 \r
 772                  //If the result is empty, return an empty list,\r
 773                  var infos = String.IsNullOrWhiteSpace(content)\r
 774                                  ? new List<ObjectInfo>()\r
 775                              //Otherwise deserialize the object list into a list of ObjectInfos\r
 776                                  : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 777 \r
 778                  foreach (var info in infos)\r
 779                  {\r
 780                      info.Container = container;\r
 781                      info.Account = account;\r
 782                      info.StorageUri = StorageUrl;\r
 783                  }\r
 784                  if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 785                  return infos;\r
 786             }\r
 787         }\r
 788 \r
 789         public IList<ObjectInfo> ListObjects(string account, Uri container, Uri folder, DateTimeOffset? since = null)\r
 790         {\r
 791 /*            if (container==null)\r
 792                 throw new ArgumentNullException("container");\r
 793             if (container.IsAbsoluteUri)\r
 794                 throw new ArgumentException("container");\r
 795             Contract.EndContractBlock();*/\r
 796 \r
 797             using (ThreadContext.Stacks["Objects"].Push("List"))\r
 798             {\r
 799                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 800 \r
 801                 var containerUri = GetTargetUri(account).Combine(container);\r
 802                 var targetUri = new Uri(String.Format("{0}?format=json&path={1}", containerUri,folder), UriKind.Absolute);\r
 803                 var content = TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result;                \r
 804 \r
 805                 //304 will result in an empty string. Empty containers return an empty json array\r
 806                 if (String.IsNullOrWhiteSpace(content))\r
 807                     return new[] { new NoModificationInfo(account, container) };\r
 808 \r
 809 \r
 810                 var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 811                 foreach (var info in infos)\r
 812                 {\r
 813                     info.Account = account;\r
 814                     if (info.Container == null)\r
 815                         info.Container = container;\r
 816                     info.StorageUri = StorageUrl;\r
 817                 }\r
 818                 if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 819                 return infos;\r
 820 /*\r
 821                 using (var client = new RestClient(_baseClient))\r
 822                 {\r
 823                     if (!String.IsNullOrWhiteSpace(account))\r
 824                         client.BaseAddress = GetAccountUrl(account);\r
 825 \r
 826                     client.Parameters.Clear();\r
 827                     client.Parameters.Add("format", "json");\r
 828                     client.Parameters.Add("path", folder.ToString());\r
 829                     client.IfModifiedSince = since;\r
 830                     var content = client.DownloadStringWithRetryRelative(container, 3);\r
 831                     client.AssertStatusOK("ListObjects failed");\r
 832 \r
 833                     if (client.StatusCode==HttpStatusCode.NotModified)\r
 834                         return new[]{new NoModificationInfo(account,container,folder)};\r
 835 \r
 836                     var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
 837                     foreach (var info in infos)\r
 838                     {\r
 839                         info.Account = account;\r
 840                         if (info.Container == null)\r
 841                             info.Container = container;\r
 842                         info.StorageUri = StorageUrl;\r
 843                     }\r
 844                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 845                     return infos;\r
 846                 }\r
 847 */\r
 848             }\r
 849         }\r
 850 \r
 851  \r
 852         public async Task<bool> ContainerExists(string account, Uri container)\r
 853         {\r
 854             if (container==null)\r
 855                 throw new ArgumentNullException("container", "The container property can't be empty");\r
 856             if (container.IsAbsoluteUri)\r
 857                 throw new ArgumentException( "The container must be relative","container");\r
 858             Contract.EndContractBlock();\r
 859 \r
 860             using (ThreadContext.Stacks["Containters"].Push("Exists"))\r
 861             {\r
 862                 if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
 863 \r
 864                 var targetUri = GetTargetUri(account).Combine(container);\r
 865 \r
 866                 using (var response =await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3))\r
 867                 {\r
 868 \r
 869                     bool result;\r
 870                     switch (response.StatusCode)\r
 871                     {\r
 872                         case HttpStatusCode.OK:\r
 873                         case HttpStatusCode.NoContent:\r
 874                             result = true;\r
 875                             break;\r
 876                         case HttpStatusCode.NotFound:\r
 877                             result = false;\r
 878                             break;\r
 879                         default:\r
 880                             throw CreateWebException("ContainerExists", response.StatusCode);\r
 881                     }\r
 882                     if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
 883 \r
 884                     return result;\r
 885                 }\r
 886 \r
 887             }\r
 888         }\r
 889 \r
 890         private Uri GetTargetUri(string account)\r
 891         {\r
 892             return new Uri(GetTargetUrl(account),UriKind.Absolute);\r
 893         }\r
 894 \r
 895         private string GetTargetUrl(string account)\r
 896         {\r
 897             return String.IsNullOrWhiteSpace(account)\r
 898                        ? _baseHttpClient.BaseAddress.ToString()\r
 899                        : GetAccountUrl(account);\r
 900         }\r
 901 \r
 902         public async Task<bool> ObjectExists(string account, Uri container, Uri objectName)\r
 903         {\r
 904             if (container == null)\r
 905                 throw new ArgumentNullException("container", "The container property can't be empty");\r
 906             if (container.IsAbsoluteUri)\r
 907                 throw new ArgumentException("The container must be relative","container");\r
 908             if (objectName == null)\r
 909                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
 910             if (objectName.IsAbsoluteUri)\r
 911                 throw new ArgumentException("The objectName must be relative","objectName");\r
 912             Contract.EndContractBlock();\r
 913 \r
 914                 var targetUri=GetTargetUri(account).Combine(container).Combine(objectName);\r
 915 \r
 916             using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).ConfigureAwait(false))\r
 917             {\r
 918                 switch (response.StatusCode)\r
 919                 {\r
 920                     case HttpStatusCode.OK:\r
 921                     case HttpStatusCode.NoContent:\r
 922                         return true;\r
 923                     case HttpStatusCode.NotFound:\r
 924                         return false;\r
 925                     default:\r
 926                         throw CreateWebException("ObjectExists", response.StatusCode);\r
 927                 }\r
 928             }\r
 929         }\r
 930 \r
 931         public async Task<ObjectInfo> GetObjectInfo(string account, Uri container, Uri objectName)\r
 932         {\r
 933             if (container == null)\r
 934                 throw new ArgumentNullException("container", "The container property can't be empty");\r
 935             if (container.IsAbsoluteUri)\r
 936                 throw new ArgumentException("The container must be relative", "container");\r
 937             if (objectName == null)\r
 938                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
 939             if (objectName.IsAbsoluteUri)\r
 940                 throw new ArgumentException("The objectName must be relative", "objectName");\r
 941             Contract.EndContractBlock();\r
 942 \r
 943             using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))\r
 944             {\r
 945 \r
 946                 var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);\r
 947                 try\r
 948                 {\r
 949                     using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true))\r
 950                     {\r
 951                         switch (response.StatusCode)\r
 952                         {\r
 953                             case HttpStatusCode.OK:\r
 954                             case HttpStatusCode.NoContent:\r
 955                                 var tags = response.Headers.GetMeta("X-Object-Meta-");\r
 956                                 var extensions = (from header in response.Headers\r
 957                                                   where\r
 958                                                       header.Key.StartsWith("X-Object-") &&\r
 959                                                       !header.Key.StartsWith("X-Object-Meta-")\r
 960                                                   select new {Name = header.Key, Value = header.Value.FirstOrDefault()})\r
 961                                     .ToDictionary(t => t.Name, t => t.Value);\r
 962 \r
 963                                 var permissions = response.Headers.GetFirstValue("X-Object-Sharing");\r
 964 \r
 965 \r
 966                                 var info = new ObjectInfo\r
 967                                                {\r
 968                                                    Account = account,\r
 969                                                    Container = container,\r
 970                                                    Name = objectName,\r
 971                                                    ETag = response.Headers.ETag.NullSafe(e=>e.Tag),\r
 972                                                    UUID = response.Headers.GetFirstValue("X-Object-UUID"),\r
 973                                                    X_Object_Hash = response.Headers.GetFirstValue("X-Object-Hash"),\r
 974                                                    Content_Type = response.Headers.GetFirstValue("Content-Type"),\r
 975                                                    Bytes = Convert.ToInt64(response.Content.Headers.ContentLength),\r
 976                                                    Tags = tags,\r
 977                                                    Last_Modified = response.Content.Headers.LastModified,\r
 978                                                    Extensions = extensions,\r
 979                                                    ContentEncoding =\r
 980                                                        response.Content.Headers.ContentEncoding.FirstOrDefault(),\r
 981                                                    ContendDisposition =\r
 982                                                        response.Content.Headers.ContentDisposition.NullSafe(c=>c.ToString()),\r
 983                                                    Manifest = response.Headers.GetFirstValue("X-Object-Manifest"),\r
 984                                                    PublicUrl = response.Headers.GetFirstValue("X-Object-Public"),\r
 985                                                    StorageUri = StorageUrl,\r
 986                                                };\r
 987                                 info.SetPermissions(permissions);\r
 988                                 return info;\r
 989                             case HttpStatusCode.NotFound:\r
 990                                 return ObjectInfo.Empty;\r
 991                             default:\r
 992                                 throw new WebException(\r
 993                                     String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",\r
 994                                                   objectName, response.StatusCode));\r
 995                         }\r
 996                     }\r
 997                 }\r
 998                 catch (RetryException)\r
 999                 {\r
1000                     Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.", objectName);\r
1001                     return ObjectInfo.Empty;\r
1002                 }\r
1003                 catch (WebException e)\r
1004                 {\r
1005                     Log.Error(\r
1006                         String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status {1}",\r
1007                                       objectName, e.Status), e);\r
1008                     throw;\r
1009                 }\r
1010             }\r
1011         }\r
1012 \r
1013 \r
1014 \r
1015         public async Task CreateFolder(string account, Uri container, Uri folder)\r
1016         {\r
1017             if (container == null)\r
1018                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1019             if (container.IsAbsoluteUri)\r
1020                 throw new ArgumentException("The container must be relative","container");\r
1021             if (folder == null)\r
1022                 throw new ArgumentNullException("folder", "The objectName property can't be empty");\r
1023             if (folder.IsAbsoluteUri)\r
1024                 throw new ArgumentException("The objectName must be relative","folder");\r
1025             Contract.EndContractBlock();\r
1026 \r
1027             var folderUri=container.Combine(folder);            \r
1028             var targetUri = GetTargetUri(account).Combine(folderUri);\r
1029             var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
1030             message.Content=new StringContent("");\r
1031             message.Content.Headers.ContentType = new MediaTypeHeaderValue(ObjectInfo.CONTENT_TYPE_DIRECTORY);\r
1032             //message.Headers.Add("Content-Length", "0");\r
1033             using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))\r
1034             {\r
1035                 if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)\r
1036                     throw CreateWebException("CreateFolder", response.StatusCode);\r
1037             }\r
1038         }\r
1039 \r
1040         private Dictionary<string, string> GetMeta(HttpResponseMessage response,string metaPrefix)\r
1041         {\r
1042             Contract.Requires<ArgumentNullException>(!String.IsNullOrWhiteSpace(metaPrefix), "metaPrefix is empty");\r
1043             Contract.EndContractBlock();\r
1044 \r
1045             var dict = (from header in response.Headers\r
1046                         where header.Key.StartsWith(metaPrefix)\r
1047                          select new { Name = header.Key, Value = String.Join(",", header.Value) })\r
1048                         .ToDictionary(t => t.Name, t => t.Value);\r
1049 \r
1050           \r
1051             return dict;\r
1052         }\r
1053 \r
1054 \r
1055         public async Task<ContainerInfo> GetContainerInfo(string account, Uri container)\r
1056         {\r
1057             if (container == null)\r
1058                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1059             if (container.IsAbsoluteUri)\r
1060                 throw new ArgumentException("The container must be relative","container");\r
1061             Contract.EndContractBlock();\r
1062 \r
1063             var targetUri = GetTargetUri(account).Combine(container);            \r
1064             using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true).ConfigureAwait(false))\r
1065             {\r
1066                 if (Log.IsDebugEnabled)\r
1067                     Log.DebugFormat("ContainerInfo data: {0}\n{1}",response,await response.Content.ReadAsStringAsync().ConfigureAwait(false));\r
1068                 switch (response.StatusCode)\r
1069                 {\r
1070                     case HttpStatusCode.OK:\r
1071                     case HttpStatusCode.NoContent:\r
1072                         var tags = GetMeta(response,"X-Container-Meta-");\r
1073                         var policies = GetMeta(response,"X-Container-Policy-");\r
1074 \r
1075                         var containerInfo = new ContainerInfo\r
1076                                                 {\r
1077                                                     Account = account,\r
1078                                                     Name = container,\r
1079                                                     StorageUrl = StorageUrl.ToString(),\r
1080                                                     Count =long.Parse(response.Headers.GetFirstValue("X-Container-Object-Count")),\r
1081                                                     Bytes = long.Parse(response.Headers.GetFirstValue("X-Container-Bytes-Used")),\r
1082                                                     BlockHash = response.Headers.GetFirstValue("X-Container-Block-Hash"),\r
1083                                                     BlockSize =\r
1084                                                         int.Parse(response.Headers.GetFirstValue("X-Container-Block-Size")),\r
1085                                                     Last_Modified = response.Content.Headers.LastModified,\r
1086                                                     Tags = tags,\r
1087                                                     Policies = policies\r
1088                                                 };\r
1089 \r
1090 \r
1091                         return containerInfo;\r
1092                     case HttpStatusCode.NotFound:\r
1093                         return ContainerInfo.Empty;\r
1094                     default:\r
1095                         throw CreateWebException("GetContainerInfo", response.StatusCode);\r
1096                 }\r
1097             }            \r
1098         }\r
1099 \r
1100         public async Task CreateContainer(string account, Uri container)\r
1101         {\r
1102             if (container == null)\r
1103                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1104             if (container.IsAbsoluteUri)\r
1105                 throw new ArgumentException("The container must be relative","container");\r
1106             Contract.EndContractBlock();\r
1107 \r
1108             var targetUri=GetTargetUri(account).Combine(container);\r
1109             var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
1110             \r
1111             //message.Content.Headers.ContentLength = 0;\r
1112             using (var response =await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))\r
1113             {            \r
1114                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};\r
1115                 if (!expectedCodes.Contains(response.StatusCode))\r
1116                     throw CreateWebException("CreateContainer", response.StatusCode);\r
1117             }\r
1118         }\r
1119 \r
1120         public async Task WipeContainer(string account, Uri container)\r
1121         {\r
1122             if (container == null)\r
1123                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1124             if (container.IsAbsoluteUri)\r
1125                 throw new ArgumentException("The container must be relative", "container");\r
1126             Contract.EndContractBlock();\r
1127 \r
1128             await DeleteContainer(account, new Uri(String.Format("{0}?delimiter=/", container), UriKind.Relative)).ConfigureAwait(false);\r
1129         }\r
1130 \r
1131 \r
1132         public async Task DeleteContainer(string account, Uri container)\r
1133         {\r
1134             if (container == null)\r
1135                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1136             if (container.IsAbsoluteUri)\r
1137                 throw new ArgumentException("The container must be relative","container");\r
1138             Contract.EndContractBlock();\r
1139 \r
1140             var targetUri = GetTargetUri(account).Combine(container);\r
1141             var message = new HttpRequestMessage(HttpMethod.Delete, targetUri);\r
1142             using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))\r
1143             {\r
1144                 var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent };\r
1145                 if (!expectedCodes.Contains(response.StatusCode))\r
1146                     throw CreateWebException("DeleteContainer", response.StatusCode);\r
1147             }\r
1148 \r
1149         }\r
1150 \r
1151         /// <summary>\r
1152         /// \r
1153         /// </summary>\r
1154         /// <param name="account"></param>\r
1155         /// <param name="container"></param>\r
1156         /// <param name="objectName"></param>\r
1157         /// <param name="fileName"></param>\r
1158         /// <param name="cancellationToken"> </param>\r
1159         /// <returns></returns>\r
1160         /// <remarks>This method should have no timeout or a very long one</remarks>\r
1161         //Asynchronously download the object specified by *objectName* in a specific *container* to \r
1162         // a local file\r
1163         public async Task GetObject(string account, Uri container, Uri objectName, string fileName,CancellationToken cancellationToken)\r
1164         {\r
1165             if (container == null)\r
1166                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1167             if (container.IsAbsoluteUri)\r
1168                 throw new ArgumentException("The container must be relative","container");\r
1169             if (objectName == null)\r
1170                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1171             if (objectName.IsAbsoluteUri)\r
1172                 throw new ArgumentException("The objectName must be relative","objectName");\r
1173             Contract.EndContractBlock();\r
1174                         \r
1175 \r
1176             try\r
1177             {\r
1178                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient\r
1179                 //object to avoid concurrency errors.\r
1180                 //\r
1181                 //Download operations take a long time therefore they have no timeout.\r
1182                 using(var client = new RestClient(_baseClient) { Timeout = 0 })\r
1183                 {\r
1184                     if (!String.IsNullOrWhiteSpace(account))\r
1185                         client.BaseAddress = GetAccountUrl(account);\r
1186 \r
1187                     //The container and objectName are relative names. They are joined with the client's\r
1188                     //BaseAddress to create the object's absolute address\r
1189                     var builder = client.GetAddressBuilder(container, objectName);\r
1190                     var uri = builder.Uri;\r
1191 \r
1192                     //Download progress is reported to the Trace log\r
1193                     Log.InfoFormat("[GET] START {0}", objectName);\r
1194                     /*client.DownloadProgressChanged += (sender, args) =>\r
1195                                                       Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1196                                                                      fileName, args.ProgressPercentage,\r
1197                                                                      args.BytesReceived,\r
1198                                                                      args.TotalBytesToReceive);*/\r
1199                     var progress = new Progress<DownloadProgressChangedEventArgs>(args =>\r
1200                                 {\r
1201                                     Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1202                                                    fileName, args.ProgressPercentage,\r
1203                                                    args.BytesReceived,\r
1204                                                    args.TotalBytesToReceive);\r
1205                                     if (DownloadProgressChanged!=null)\r
1206                                         DownloadProgressChanged(this, new DownloadArgs(args));\r
1207                                 });\r
1208                     \r
1209                     //Start downloading the object asynchronously                    \r
1210                     await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false);\r
1211 \r
1212                     //Once the download completes\r
1213                     //Delete the local client object\r
1214                 }\r
1215                 //And report failure or completion\r
1216             }\r
1217             catch (Exception exc)\r
1218             {\r
1219                 Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);\r
1220                 throw;\r
1221             }\r
1222 \r
1223             Log.InfoFormat("[GET] END {0}", objectName);                                             \r
1224 \r
1225 \r
1226         }\r
1227 \r
1228         public async Task<IList<string>> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash)\r
1229         {\r
1230             if (container == null)\r
1231                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1232             if (container.IsAbsoluteUri)\r
1233                 throw new ArgumentException("The container must be relative","container");\r
1234             if (objectName == null)\r
1235                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1236             if (objectName.IsAbsoluteUri)\r
1237                 throw new ArgumentException("The objectName must be relative","objectName");\r
1238             if (hash == null)\r
1239                 throw new ArgumentNullException("hash");\r
1240             if (String.IsNullOrWhiteSpace(Token))\r
1241                 throw new InvalidOperationException("Invalid Token");\r
1242             if (StorageUrl == null)\r
1243                 throw new InvalidOperationException("Invalid Storage Url");\r
1244             Contract.EndContractBlock();\r
1245 \r
1246             \r
1247 \r
1248             //The container and objectName are relative names. They are joined with the client's\r
1249             //BaseAddress to create the object's absolute address\r
1250 \r
1251             var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);\r
1252   \r
1253 \r
1254             var uri = new Uri(String.Format("{0}?format=json&hashmap",targetUri),UriKind.Absolute);\r
1255 \r
1256             \r
1257             //Send the tree hash as Json to the server            \r
1258             var jsonHash = hash.ToJson();\r
1259             if (Log.IsDebugEnabled)\r
1260                 Log.DebugFormat("Hashes:\r\n{0}", jsonHash);\r
1261 \r
1262             var mimeType = objectName.GetMimeType();\r
1263 \r
1264             var message = new HttpRequestMessage(HttpMethod.Put, uri)\r
1265             {\r
1266                 Content = new StringContent(jsonHash)\r
1267             };\r
1268             message.Content.Headers.ContentType = mimeType;\r
1269             message.Headers.Add("ETag",hash.TopHash.ToHashString());\r
1270             \r
1271             \r
1272             //Don't use a timeout because putting the hashmap may be a long process\r
1273 \r
1274             using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false))\r
1275             {\r
1276                 var empty = (IList<string>)new List<string>();\r
1277                 \r
1278                 switch (response.StatusCode)\r
1279                 {\r
1280                     case HttpStatusCode.Created:\r
1281                         //The server will respond either with 201-created if all blocks were already on the server\r
1282                         return empty;\r
1283                     case HttpStatusCode.Conflict:\r
1284                         //or with a 409-conflict and return the list of missing parts\r
1285                         using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))\r
1286                         using(var reader=stream.GetLoggedReader(Log))\r
1287                         {                            \r
1288                             var serializer = new JsonSerializer();                            \r
1289                             serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);\r
1290                             var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));\r
1291                             return hashes;\r
1292                         }                        \r
1293                     default:\r
1294                         //All other cases are unexpected\r
1295                         //Ensure that failure codes raise exceptions\r
1296                         response.EnsureSuccessStatusCode();\r
1297                         //And log any other codes as warngings, but continute processing\r
1298                         Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",response.StatusCode,response.ReasonPhrase);\r
1299                         return empty;\r
1300                 }\r
1301             }\r
1302 \r
1303         }\r
1304 \r
1305 \r
1306         public async Task<byte[]> GetBlock(string account, Uri container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)\r
1307         {\r
1308             if (String.IsNullOrWhiteSpace(Token))\r
1309                 throw new InvalidOperationException("Invalid Token");\r
1310             if (StorageUrl == null)\r
1311                 throw new InvalidOperationException("Invalid Storage Url");\r
1312             if (container == null)\r
1313                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1314             if (container.IsAbsoluteUri)\r
1315                 throw new ArgumentException("The container must be relative","container");\r
1316             if (relativeUrl == null)\r
1317                 throw new ArgumentNullException("relativeUrl");\r
1318             if (end.HasValue && end < 0)\r
1319                 throw new ArgumentOutOfRangeException("end");\r
1320             if (start < 0)\r
1321                 throw new ArgumentOutOfRangeException("start");\r
1322             Contract.EndContractBlock();\r
1323 \r
1324 \r
1325             var targetUri = GetTargetUri(account).Combine(container).Combine(relativeUrl);\r
1326             var message = new HttpRequestMessage(HttpMethod.Get, targetUri);\r
1327             //Don't add a range if start=0, end=null (empty files)\r
1328             if (start!=0 || end!=null)\r
1329                 message.Headers.Range=new RangeHeaderValue(start,end);\r
1330 \r
1331             //Don't use a timeout because putting the hashmap may be a long process\r
1332 \r
1333             IProgress<DownloadArgs> progress = new Progress<DownloadArgs>(args =>\r
1334                 {\r
1335                     Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
1336                                     targetUri.Segments.Last(), args.ProgressPercentage,\r
1337                                     args.BytesReceived,\r
1338                                     args.TotalBytesToReceive);\r
1339 \r
1340                     if (DownloadProgressChanged!=null)\r
1341                         DownloadProgressChanged(this,  args);\r
1342                 });\r
1343 \r
1344 \r
1345             using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false,HttpCompletionOption.ResponseHeadersRead,\r
1346                                                           cancellationToken).ConfigureAwait(false))\r
1347             using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))\r
1348             {\r
1349                 long totalSize = response.Content.Headers.ContentLength ?? 0;\r
1350                     byte[] buffer,streambuf;\r
1351                     lock (_bufferManager)\r
1352                     {\r
1353                         buffer = _bufferManager.TakeBuffer(65536);\r
1354                         streambuf = _bufferManager.TakeBuffer((int)totalSize);\r
1355                     }\r
1356 \r
1357                 using (var targetStream = new MemoryStream(streambuf))\r
1358                 {\r
1359 \r
1360                     long total = 0;\r
1361                     try\r
1362                     {\r
1363 \r
1364                         int read;\r
1365                         while ((read = await stream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) != 0)\r
1366                         {\r
1367                             total += read;\r
1368                             progress.Report(new DownloadArgs(total, totalSize));\r
1369                             await targetStream.WriteAsync(buffer, 0, read).ConfigureAwait(false);\r
1370                         }\r
1371                     }\r
1372                     finally\r
1373                     {\r
1374                         lock (_bufferManager)\r
1375                         {\r
1376                             _bufferManager.ReturnBuffer(buffer);\r
1377                             _bufferManager.ReturnBuffer(streambuf);\r
1378                         }\r
1379                     }\r
1380                     var result = targetStream.ToArray();\r
1381                     return result;\r
1382                 }\r
1383             }\r
1384 \r
1385         }\r
1386 \r
1387         public event EventHandler<UploadArgs> UploadProgressChanged;\r
1388         public event EventHandler<DownloadArgs> DownloadProgressChanged;\r
1389         \r
1390 \r
1391         public async Task PostBlock(string account, Uri container, byte[] block, int offset, int count,string blockHash,CancellationToken token)\r
1392         {\r
1393             if (container == null)\r
1394                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1395             if (container.IsAbsoluteUri)\r
1396                 throw new ArgumentException("The container must be relative","container");\r
1397             if (block == null)\r
1398                 throw new ArgumentNullException("block");\r
1399             if (offset < 0 || offset >= block.Length)\r
1400                 throw new ArgumentOutOfRangeException("offset");\r
1401             if (count < 0 || count > block.Length)\r
1402                 throw new ArgumentOutOfRangeException("count");\r
1403             if (String.IsNullOrWhiteSpace(Token))\r
1404                 throw new InvalidOperationException("Invalid Token");\r
1405             if (StorageUrl == null)\r
1406                 throw new InvalidOperationException("Invalid Storage Url");                        \r
1407             Contract.EndContractBlock();\r
1408 \r
1409 \r
1410             try\r
1411             {\r
1412                 var containerUri = GetTargetUri(account).Combine(container);\r
1413                 var targetUri = new Uri(String.Format("{0}?update", containerUri));\r
1414 \r
1415 \r
1416                 //Don't use a timeout because putting the hashmap may be a long process\r
1417 \r
1418 \r
1419                 Log.InfoFormat("[BLOCK POST] START");\r
1420 \r
1421 \r
1422                 var progress = new Progress<UploadArgs>(args =>\r
1423                 {\r
1424                     Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",\r
1425                         args.ProgressPercentage,\r
1426                         args.BytesSent,\r
1427                         args.TotalBytesToSend);\r
1428                     if (UploadProgressChanged != null)\r
1429                         UploadProgressChanged(this,args);\r
1430                 });\r
1431 \r
1432                 var message = new HttpRequestMessage(HttpMethod.Post, targetUri)\r
1433                                   {\r
1434                                       Content = new ByteArrayContentWithProgress(block, offset, count,progress)\r
1435                                   };\r
1436                 message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream");\r
1437 \r
1438                 //Send the block\r
1439                 using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3,false,HttpCompletionOption.ResponseContentRead,token).ConfigureAwait(false))\r
1440                 {                    \r
1441                     Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");\r
1442                     response.EnsureSuccessStatusCode();\r
1443                     var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false);\r
1444                     var cleanHash = responseHash.TrimEnd();\r
1445                     Debug.Assert(blockHash==cleanHash);\r
1446                     if (!blockHash.Equals(cleanHash, StringComparison.OrdinalIgnoreCase))\r
1447                         Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]", account, container, blockHash, cleanHash);\r
1448                 }\r
1449                 Log.InfoFormat("[BLOCK POST] END");               \r
1450             }\r
1451             catch (TaskCanceledException )\r
1452             {\r
1453                 Log.Info("Aborting block");\r
1454                 throw;\r
1455             }\r
1456             catch (Exception exc)\r
1457             {\r
1458                 Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);\r
1459                 throw;\r
1460             }\r
1461         }\r
1462 \r
1463         public async Task PostBlock(string account, Uri container, string filePath, long offset, int count, string blockHash, CancellationToken token)\r
1464         {\r
1465             if (container == null)\r
1466                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1467             if (container.IsAbsoluteUri)\r
1468                 throw new ArgumentException("The container must be relative", "container");\r
1469             if (String.IsNullOrWhiteSpace(filePath))\r
1470                 throw new ArgumentNullException("filePath");\r
1471             if (!File.Exists(filePath))\r
1472                 throw new FileNotFoundException("Missing file","filePath");\r
1473             if (String.IsNullOrWhiteSpace(Token))\r
1474                 throw new InvalidOperationException("Invalid Token");\r
1475             if (StorageUrl == null)\r
1476                 throw new InvalidOperationException("Invalid Storage Url");\r
1477             Contract.EndContractBlock();\r
1478 \r
1479 \r
1480             try\r
1481             {\r
1482                 var containerUri = GetTargetUri(account).Combine(container);\r
1483                 var targetUri = new Uri(String.Format("{0}?update", containerUri));\r
1484 \r
1485 \r
1486                 //Don't use a timeout because putting the hashmap may be a long process\r
1487 \r
1488 \r
1489                 Log.InfoFormat("[BLOCK POST] START");\r
1490 \r
1491 \r
1492                 var progress = new Progress<UploadArgs>(args =>\r
1493                 {\r
1494                     Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2} at {3:###,} Kbps ",\r
1495                         args.ProgressPercentage,\r
1496                         args.BytesSent,\r
1497                         args.TotalBytesToSend,args.Speed);\r
1498                     if (UploadProgressChanged != null)\r
1499                         UploadProgressChanged(this, args);\r
1500                 });\r
1501 \r
1502                 var message = new HttpRequestMessage(HttpMethod.Post, targetUri)\r
1503                 {\r
1504                     Content = new FileBlockContent(filePath, offset, count, progress)\r
1505                 };\r
1506                 message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream");\r
1507 \r
1508                 //Send the block\r
1509                 using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false, HttpCompletionOption.ResponseContentRead, token).ConfigureAwait(false))\r
1510                 {\r
1511                     Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");\r
1512                     response.EnsureSuccessStatusCode();\r
1513                     var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false);\r
1514                     var cleanHash = responseHash.TrimEnd();\r
1515                     Debug.Assert(blockHash == cleanHash);\r
1516                     if (!blockHash.Equals(cleanHash, StringComparison.OrdinalIgnoreCase))\r
1517                         Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]", account, container, blockHash, cleanHash);\r
1518                 }\r
1519                 Log.InfoFormat("[BLOCK POST] END");\r
1520             }\r
1521             catch (TaskCanceledException)\r
1522             {\r
1523                 Log.Info("Aborting block");\r
1524                 throw;\r
1525             }                \r
1526             catch (Exception exc)\r
1527             {\r
1528                 Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);\r
1529                 throw;\r
1530             }\r
1531         }\r
1532 \r
1533 \r
1534         public async Task<TreeHash> GetHashMap(string account, Uri container, Uri objectName)\r
1535         {\r
1536             if (container == null)\r
1537                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1538             if (container.IsAbsoluteUri)\r
1539                 throw new ArgumentException("The container must be relative","container");\r
1540             if (objectName == null)\r
1541                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1542             if (objectName.IsAbsoluteUri)\r
1543                 throw new ArgumentException("The objectName must be relative","objectName");\r
1544             if (String.IsNullOrWhiteSpace(Token))\r
1545                 throw new InvalidOperationException("Invalid Token");\r
1546             if (StorageUrl == null)\r
1547                 throw new InvalidOperationException("Invalid Storage Url");\r
1548             Contract.EndContractBlock();\r
1549 \r
1550             try\r
1551             {\r
1552 \r
1553                 var objectUri = GetTargetUri(account).Combine(container).Combine(objectName);\r
1554                 var targetUri = new Uri(String.Format("{0}?format=json&hashmap", objectUri));\r
1555 \r
1556                 //Start downloading the object asynchronously\r
1557                 var json = await GetStringAsync(targetUri, "").ConfigureAwait(false);\r
1558                 var treeHash = TreeHash.Parse(json);\r
1559                 Log.InfoFormat("[GET HASH] END {0}", objectName);\r
1560                 return treeHash;\r
1561 \r
1562             }\r
1563             catch (Exception exc)\r
1564             {\r
1565                 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);\r
1566                 throw;\r
1567             }\r
1568 \r
1569         }\r
1570 \r
1571 \r
1572         /// <summary>\r
1573         /// \r
1574         /// </summary>\r
1575         /// <param name="account"></param>\r
1576         /// <param name="container"></param>\r
1577         /// <param name="objectName"></param>\r
1578         /// <param name="fileName"></param>\r
1579         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>\r
1580         /// <param name="contentType"> </param>\r
1581         /// <remarks>>This method should have no timeout or a very long one</remarks>\r
1582         public async Task PutObject(string account, Uri container, Uri objectName, string fileName, string hash = Signature.MERKLE_EMPTY, string contentType = "application/octet-stream")\r
1583         {\r
1584             if (container == null)\r
1585                 throw new ArgumentNullException("container", "The container property can't be empty");\r
1586             if (container.IsAbsoluteUri)\r
1587                 throw new ArgumentException("The container must be relative","container");\r
1588             if (objectName == null)\r
1589                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1590             if (objectName.IsAbsoluteUri)\r
1591                 throw new ArgumentException("The objectName must be relative","objectName");\r
1592             if (String.IsNullOrWhiteSpace(fileName))\r
1593                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");\r
1594             try\r
1595             {\r
1596 \r
1597                 using (var client = new RestClient(_baseClient) { Timeout = 0 })\r
1598                 {\r
1599                     if (!String.IsNullOrWhiteSpace(account))\r
1600                         client.BaseAddress = GetAccountUrl(account);\r
1601 \r
1602                     var builder = client.GetAddressBuilder(container, objectName);\r
1603                     var uri = builder.Uri;\r
1604 \r
1605                     string etag = hash ;\r
1606 \r
1607                     client.Headers.Add("Content-Type", contentType);\r
1608                     if (contentType!=ObjectInfo.CONTENT_TYPE_DIRECTORY)\r
1609                         client.Headers.Add("ETag", etag);\r
1610 \r
1611 \r
1612                     Log.InfoFormat("[PUT] START {0}", objectName);\r
1613                     client.UploadProgressChanged += (sender, args) =>\r
1614                                                         {\r
1615                                                             using (ThreadContext.Stacks["PUT"].Push("Progress"))\r
1616                                                             {\r
1617                                                                 Log.InfoFormat("{0} {1}% {2} of {3}", fileName,\r
1618                                                                                args.ProgressPercentage,\r
1619                                                                                args.BytesSent, args.TotalBytesToSend);\r
1620                                                             }\r
1621                                                         };\r
1622 \r
1623                     client.UploadFileCompleted += (sender, args) =>\r
1624                                                       {\r
1625                                                           using (ThreadContext.Stacks["PUT"].Push("Progress"))\r
1626                                                           {\r
1627                                                               Log.InfoFormat("Completed {0}", fileName);\r
1628                                                           }\r
1629                                                       }; \r
1630                     \r
1631                     if (contentType==ObjectInfo.CONTENT_TYPE_DIRECTORY)\r
1632                         await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false);\r
1633                     else\r
1634                         await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false);\r
1635                 }\r
1636 \r
1637                 Log.InfoFormat("[PUT] END {0}", objectName);\r
1638             }\r
1639             catch (Exception exc)\r
1640             {\r
1641                 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);\r
1642                 throw;\r
1643             }                \r
1644 \r
1645         }\r
1646         \r
1647         public async Task MoveObject(string account, Uri sourceContainer, Uri oldObjectName, Uri targetContainer, Uri newObjectName)\r
1648         {\r
1649             if (sourceContainer == null)\r
1650                 throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");\r
1651             if (sourceContainer.IsAbsoluteUri)\r
1652                 throw new ArgumentException("The sourceContainer must be relative","sourceContainer");\r
1653             if (oldObjectName == null)\r
1654                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");\r
1655             if (oldObjectName.IsAbsoluteUri)\r
1656                 throw new ArgumentException("The oldObjectName must be relative","oldObjectName");\r
1657             if (targetContainer == null)\r
1658                 throw new ArgumentNullException("targetContainer", "The targetContainer property can't be empty");\r
1659             if (targetContainer.IsAbsoluteUri)\r
1660                 throw new ArgumentException("The targetContainer must be relative","targetContainer");\r
1661             if (newObjectName == null)\r
1662                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");\r
1663             if (newObjectName.IsAbsoluteUri)\r
1664                 throw new ArgumentException("The newObjectName must be relative","newObjectName");\r
1665             Contract.EndContractBlock();\r
1666 \r
1667             var baseUri = GetTargetUri(account);\r
1668             var targetUri = baseUri.Combine(targetContainer).Combine(newObjectName);\r
1669             var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, oldObjectName),UriKind.Relative);\r
1670 \r
1671             var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
1672             message.Headers.Add("X-Move-From", sourceUri.ToString());\r
1673             using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))\r
1674             {\r
1675                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
1676                 if (!expectedCodes.Contains(response.StatusCode))\r
1677                     throw CreateWebException("MoveObject", response.StatusCode);\r
1678             }\r
1679         }\r
1680 \r
1681         public async Task DeleteObject(string account, Uri sourceContainer, Uri objectName, bool isDirectory)\r
1682         {\r
1683             if (sourceContainer == null)\r
1684                 throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");\r
1685             if (sourceContainer.IsAbsoluteUri)\r
1686                 throw new ArgumentException("The sourceContainer must be relative","sourceContainer");\r
1687             if (objectName == null)\r
1688                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
1689             if (objectName.IsAbsoluteUri)\r
1690                 throw new ArgumentException("The objectName must be relative","objectName");\r
1691             Contract.EndContractBlock();\r
1692 \r
1693 \r
1694 \r
1695             var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, objectName),UriKind.Relative);\r
1696 \r
1697             \r
1698             if (objectName.OriginalString.EndsWith(".ignore"))\r
1699                 using(var response = await _baseHttpClient.DeleteAsync(sourceUri)){}\r
1700             else\r
1701             {\r
1702                 var relativeUri = new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName),\r
1703                                                 UriKind.Relative);\r
1704 \r
1705 /*\r
1706                 var relativeUri = isDirectory\r
1707                                       ? new Uri(\r
1708                                             String.Format("{0}/{1}?delimiter=/", FolderConstants.TrashContainer,\r
1709                                                           objectName), UriKind.Relative)\r
1710                                       : new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName),\r
1711                                                 UriKind.Relative);\r
1712 \r
1713 */\r
1714                 var targetUri = GetTargetUri(account).Combine(relativeUri);\r
1715 \r
1716 \r
1717                 var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
1718                 message.Headers.Add("X-Move-From", sourceUri.ToString());\r
1719 \r
1720                 Log.InfoFormat("[TRASH] [{0}] to [{1}]", sourceUri, targetUri);\r
1721                 using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3))\r
1722                 {\r
1723                     var expectedCodes = new[]\r
1724                                             {\r
1725                                                 HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,\r
1726                                                 HttpStatusCode.NotFound\r
1727                                             };\r
1728                     if (!expectedCodes.Contains(response.StatusCode))\r
1729                         throw CreateWebException("DeleteObject", response.StatusCode);\r
1730                 }\r
1731             }\r
1732 /*\r
1733             \r
1734 \r
1735             var targetUrl = FolderConstants.TrashContainer + "/" + objectName;\r
1736 /*\r
1737             if (isDirectory)\r
1738                 targetUrl = targetUrl + "?delimiter=/";\r
1739 #1#\r
1740 \r
1741             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);\r
1742 \r
1743             using (var client = new RestClient(_baseClient))\r
1744             {\r
1745                 if (!String.IsNullOrWhiteSpace(account))\r
1746                     client.BaseAddress = GetAccountUrl(account);\r
1747 \r
1748                 client.Headers.Add("X-Move-From", sourceUrl);\r
1749                 client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);\r
1750                 Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);\r
1751                 client.PutWithRetry(new Uri(targetUrl,UriKind.Relative), 3);\r
1752 \r
1753                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};\r
1754                 if (!expectedCodes.Contains(client.StatusCode))\r
1755                     throw CreateWebException("DeleteObject", client.StatusCode);\r
1756             }\r
1757 */\r
1758         }\r
1759 \r
1760       \r
1761         private static WebException CreateWebException(string operation, HttpStatusCode statusCode)\r
1762         {\r
1763             return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));\r
1764         }\r
1765 \r
1766 \r
1767         public async Task<bool> CanUpload(string account, ObjectInfo cloudFile)\r
1768         {\r
1769             Contract.Requires(!String.IsNullOrWhiteSpace(account));\r
1770             Contract.Requires(cloudFile!=null);\r
1771 \r
1772                 var parts = cloudFile.Name.ToString().Split('/');\r
1773                 var folder = String.Join("/", parts,0,parts.Length-1);\r
1774 \r
1775                 var fileName = String.Format("{0}/{1}.pithos.ignore", folder, Guid.NewGuid());\r
1776                 var fileUri=fileName.ToEscapedUri();                                            \r
1777 \r
1778                 try\r
1779                 {\r
1780                     var relativeUri = cloudFile.Container.Combine(fileUri);\r
1781                     var targetUri = GetTargetUri(account).Combine(relativeUri);\r
1782                     var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
1783                     message.Content.Headers.ContentType =new MediaTypeHeaderValue("application/octet-stream");\r
1784                     var response=await _baseHttpClient.SendAsyncWithRetries(message, 3);                    \r
1785                     var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
1786                     var result=(expectedCodes.Contains(response.StatusCode));\r
1787                     await DeleteObject(account, cloudFile.Container, fileUri, cloudFile.IsDirectory);\r
1788                     return result;\r
1789                 }\r
1790                 catch\r
1791                 {\r
1792                     return false;\r
1793                 }\r
1794             \r
1795         }\r
1796 \r
1797         ~CloudFilesClient()\r
1798         {\r
1799             Dispose(false);\r
1800         }\r
1801 \r
1802         public void Dispose()\r
1803         {\r
1804             Dispose(true);\r
1805             GC.SuppressFinalize(this);\r
1806         }\r
1807 \r
1808         protected virtual void Dispose(bool disposing)\r
1809         {\r
1810             if (disposing)\r
1811             {\r
1812                 if (_httpClientHandler!=null)\r
1813                     _httpClientHandler.Dispose();\r
1814                 if (_baseClient!=null)\r
1815                     _baseClient.Dispose();\r
1816                 if(_baseHttpClient!=null)\r
1817                     _baseHttpClient.Dispose();\r
1818                 if (_baseHttpClientNoTimeout!=null)\r
1819                     _baseHttpClientNoTimeout.Dispose();\r
1820             }\r
1821             _httpClientHandler = null;\r
1822             _baseClient = null;\r
1823             _baseHttpClient = null;\r
1824             _baseHttpClientNoTimeout = null;\r
1825         }\r
1826 \r
1827         public async Task<string> ResolveName(Guid accountToken)\r
1828         {\r
1829             string format = string.Format("{{\"uuids\":[\"{0}\"]}}", accountToken);\r
1830             var content = new StringContent(format,Encoding.UTF8);\r
1831             //content.Headers.ContentType=new MediaTypeHeaderValue("text/html; charset=utf-8");\r
1832             string catalogEntry;\r
1833             var catalogUrl = new Uri(_baseHttpClient.BaseAddress.Scheme + "://" +_baseHttpClient.BaseAddress.Host,UriKind.Absolute).Combine("user_catalogs");\r
1834             using (var response = await _baseHttpClient.PostAsync(catalogUrl, content).ConfigureAwait(false))\r
1835             {\r
1836                 catalogEntry=await response.Content.ReadAsStringAsync().ConfigureAwait(false);\r
1837             }\r
1838 \r
1839             var entry = (JContainer)JsonConvert.DeserializeObject(catalogEntry);\r
1840             string key = accountToken.ToString();\r
1841             return (string)entry["uuid_catalog"][key];\r
1842 \r
1843         }\r
1844 \r
1845         public async Task<Guid> ResolveToken(string displayName)\r
1846         {\r
1847             string format = string.Format("{{\"displaynames\":[\"{0}\"]}}", displayName);\r
1848             var content = new StringContent(format,Encoding.UTF8);\r
1849             //content.Headers.ContentType=new MediaTypeHeaderValue("text/html; charset=utf-8");\r
1850             string catalogEntry;\r
1851             var catalogUrl = new Uri(_baseHttpClient.BaseAddress.Scheme + "://" +_baseHttpClient.BaseAddress.Host,UriKind.Absolute).Combine("user_catalogs");\r
1852             using (var response = await _baseHttpClient.PostAsync(catalogUrl, content).ConfigureAwait(false))\r
1853             {\r
1854                 catalogEntry=await response.Content.ReadAsStringAsync().ConfigureAwait(false);\r
1855             }\r
1856 \r
1857             var entry = (JContainer)JsonConvert.DeserializeObject(catalogEntry);\r
1858             return new Guid((string)entry["displayname_catalog"][displayName]);\r
1859 \r
1860         }\r
1861     }\r
1862 \r
1863     public class ShareAccountInfo\r
1864     {\r
1865         public DateTime? last_modified { get; set; }\r
1866         public string name { get; set; }\r
1867     }\r
1868 }\r