code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/RestClient.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="RestClient.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System.Collections.Specialized;
  43 using System.Diagnostics;
  44 using System.Diagnostics.Contracts;
  45 using System.IO;
  46 using System.Net;
  47 using System.Reflection;
  48 using System.Runtime.Serialization;
  49 using System.Threading.Tasks;
  50 using log4net;
  51
  52
  53 namespace Pithos.Network
  54 {
  55     using System;
  56     using System.Collections.Generic;
  57     using System.Linq;
  58     using System.Text;
  59
  60     /// <summary>
  61     /// TODO: Update summary.
  62     /// </summary>
  63     public class RestClient:WebClient
  64     {
  65         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  66
  67         public int Timeout { get; set; }
  68
  69         public bool TimedOut { get; set; }
  70
  71         public HttpStatusCode StatusCode { get; private set; }
  72
  73         public string StatusDescription { get; set; }
  74
  75         public long? RangeFrom { get; set; }
  76         public long? RangeTo { get; set; }
  77
  78         public int Retries { get; set; }
  79
  80         private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
  81         public Dictionary<string, string> Parameters
  82         {
  83             get
  84             {
  85                 Contract.Ensures(_parameters!=null);
  86                 return _parameters;
  87             }
  88         }
  89
  90
  91         [ContractInvariantMethod]
  92         private void Invariants()
  93         {
  94             Contract.Invariant(Headers!=null);
  95         }
  96
  97         public RestClient():base()
  98         {
  99             //The maximum error response must be large because missing server hashes are return as a Conflivt (409) error response
 100             //Any value above 2^21-1 will result in an empty response.
 101             //-1 essentially ignores the maximum length
 102             HttpWebRequest.DefaultMaximumErrorResponseLength = -1;
 103         }
 104
 105
 106         public RestClient(RestClient other)
 107             : base()
 108         {
 109             if (other==null)
 110                 throw new ArgumentNullException("other");
 111             Contract.EndContractBlock();
 112
 113             //The maximum error response must be large because missing server hashes are return as a Conflivt (409) error response
 114             //Any value above 2^21-1 will result in an empty response.
 115             //-1 essentially ignores the maximum length
 116             HttpWebRequest.DefaultMaximumErrorResponseLength = -1;
 117
 118             CopyHeaders(other);
 119             Timeout = other.Timeout;
 120             Retries = other.Retries;
 121             BaseAddress = other.BaseAddress;
 122
 123             foreach (var parameter in other.Parameters)
 124             {
 125                 Parameters.Add(parameter.Key,parameter.Value);
 126             }
 127
 128             this.Proxy = other.Proxy;
 129         }
 130
 131
 132         protected override WebRequest GetWebRequest(Uri address)
 133         {
 134             TimedOut = false;
 135             var webRequest = base.GetWebRequest(address);
 136             var request = (HttpWebRequest)webRequest;
 137             request.ServicePoint.ConnectionLimit = 50;
 138             if (IfModifiedSince.HasValue)
 139                 request.IfModifiedSince = IfModifiedSince.Value;
 140             request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
 141             if(Timeout>0)
 142                 request.Timeout = Timeout;
 143
 144             if (RangeFrom.HasValue)
 145             {
 146                 if (RangeTo.HasValue)
 147                     request.AddRange(RangeFrom.Value, RangeTo.Value);
 148                 else
 149                     request.AddRange(RangeFrom.Value);
 150             }
 151             return request;
 152         }
 153
 154         public DateTime? IfModifiedSince { get; set; }
 155
 156         //Asynchronous version
 157         protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
 158         {
 159             Log.InfoFormat("[{0}] {1}", request.Method, request.RequestUri);
 160             HttpWebResponse response = null;
 161
 162             try
 163             {
 164                 response = (HttpWebResponse)base.GetWebResponse(request, result);
 165             }
 166             catch (WebException exc)
 167             {
 168                 if (!TryGetResponse(exc, request,out response))
 169                     throw;
 170             }
 171
 172             StatusCode = response.StatusCode;
 173             LastModified = response.LastModified;
 174             StatusDescription = response.StatusDescription;
 175             return response;
 176
 177         }
 178
 179
 180         //Synchronous version
 181         protected override WebResponse GetWebResponse(WebRequest request)
 182         {
 183             HttpWebResponse response = null;
 184             try
 185             {
 186                 Log.InfoFormat("[{0}] {1}",request.Method,request.RequestUri);
 187                 response = (HttpWebResponse)base.GetWebResponse(request);
 188             }
 189             catch (WebException exc)
 190             {
 191                 if (!TryGetResponse(exc, request,out response))
 192                     throw;
 193             }
 194
 195             StatusCode = response.StatusCode;
 196             LastModified = response.LastModified;
 197             StatusDescription = response.StatusDescription;
 198             return response;
 199         }
 200
 201         private bool TryGetResponse(WebException exc, WebRequest request,out HttpWebResponse response)
 202         {
 203             response = null;
 204             //Fail on empty response
 205             if (exc.Response == null)
 206             {
 207                 Log.WarnFormat("[{0}] {1} {2}", request.Method, exc.Status, request.RequestUri);
 208                 return false;
 209             }
 210
 211             response = (exc.Response as HttpWebResponse);
 212             var statusCode = (int)response.StatusCode;
 213             //Succeed on allowed status codes
 214             if (AllowedStatusCodes.Contains(response.StatusCode))
 215             {
 216                 if (Log.IsDebugEnabled)
 217                     Log.DebugFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 218                 return true;
 219             }
 220
 221             Log.WarnFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 222
 223             //Does the response have any content to log?
 224             if (exc.Response.ContentLength > 0)
 225             {
 226                 var content = LogContent(exc.Response);
 227                 Log.ErrorFormat(content);
 228             }
 229             return false;
 230         }
 231
 232         private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
 233
 234         public List<HttpStatusCode> AllowedStatusCodes
 235         {
 236             get
 237             {
 238                 return _allowedStatusCodes;
 239             }
 240         }
 241
 242         public DateTime LastModified { get; private set; }
 243
 244         private static string LogContent(WebResponse webResponse)
 245         {
 246             if (webResponse == null)
 247                 throw new ArgumentNullException("webResponse");
 248             Contract.EndContractBlock();
 249
 250             //The response stream must be copied to avoid affecting other code by disposing of the
 251             //original response stream.
 252             var stream = webResponse.GetResponseStream();
 253             using(var memStream=new MemoryStream())
 254             using (var reader = new StreamReader(memStream))
 255             {
 256                 stream.CopyTo(memStream);
 257                 string content = reader.ReadToEnd();
 258
 259                 stream.Seek(0,SeekOrigin.Begin);
 260                 return content;
 261             }
 262         }
 263
 264         public string DownloadStringWithRetry(string address,int retries=0)
 265         {
 266
 267             if (address == null)
 268                 throw new ArgumentNullException("address");
 269
 270             var actualAddress = GetActualAddress(address);
 271
 272             TraceStart("GET",actualAddress);
 273
 274             var actualRetries = (retries == 0) ? Retries : retries;
 275
 276             var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
 277
 278             var task = Retry(() =>
 279             {
 280                 var content = base.DownloadString(uriString);
 281
 282                 if (StatusCode == HttpStatusCode.NoContent)
 283                     return String.Empty;
 284                 return content;
 285
 286             }, actualRetries);
 287
 288             try
 289             {
 290                     var result = task.Result;
 291                 return result;
 292
 293             }
 294             catch (AggregateException exc)
 295             {
 296                 //If the task fails, propagate the original exception
 297                 if (exc.InnerException!=null)
 298                     throw exc.InnerException;
 299                 throw;
 300             }
 301         }
 302
 303         public void Head(string address,int retries=0)
 304         {
 305             AllowedStatusCodes.Add(HttpStatusCode.NotFound);
 306             RetryWithoutContent(address, retries, "HEAD");
 307         }
 308
 309         public void PutWithRetry(string address, int retries = 0)
 310         {
 311             RetryWithoutContent(address, retries, "PUT");
 312         }
 313
 314         public void DeleteWithRetry(string address,int retries=0)
 315         {
 316             RetryWithoutContent(address, retries, "DELETE");
 317         }
 318
 319         public string GetHeaderValue(string headerName,bool optional=false)
 320         {
 321             if (this.ResponseHeaders==null)
 322                 throw new InvalidOperationException("ResponseHeaders are null");
 323             Contract.EndContractBlock();
 324
 325             var values=this.ResponseHeaders.GetValues(headerName);
 326             if (values != null)
 327                 return values[0];
 328
 329             if (optional)
 330                 return null;
 331             //A required header was not found
 332             throw new WebException(String.Format("The {0}  header is missing", headerName));
 333         }
 334
 335         public void SetNonEmptyHeaderValue(string headerName, string value)
 336         {
 337             if (String.IsNullOrWhiteSpace(value))
 338                 return;
 339             Headers.Add(headerName,value);
 340         }
 341
 342         private void RetryWithoutContent(string address, int retries, string method)
 343         {
 344             if (address == null)
 345                 throw new ArgumentNullException("address");
 346
 347             var actualAddress = GetActualAddress(address);
 348             var actualRetries = (retries == 0) ? Retries : retries;
 349
 350             var task = Retry(() =>
 351             {
 352                 var uriString = String.Join("/",BaseAddress ,actualAddress);
 353                 var uri = new Uri(uriString);
 354                 var request =  GetWebRequest(uri);
 355                 request.Method = method;
 356                 if (ResponseHeaders!=null)
 357                     ResponseHeaders.Clear();
 358
 359                 TraceStart(method, uriString);
 360                 if (method == "PUT")
 361                     request.ContentLength = 0;
 362
 363                 //Have to use try/finally instead of using here, because WebClient needs a valid WebResponse object
 364                 //in order to return response headers
 365                 var response = (HttpWebResponse)GetWebResponse(request);
 366                 try
 367                 {
 368                     LastModified = response.LastModified;
 369                     StatusCode = response.StatusCode;
 370                     StatusDescription = response.StatusDescription;
 371                 }
 372                 finally
 373                 {
 374                     response.Close();
 375                 }
 376
 377
 378                 return 0;
 379             }, actualRetries);
 380
 381             try
 382             {
 383                 task.Wait();
 384             }
 385             catch (AggregateException ex)
 386             {
 387                 var exc = ex.InnerException;
 388                 if (exc is RetryException)
 389                 {
 390                     Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
 391                 }
 392                 else
 393                 {
 394                     Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
 395                 }
 396                 throw exc;
 397
 398             }
 399             catch(Exception ex)
 400             {
 401                 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
 402                 throw;
 403             }
 404         }
 405
 406         private static void TraceStart(string method, string actualAddress)
 407         {
 408             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
 409         }
 410
 411         private string GetActualAddress(string address)
 412         {
 413             if (Parameters.Count == 0)
 414                 return address;
 415             var addressBuilder=new StringBuilder(address);
 416
 417             bool isFirst = true;
 418             foreach (var parameter in Parameters)
 419             {
 420                 if(isFirst)
 421                     addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
 422                 else
 423                     addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
 424                 isFirst = false;
 425             }
 426             return addressBuilder.ToString();
 427         }
 428
 429         public string DownloadStringWithRetry(Uri address,int retries=0)
 430         {
 431             if (address == null)
 432                 throw new ArgumentNullException("address");
 433
 434             var actualRetries = (retries == 0) ? Retries : retries;
 435             var task = Retry(() =>
 436             {
 437                 var content = base.DownloadString(address);
 438
 439                 if (StatusCode == HttpStatusCode.NoContent)
 440                     return String.Empty;
 441                 return content;
 442
 443             }, actualRetries);
 444
 445             var result = task.Result;
 446             return result;
 447         }
 448
 449
 450         /// <summary>
 451         /// Copies headers from another RestClient
 452         /// </summary>
 453         /// <param name="source">The RestClient from which the headers are copied</param>
 454         public void CopyHeaders(RestClient source)
 455         {
 456             if (source == null)
 457                 throw new ArgumentNullException("source", "source can't be null");
 458             Contract.EndContractBlock();
 459             //The Headers getter initializes the property, it is never null
 460             Contract.Assume(Headers!=null);
 461
 462             CopyHeaders(source.Headers,Headers);
 463         }
 464
 465         /// <summary>
 466         /// Copies headers from one header collection to another
 467         /// </summary>
 468         /// <param name="source">The source collection from which the headers are copied</param>
 469         /// <param name="target">The target collection to which the headers are copied</param>
 470         public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
 471         {
 472             if (source == null)
 473                 throw new ArgumentNullException("source", "source can't be null");
 474             if (target == null)
 475                 throw new ArgumentNullException("target", "target can't be null");
 476             Contract.EndContractBlock();
 477
 478             for (int i = 0; i < source.Count; i++)
 479             {
 480                 target.Add(source.GetKey(i), source[i]);
 481             }
 482         }
 483
 484         public void AssertStatusOK(string message)
 485         {
 486             if (StatusCode >= HttpStatusCode.BadRequest)
 487                 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
 488         }
 489
 490
 491         private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
 492         {
 493             if (original==null)
 494                 throw new ArgumentNullException("original");
 495             Contract.EndContractBlock();
 496
 497             if (tcs == null)
 498                 tcs = new TaskCompletionSource<T>();
 499             Task.Factory.StartNew(original).ContinueWith(_original =>
 500                 {
 501                     if (!_original.IsFaulted)
 502                         tcs.SetFromTask(_original);
 503                     else
 504                     {
 505                         var e = _original.Exception.InnerException;
 506                         var we = (e as WebException);
 507                         if (we==null)
 508                             tcs.SetException(e);
 509                         else
 510                         {
 511                             var statusCode = GetStatusCode(we);
 512
 513                             //Return null for 404
 514                             if (statusCode == HttpStatusCode.NotFound)
 515                                 tcs.SetResult(default(T));
 516                             //Retry for timeouts and service unavailable
 517                             else if (we.Status == WebExceptionStatus.Timeout ||
 518                                 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
 519                             {
 520                                 TimedOut = true;
 521                                 if (retryCount == 0)
 522                                 {
 523                                     Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
 524                                     tcs.SetException(new RetryException("Timed out too many times.", e));
 525                                 }
 526                                 else
 527                                 {
 528                                     Log.ErrorFormat(
 529                                         "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
 530                                         retryCount, e);
 531                                     Retry(original, retryCount - 1, tcs);
 532                                 }
 533                             }
 534                             else
 535                                 tcs.SetException(e);
 536                         }
 537                     };
 538                 });
 539             return tcs.Task;
 540         }
 541
 542         private HttpStatusCode GetStatusCode(WebException we)
 543         {
 544             if (we==null)
 545                 throw new ArgumentNullException("we");
 546             var statusCode = HttpStatusCode.RequestTimeout;
 547             if (we.Response != null)
 548             {
 549                 statusCode = ((HttpWebResponse) we.Response).StatusCode;
 550                 this.StatusCode = statusCode;
 551             }
 552             return statusCode;
 553         }
 554
 555         public UriBuilder GetAddressBuilder(string container, string objectName)
 556         {
 557             var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
 558             return builder;
 559         }
 560
 561         public Dictionary<string, string> GetMeta(string metaPrefix)
 562         {
 563             if (String.IsNullOrWhiteSpace(metaPrefix))
 564                 throw new ArgumentNullException("metaPrefix");
 565             Contract.EndContractBlock();
 566
 567             var keys = ResponseHeaders.AllKeys.AsQueryable();
 568             var dict = (from key in keys
 569                         where key.StartsWith(metaPrefix)
 570                         let name = key.Substring(metaPrefix.Length)
 571                         select new { Name = name, Value = ResponseHeaders[key] })
 572                         .ToDictionary(t => t.Name, t => t.Value);
 573             return dict;
 574         }
 575     }
 576
 577     public class RetryException:Exception
 578     {
 579         public RetryException()
 580             :base()
 581         {
 582
 583         }
 584
 585         public RetryException(string message)
 586             :base(message)
 587         {
 588
 589         }
 590
 591         public RetryException(string message,Exception innerException)
 592             :base(message,innerException)
 593         {
 594
 595         }
 596
 597         public RetryException(SerializationInfo info,StreamingContext context)
 598             :base(info,context)
 599         {
 600
 601         }
 602     }
 603 }