code.grnet.gr Git - pithos-ms-client/blob - trunk%2FPithos.Network%2FRestClient.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="RestClient.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System.Collections.Specialized;
  43 using System.Diagnostics;
  44 using System.Diagnostics.Contracts;
  45 using System.IO;
  46 using System.Net;
  47 using System.Reflection;
  48 using System.Runtime.Serialization;
  49 using System.Threading.Tasks;
  50 using log4net;
  51
  52
  53 namespace Pithos.Network
  54 {
  55     using System;
  56     using System.Collections.Generic;
  57     using System.Linq;
  58     using System.Text;
  59
  60     /// <summary>
  61     /// TODO: Update summary.
  62     /// </summary>
  63     public class RestClient:WebClient
  64     {
  65         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  66
  67         public int Timeout { get; set; }
  68
  69         public bool TimedOut { get; set; }
  70
  71         public HttpStatusCode StatusCode { get; private set; }
  72
  73         public string StatusDescription { get; set; }
  74
  75         public long? RangeFrom { get; set; }
  76         public long? RangeTo { get; set; }
  77
  78         public int Retries { get; set; }
  79
  80         private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
  81         public Dictionary<string, string> Parameters
  82         {
  83             get
  84             {
  85                 Contract.Ensures(_parameters!=null);
  86                 return _parameters;
  87             }
  88         }
  89
  90
  91         [ContractInvariantMethod]
  92         private void Invariants()
  93         {
  94             Contract.Invariant(Headers!=null);
  95         }
  96
  97         public RestClient():base()
  98         {
  99             //The maximum error response must be large because missing server hashes are return as a Conflivt (409) error response
 100             //Any value above 2^21-1 will result in an empty response.
 101             //-1 essentially ignores the maximum length
 102             HttpWebRequest.DefaultMaximumErrorResponseLength = -1;
 103         }
 104
 105
 106         public RestClient(RestClient other)
 107             : base()
 108         {
 109             if (other==null)
 110                 throw new ArgumentNullException("other");
 111             Contract.EndContractBlock();
 112
 113             //The maximum error response must be large because missing server hashes are return as a Conflivt (409) error response
 114             //Any value above 2^21-1 will result in an empty response.
 115             //-1 essentially ignores the maximum length
 116             HttpWebRequest.DefaultMaximumErrorResponseLength = -1;
 117
 118             CopyHeaders(other);
 119             Timeout = other.Timeout;
 120             Retries = other.Retries;
 121             BaseAddress = other.BaseAddress;
 122
 123             foreach (var parameter in other.Parameters)
 124             {
 125                 Parameters.Add(parameter.Key,parameter.Value);
 126             }
 127
 128             this.Proxy = other.Proxy;
 129         }
 130
 131
 132         protected override WebRequest GetWebRequest(Uri address)
 133         {
 134             TimedOut = false;
 135             var webRequest = base.GetWebRequest(address);
 136             var request = (HttpWebRequest)webRequest;
 137             request.ServicePoint.ConnectionLimit = 50;
 138             if (IfModifiedSince.HasValue)
 139                 request.IfModifiedSince = IfModifiedSince.Value;
 140             request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
 141             if(Timeout>0)
 142                 request.Timeout = Timeout;
 143
 144             if (RangeFrom.HasValue)
 145             {
 146                 if (RangeTo.HasValue)
 147                     request.AddRange(RangeFrom.Value, RangeTo.Value);
 148                 else
 149                     request.AddRange(RangeFrom.Value);
 150             }
 151             return request;
 152         }
 153
 154         public DateTime? IfModifiedSince { get; set; }
 155
 156         //Asynchronous version
 157         protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
 158         {
 159             Log.InfoFormat("[{0}] {1}", request.Method, request.RequestUri);
 160             HttpWebResponse response = null;
 161
 162             try
 163             {
 164                 response = (HttpWebResponse)base.GetWebResponse(request, result);
 165             }
 166             catch (WebException exc)
 167             {
 168                 if (!TryGetResponse(exc, request,out response))
 169                     throw;
 170             }
 171
 172             StatusCode = response.StatusCode;
 173             LastModified = response.LastModified;
 174             StatusDescription = response.StatusDescription;
 175             return response;
 176
 177         }
 178
 179
 180         //Synchronous version
 181         protected override WebResponse GetWebResponse(WebRequest request)
 182         {
 183             HttpWebResponse response = null;
 184             try
 185             {
 186                 Log.InfoFormat("[{0}] {1}",request.Method,request.RequestUri);
 187                 response = (HttpWebResponse)base.GetWebResponse(request);
 188             }
 189             catch (WebException exc)
 190             {
 191                 if (!TryGetResponse(exc, request,out response))
 192                     throw;
 193             }
 194
 195             StatusCode = response.StatusCode;
 196             LastModified = response.LastModified;
 197             StatusDescription = response.StatusDescription;
 198             return response;
 199         }
 200
 201         private bool TryGetResponse(WebException exc, WebRequest request,out HttpWebResponse response)
 202         {
 203             response = null;
 204             //Fail on empty response
 205             if (exc.Response == null)
 206             {
 207                 Log.WarnFormat("[{0}] {1} {2}", request.Method, exc.Status, request.RequestUri);
 208                 return false;
 209             }
 210
 211             response = (exc.Response as HttpWebResponse);
 212             var statusCode = (int)response.StatusCode;
 213             //Succeed on allowed status codes
 214             if (AllowedStatusCodes.Contains(response.StatusCode))
 215             {
 216                 if (Log.IsDebugEnabled)
 217                     Log.DebugFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 218                 return true;
 219             }
 220
 221             Log.WarnFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 222
 223             //Does the response have any content to log?
 224             if (exc.Response.ContentLength > 0)
 225             {
 226                 var content = LogContent(exc.Response);
 227                 Log.ErrorFormat(content);
 228             }
 229             return false;
 230         }
 231
 232         private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
 233
 234         public List<HttpStatusCode> AllowedStatusCodes
 235         {
 236             get
 237             {
 238                 return _allowedStatusCodes;
 239             }
 240         }
 241
 242         public DateTime LastModified { get; private set; }
 243
 244         private static string LogContent(WebResponse webResponse)
 245         {
 246             if (webResponse == null)
 247                 throw new ArgumentNullException("webResponse");
 248             Contract.EndContractBlock();
 249
 250             //The response stream must be copied to avoid affecting other code by disposing of the
 251             //original response stream.
 252             var stream = webResponse.GetResponseStream();
 253             using(var memStream=new MemoryStream())
 254             using (var reader = new StreamReader(memStream))
 255             {
 256                 stream.CopyTo(memStream);
 257                 string content = reader.ReadToEnd();
 258
 259                 stream.Seek(0,SeekOrigin.Begin);
 260                 return content;
 261             }
 262         }
 263
 264         public string DownloadStringWithRetry(string address,int retries=0)
 265         {
 266
 267             if (address == null)
 268                 throw new ArgumentNullException("address");
 269
 270             var actualAddress = GetActualAddress(address);
 271
 272             TraceStart("GET",actualAddress);
 273
 274             var actualRetries = (retries == 0) ? Retries : retries;
 275
 276             var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
 277
 278             var task = Retry(() =>
 279             {
 280                 var content = base.DownloadString(uriString);
 281
 282                 if (StatusCode == HttpStatusCode.NoContent)
 283                     return String.Empty;
 284                 return content;
 285
 286             }, actualRetries);
 287
 288             try
 289             {
 290                     var result = task.Result;
 291                 return result;
 292
 293             }
 294             catch (AggregateException exc)
 295             {
 296                 //If the task fails, propagate the original exception
 297                 if (exc.InnerException!=null)
 298                     throw exc.InnerException;
 299                 throw;
 300             }
 301         }
 302
 303         public void Head(string address,int retries=0)
 304         {
 305             AllowedStatusCodes.Add(HttpStatusCode.NotFound);
 306             RetryWithoutContent(address, retries, "HEAD");
 307         }
 308
 309         public void PutWithRetry(string address, int retries = 0)
 310         {
 311             RetryWithoutContent(address, retries, "PUT");
 312         }
 313
 314         public void PostWithRetry(string address,string contentType)
 315         {
 316             RetryWithoutContent(address, 0, "POST",contentType);
 317         }
 318
 319         public void DeleteWithRetry(string address,int retries=0)
 320         {
 321             RetryWithoutContent(address, retries, "DELETE");
 322         }
 323
 324         public string GetHeaderValue(string headerName,bool optional=false)
 325         {
 326             if (this.ResponseHeaders==null)
 327                 throw new InvalidOperationException("ResponseHeaders are null");
 328             Contract.EndContractBlock();
 329
 330             var values=this.ResponseHeaders.GetValues(headerName);
 331             if (values != null)
 332                 return values[0];
 333
 334             if (optional)
 335                 return null;
 336             //A required header was not found
 337             throw new WebException(String.Format("The {0}  header is missing", headerName));
 338         }
 339
 340         public void SetNonEmptyHeaderValue(string headerName, string value)
 341         {
 342             if (String.IsNullOrWhiteSpace(value))
 343                 return;
 344             Headers.Add(headerName,value);
 345         }
 346
 347         private void RetryWithoutContent(string address, int retries, string method,string contentType=null)
 348         {
 349             if (address == null)
 350                 throw new ArgumentNullException("address");
 351
 352             var actualAddress = GetActualAddress(address);
 353             var actualRetries = (retries == 0) ? Retries : retries;
 354
 355             var task = Retry(() =>
 356             {
 357                 var uriString = String.Join("/",BaseAddress ,actualAddress);
 358                 var uri = new Uri(uriString);
 359                 var request =  GetWebRequest(uri);
 360                 if (contentType!=null)
 361                 {
 362                     request.ContentType = contentType;
 363                     request.ContentLength = 0;
 364                 }
 365                 request.Method = method;
 366                 if (ResponseHeaders!=null)
 367                     ResponseHeaders.Clear();
 368
 369                 TraceStart(method, uriString);
 370                 if (method == "PUT")
 371                     request.ContentLength = 0;
 372
 373                 //Have to use try/finally instead of using here, because WebClient needs a valid WebResponse object
 374                 //in order to return response headers
 375                 var response = (HttpWebResponse)GetWebResponse(request);
 376                 try
 377                 {
 378                     LastModified = response.LastModified;
 379                     StatusCode = response.StatusCode;
 380                     StatusDescription = response.StatusDescription;
 381                 }
 382                 finally
 383                 {
 384                     response.Close();
 385                 }
 386
 387
 388                 return 0;
 389             }, actualRetries);
 390
 391             try
 392             {
 393                 task.Wait();
 394             }
 395             catch (AggregateException ex)
 396             {
 397                 var exc = ex.InnerException;
 398                 if (exc is RetryException)
 399                 {
 400                     Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
 401                 }
 402                 else
 403                 {
 404                     Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
 405                 }
 406                 throw exc;
 407
 408             }
 409             catch(Exception ex)
 410             {
 411                 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
 412                 throw;
 413             }
 414         }
 415
 416         private static void TraceStart(string method, string actualAddress)
 417         {
 418             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
 419         }
 420
 421         private string GetActualAddress(string address)
 422         {
 423             if (Parameters.Count == 0)
 424                 return address;
 425             var addressBuilder=new StringBuilder(address);
 426
 427             bool isFirst = true;
 428             foreach (var parameter in Parameters)
 429             {
 430                 if(isFirst)
 431                     addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
 432                 else
 433                     addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
 434                 isFirst = false;
 435             }
 436             return addressBuilder.ToString();
 437         }
 438
 439         public string DownloadStringWithRetry(Uri address,int retries=0)
 440         {
 441             if (address == null)
 442                 throw new ArgumentNullException("address");
 443
 444             var actualRetries = (retries == 0) ? Retries : retries;
 445             var task = Retry(() =>
 446             {
 447                 var content = base.DownloadString(address);
 448
 449                 if (StatusCode == HttpStatusCode.NoContent)
 450                     return String.Empty;
 451                 return content;
 452
 453             }, actualRetries);
 454
 455             var result = task.Result;
 456             return result;
 457         }
 458
 459
 460         /// <summary>
 461         /// Copies headers from another RestClient
 462         /// </summary>
 463         /// <param name="source">The RestClient from which the headers are copied</param>
 464         public void CopyHeaders(RestClient source)
 465         {
 466             if (source == null)
 467                 throw new ArgumentNullException("source", "source can't be null");
 468             Contract.EndContractBlock();
 469             //The Headers getter initializes the property, it is never null
 470             Contract.Assume(Headers!=null);
 471
 472             CopyHeaders(source.Headers,Headers);
 473         }
 474
 475         /// <summary>
 476         /// Copies headers from one header collection to another
 477         /// </summary>
 478         /// <param name="source">The source collection from which the headers are copied</param>
 479         /// <param name="target">The target collection to which the headers are copied</param>
 480         public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
 481         {
 482             if (source == null)
 483                 throw new ArgumentNullException("source", "source can't be null");
 484             if (target == null)
 485                 throw new ArgumentNullException("target", "target can't be null");
 486             Contract.EndContractBlock();
 487
 488             for (int i = 0; i < source.Count; i++)
 489             {
 490                 target.Add(source.GetKey(i), source[i]);
 491             }
 492         }
 493
 494         public void AssertStatusOK(string message)
 495         {
 496             if (StatusCode >= HttpStatusCode.BadRequest)
 497                 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
 498         }
 499
 500
 501         private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
 502         {
 503             if (original==null)
 504                 throw new ArgumentNullException("original");
 505             Contract.EndContractBlock();
 506
 507             if (tcs == null)
 508                 tcs = new TaskCompletionSource<T>();
 509             Task.Factory.StartNew(original).ContinueWith(_original =>
 510                 {
 511                     if (!_original.IsFaulted)
 512                         tcs.SetFromTask(_original);
 513                     else
 514                     {
 515                         var e = _original.Exception.InnerException;
 516                         var we = (e as WebException);
 517                         if (we==null)
 518                             tcs.SetException(e);
 519                         else
 520                         {
 521                             var statusCode = GetStatusCode(we);
 522
 523                             //Return null for 404
 524                             if (statusCode == HttpStatusCode.NotFound)
 525                                 tcs.SetResult(default(T));
 526                             //Retry for timeouts and service unavailable
 527                             else if (we.Status == WebExceptionStatus.Timeout ||
 528                                 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
 529                             {
 530                                 TimedOut = true;
 531                                 if (retryCount == 0)
 532                                 {
 533                                     Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
 534                                     tcs.SetException(new RetryException("Timed out too many times.", e));
 535                                 }
 536                                 else
 537                                 {
 538                                     Log.ErrorFormat(
 539                                         "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
 540                                         retryCount, e);
 541                                     Retry(original, retryCount - 1, tcs);
 542                                 }
 543                             }
 544                             else
 545                                 tcs.SetException(e);
 546                         }
 547                     };
 548                 });
 549             return tcs.Task;
 550         }
 551
 552         private HttpStatusCode GetStatusCode(WebException we)
 553         {
 554             if (we==null)
 555                 throw new ArgumentNullException("we");
 556             var statusCode = HttpStatusCode.RequestTimeout;
 557             if (we.Response != null)
 558             {
 559                 statusCode = ((HttpWebResponse) we.Response).StatusCode;
 560                 this.StatusCode = statusCode;
 561             }
 562             return statusCode;
 563         }
 564
 565         public UriBuilder GetAddressBuilder(string container, string objectName)
 566         {
 567             var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
 568             return builder;
 569         }
 570
 571         public Dictionary<string, string> GetMeta(string metaPrefix)
 572         {
 573             if (String.IsNullOrWhiteSpace(metaPrefix))
 574                 throw new ArgumentNullException("metaPrefix");
 575             Contract.EndContractBlock();
 576
 577             var keys = ResponseHeaders.AllKeys.AsQueryable();
 578             var dict = (from key in keys
 579                         where key.StartsWith(metaPrefix)
 580                         let name = key.Substring(metaPrefix.Length)
 581                         select new { Name = name, Value = ResponseHeaders[key] })
 582                         .ToDictionary(t => t.Name, t => t.Value);
 583             return dict;
 584         }
 585
 586     }
 587
 588     public class RetryException:Exception
 589     {
 590         public RetryException()
 591             :base()
 592         {
 593
 594         }
 595
 596         public RetryException(string message)
 597             :base(message)
 598         {
 599
 600         }
 601
 602         public RetryException(string message,Exception innerException)
 603             :base(message,innerException)
 604         {
 605
 606         }
 607
 608         public RetryException(SerializationInfo info,StreamingContext context)
 609             :base(info,context)
 610         {
 611
 612         }
 613     }
 614 }