code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/RestClient.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="RestClient.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System.Collections.Specialized;
  43 using System.Diagnostics;
  44 using System.Diagnostics.Contracts;
  45 using System.IO;
  46 using System.Net;
  47 using System.Reflection;
  48 using System.Runtime.Serialization;
  49 using System.Threading.Tasks;
  50 using log4net;
  51
  52
  53 namespace Pithos.Network
  54 {
  55     using System;
  56     using System.Collections.Generic;
  57     using System.Linq;
  58     using System.Text;
  59
  60     /// <summary>
  61     /// TODO: Update summary.
  62     /// </summary>
  63     public class RestClient:WebClient
  64     {
  65         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  66
  67         public int Timeout { get; set; }
  68
  69         public bool TimedOut { get; set; }
  70
  71         public HttpStatusCode StatusCode { get; private set; }
  72
  73         public string StatusDescription { get; set; }
  74
  75         public long? RangeFrom { get; set; }
  76         public long? RangeTo { get; set; }
  77
  78         public int Retries { get; set; }
  79
  80         private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
  81         public Dictionary<string, string> Parameters
  82         {
  83             get
  84             {
  85                 Contract.Ensures(_parameters!=null);
  86                 return _parameters;
  87             }
  88         }
  89
  90
  91         [ContractInvariantMethod]
  92         private void Invariants()
  93         {
  94             Contract.Invariant(Headers!=null);
  95         }
  96
  97         public RestClient():base()
  98         {
  99             //The maximum error response must be large because missing server hashes are return as a Conflivt (409) error response
 100             //Any value above 2^21-1 will result in an empty response.
 101             //-1 essentially ignores the maximum length
 102             HttpWebRequest.DefaultMaximumErrorResponseLength = -1;
 103         }
 104
 105
 106         public RestClient(RestClient other)
 107             : base()
 108         {
 109             if (other==null)
 110                 //Log.ErrorFormat("[ERROR] No parameters provided to the rest client. \n{0}\n", other);
 111                 throw new ArgumentNullException("other");
 112             Contract.EndContractBlock();
 113
 114             //The maximum error response must be large because missing server hashes are return as a Conflivt (409) error response
 115             //Any value above 2^21-1 will result in an empty response.
 116             //-1 essentially ignores the maximum length
 117             HttpWebRequest.DefaultMaximumErrorResponseLength = -1;
 118
 119
 120             CopyHeaders(other);
 121             Timeout = other.Timeout;
 122             Retries = other.Retries;
 123             BaseAddress = other.BaseAddress;
 124
 125             foreach (var parameter in other.Parameters)
 126             {
 127                 Parameters.Add(parameter.Key,parameter.Value);
 128             }
 129
 130             this.Proxy = other.Proxy;
 131         }
 132
 133
 134         protected override WebRequest GetWebRequest(Uri address)
 135         {
 136             TimedOut = false;
 137             var webRequest = base.GetWebRequest(address);
 138             var request = (HttpWebRequest)webRequest;
 139             request.CookieContainer=new CookieContainer();
 140             request.ServicePoint.ConnectionLimit = 50;
 141             if (IfModifiedSince.HasValue)
 142                 request.IfModifiedSince = IfModifiedSince.Value;
 143             request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
 144             if(Timeout>0)
 145                 request.Timeout = Timeout;
 146
 147             if (RangeFrom.HasValue)
 148             {
 149                 if (RangeTo.HasValue)
 150                     request.AddRange(RangeFrom.Value, RangeTo.Value);
 151                 else
 152                     request.AddRange(RangeFrom.Value);
 153             }
 154             return request;
 155         }
 156
 157         public DateTime? IfModifiedSince { get; set; }
 158
 159         //Asynchronous version
 160         protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
 161         {
 162             Log.InfoFormat("[{0}] {1}", request.Method, request.RequestUri);
 163             HttpWebResponse response = null;
 164
 165             try
 166             {
 167                 response = (HttpWebResponse)base.GetWebResponse(request, result);
 168             }
 169             catch (WebException exc)
 170             {
 171                 if (!TryGetResponse(exc, request,out response))
 172                     throw;
 173             }
 174
 175             StatusCode = response.StatusCode;
 176             LastModified = response.LastModified;
 177             StatusDescription = response.StatusDescription;
 178             return response;
 179
 180         }
 181
 182
 183         //Synchronous version
 184         protected override WebResponse GetWebResponse(WebRequest request)
 185         {
 186             HttpWebResponse response = null;
 187             try
 188             {
 189                 Log.InfoFormat("[{0}] {1}",request.Method,request.RequestUri);
 190                 response = (HttpWebResponse)base.GetWebResponse(request);
 191             }
 192             catch (WebException exc)
 193             {
 194                 if (!TryGetResponse(exc, request,out response))
 195                     throw;
 196             }
 197
 198             StatusCode = response.StatusCode;
 199             LastModified = response.LastModified;
 200             StatusDescription = response.StatusDescription;
 201             return response;
 202         }
 203
 204         private bool TryGetResponse(WebException exc, WebRequest request,out HttpWebResponse response)
 205         {
 206             response = null;
 207             //Fail on empty response
 208             if (exc.Response == null)
 209             {
 210                 Log.WarnFormat("[{0}] {1} {2}", request.Method, exc.Status, request.RequestUri);
 211                 return false;
 212             }
 213
 214             response = (exc.Response as HttpWebResponse);
 215             var statusCode = (int)response.StatusCode;
 216             //Succeed on allowed status codes
 217             if (AllowedStatusCodes.Contains(response.StatusCode))
 218             {
 219                 if (Log.IsDebugEnabled)
 220                     Log.DebugFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 221                 return true;
 222             }
 223
 224             Log.WarnFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 225
 226             //Does the response have any content to log?
 227             if (exc.Response.ContentLength > 0)
 228             {
 229                 var content = LogContent(exc.Response);
 230                 Log.ErrorFormat(content);
 231             }
 232             return false;
 233         }
 234
 235         private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
 236
 237         public List<HttpStatusCode> AllowedStatusCodes
 238         {
 239             get
 240             {
 241                 return _allowedStatusCodes;
 242             }
 243         }
 244
 245         public DateTime LastModified { get; private set; }
 246
 247         private static string LogContent(WebResponse webResponse)
 248         {
 249             if (webResponse == null)
 250                 throw new ArgumentNullException("webResponse");
 251             Contract.EndContractBlock();
 252
 253             //The response stream must be copied to avoid affecting other code by disposing of the
 254             //original response stream.
 255             var stream = webResponse.GetResponseStream();
 256             using(var memStream=new MemoryStream())
 257             using (var reader = new StreamReader(memStream))
 258             {
 259                 stream.CopyTo(memStream);
 260                 string content = reader.ReadToEnd();
 261
 262                 stream.Seek(0,SeekOrigin.Begin);
 263                 return content;
 264             }
 265         }
 266
 267         public string DownloadStringWithRetry(string address,int retries=0)
 268         {
 269
 270             if (address == null)
 271                 throw new ArgumentNullException("address");
 272
 273             var actualAddress = GetActualAddress(address);
 274
 275             TraceStart("GET",actualAddress);
 276
 277             var actualRetries = (retries == 0) ? Retries : retries;
 278
 279             var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
 280
 281             var task = Retry(() =>
 282             {
 283                 var content = DownloadString(uriString);
 284
 285                 if (StatusCode == HttpStatusCode.NoContent)
 286                     return String.Empty;
 287                 return content;
 288
 289             }, actualRetries);
 290
 291             try
 292             {
 293                     var result = task.Result;
 294                 return result;
 295
 296             }
 297             catch (AggregateException exc)
 298             {
 299                 //If the task fails, propagate the original exception
 300                 if (exc.InnerException!=null)
 301                     throw exc.InnerException;
 302                 throw;
 303             }
 304         }
 305
 306         public void Head(string address,int retries=0)
 307         {
 308             AllowedStatusCodes.Add(HttpStatusCode.NotFound);
 309             RetryWithoutContent(address, retries, "HEAD");
 310         }
 311
 312         public void PutWithRetry(string address, int retries = 0, string contentType=null)
 313         {
 314             RetryWithoutContent(address, retries, "PUT",contentType);
 315         }
 316
 317         public void PostWithRetry(string address,string contentType)
 318         {
 319             RetryWithoutContent(address, 0, "POST",contentType);
 320         }
 321
 322         public void DeleteWithRetry(string address,int retries=0)
 323         {
 324             RetryWithoutContent(address, retries, "DELETE");
 325         }
 326
 327         public string GetHeaderValue(string headerName,bool optional=false)
 328         {
 329             if (this.ResponseHeaders==null)
 330                 throw new InvalidOperationException("ResponseHeaders are null");
 331             Contract.EndContractBlock();
 332
 333             var values=this.ResponseHeaders.GetValues(headerName);
 334             if (values != null)
 335                 return values[0];
 336
 337             if (optional)
 338                 return null;
 339             //A required header was not found
 340             throw new WebException(String.Format("The {0}  header is missing", headerName));
 341         }
 342
 343         public void SetNonEmptyHeaderValue(string headerName, string value)
 344         {
 345             if (String.IsNullOrWhiteSpace(value))
 346                 return;
 347             Headers.Add(headerName,value);
 348         }
 349
 350         private void RetryWithoutContent(string address, int retries, string method,string contentType=null)
 351         {
 352             if (address == null)
 353                 throw new ArgumentNullException("address");
 354
 355             var actualAddress = GetActualAddress(address);
 356             var actualRetries = (retries == 0) ? Retries : retries;
 357
 358             var task = Retry(() =>
 359             {
 360                 var uriString = String.Join("/",BaseAddress ,actualAddress);
 361                 var uri = new Uri(uriString);
 362                 var request =  GetWebRequest(uri);
 363                 if (contentType!=null)
 364                 {
 365                     request.ContentType = contentType;
 366                     request.ContentLength = 0;
 367                 }
 368                 request.Method = method;
 369                 if (ResponseHeaders!=null)
 370                     ResponseHeaders.Clear();
 371
 372                 TraceStart(method, uriString);
 373                 if (method == "PUT")
 374                     request.ContentLength = 0;
 375
 376                 //Have to use try/finally instead of using here, because WebClient needs a valid WebResponse object
 377                 //in order to return response headers
 378                 var response = (HttpWebResponse)GetWebResponse(request);
 379                 try
 380                 {
 381                     LastModified = response.LastModified;
 382                     StatusCode = response.StatusCode;
 383                     StatusDescription = response.StatusDescription;
 384                 }
 385                 finally
 386                 {
 387                     response.Close();
 388                 }
 389
 390
 391                 return 0;
 392             }, actualRetries);
 393
 394             try
 395             {
 396                 task.Wait();
 397             }
 398             catch (AggregateException ex)
 399             {
 400                 var exc = ex.InnerException;
 401                 if (exc is RetryException)
 402                 {
 403                     Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
 404                 }
 405                 else
 406                 {
 407                     Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
 408                 }
 409                 throw exc;
 410
 411             }
 412             catch(Exception ex)
 413             {
 414                 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
 415                 throw;
 416             }
 417         }
 418
 419         private static void TraceStart(string method, string actualAddress)
 420         {
 421             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
 422         }
 423
 424         private string GetActualAddress(string address)
 425         {
 426             if (Parameters.Count == 0)
 427                 return address;
 428             var addressBuilder=new StringBuilder(address);
 429
 430             bool isFirst = true;
 431             foreach (var parameter in Parameters)
 432             {
 433                 if(isFirst)
 434                     addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
 435                 else
 436                     addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
 437                 isFirst = false;
 438             }
 439             return addressBuilder.ToString();
 440         }
 441
 442         public string DownloadStringWithRetry(Uri address,int retries=0)
 443         {
 444             if (address == null)
 445                 throw new ArgumentNullException("address");
 446
 447             var actualRetries = (retries == 0) ? Retries : retries;
 448             var task = Retry(() =>
 449             {
 450                 var content = base.DownloadString(address);
 451
 452                 if (StatusCode == HttpStatusCode.NoContent)
 453                     return String.Empty;
 454                 return content;
 455
 456             }, actualRetries);
 457
 458             var result = task.Result;
 459             return result;
 460         }
 461
 462
 463         /// <summary>
 464         /// Copies headers from another RestClient
 465         /// </summary>
 466         /// <param name="source">The RestClient from which the headers are copied</param>
 467         public void CopyHeaders(RestClient source)
 468         {
 469             if (source == null)
 470                 throw new ArgumentNullException("source", "source can't be null");
 471             Contract.EndContractBlock();
 472             //The Headers getter initializes the property, it is never null
 473             Contract.Assume(Headers!=null);
 474
 475             CopyHeaders(source.Headers,Headers);
 476         }
 477
 478         /// <summary>
 479         /// Copies headers from one header collection to another
 480         /// </summary>
 481         /// <param name="source">The source collection from which the headers are copied</param>
 482         /// <param name="target">The target collection to which the headers are copied</param>
 483         public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
 484         {
 485             if (source == null)
 486                 throw new ArgumentNullException("source", "source can't be null");
 487             if (target == null)
 488                 throw new ArgumentNullException("target", "target can't be null");
 489             Contract.EndContractBlock();
 490
 491             for (int i = 0; i < source.Count; i++)
 492             {
 493                 target.Add(source.GetKey(i), source[i]);
 494             }
 495         }
 496
 497         public void AssertStatusOK(string message)
 498         {
 499             if (StatusCode >= HttpStatusCode.BadRequest)
 500                 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
 501         }
 502
 503
 504         private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
 505         {
 506             if (original==null)
 507                 throw new ArgumentNullException("original");
 508             Contract.EndContractBlock();
 509
 510             if (tcs == null)
 511                 tcs = new TaskCompletionSource<T>();
 512             Task.Factory.StartNew(original).ContinueWith(_original =>
 513                 {
 514                     if (!_original.IsFaulted)
 515                         tcs.SetFromTask(_original);
 516                     else
 517                     {
 518                         var e = _original.Exception.InnerException;
 519                         var we = (e as WebException);
 520                         if (we==null)
 521                             tcs.SetException(e);
 522                         else
 523                         {
 524                             var statusCode = GetStatusCode(we);
 525
 526                             //Return null for 404
 527                             if (statusCode == HttpStatusCode.NotFound)
 528                                 tcs.SetResult(default(T));
 529                             //Retry for timeouts and service unavailable
 530                             else if (we.Status == WebExceptionStatus.Timeout ||
 531                                 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
 532                             {
 533                                 TimedOut = true;
 534                                 if (retryCount == 0)
 535                                 {
 536                                     Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
 537                                     tcs.SetException(new RetryException("Timed out too many times.", e));
 538                                 }
 539                                 else
 540                                 {
 541                                     Log.ErrorFormat(
 542                                         "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
 543                                         retryCount, e);
 544                                     Retry(original, retryCount - 1, tcs);
 545                                 }
 546                             }
 547                             else
 548                                 tcs.SetException(e);
 549                         }
 550                     };
 551                 });
 552             return tcs.Task;
 553         }
 554
 555         private HttpStatusCode GetStatusCode(WebException we)
 556         {
 557             if (we==null)
 558                 throw new ArgumentNullException("we");
 559             var statusCode = HttpStatusCode.RequestTimeout;
 560             if (we.Response != null)
 561             {
 562                 statusCode = ((HttpWebResponse) we.Response).StatusCode;
 563                 this.StatusCode = statusCode;
 564             }
 565             return statusCode;
 566         }
 567
 568         public UriBuilder GetAddressBuilder(string container, string objectName)
 569         {
 570             var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
 571             return builder;
 572         }
 573
 574         public Dictionary<string, string> GetMeta(string metaPrefix)
 575         {
 576             if (String.IsNullOrWhiteSpace(metaPrefix))
 577                 throw new ArgumentNullException("metaPrefix");
 578             Contract.EndContractBlock();
 579
 580             var keys = ResponseHeaders.AllKeys.AsQueryable();
 581             var dict = (from key in keys
 582                         where key.StartsWith(metaPrefix)
 583                         let name = key.Substring(metaPrefix.Length)
 584                         select new { Name = name, Value = ResponseHeaders[key] })
 585                         .ToDictionary(t => t.Name, t => t.Value);
 586             return dict;
 587         }
 588
 589     }
 590
 591     public class RetryException:Exception
 592     {
 593         public RetryException()
 594             :base()
 595         {
 596
 597         }
 598
 599         public RetryException(string message)
 600             :base(message)
 601         {
 602
 603         }
 604
 605         public RetryException(string message,Exception innerException)
 606             :base(message,innerException)
 607         {
 608
 609         }
 610
 611         public RetryException(SerializationInfo info,StreamingContext context)
 612             :base(info,context)
 613         {
 614
 615         }
 616     }
 617 }