code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/RestClient.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="RestClient.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System.Collections.Specialized;
  43 using System.Diagnostics;
  44 using System.Diagnostics.Contracts;
  45 using System.IO;
  46 using System.Net;
  47 using System.Reflection;
  48 using System.Runtime.Serialization;
  49 using System.Threading.Tasks;
  50 using log4net;
  51
  52
  53 namespace Pithos.Network
  54 {
  55     using System;
  56     using System.Collections.Generic;
  57     using System.Linq;
  58     using System.Text;
  59
  60     /// <summary>
  61     /// TODO: Update summary.
  62     /// </summary>
  63     public class RestClient:WebClient
  64     {
  65         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  66
  67         public int Timeout { get; set; }
  68
  69         public bool TimedOut { get; set; }
  70
  71         public HttpStatusCode StatusCode { get; private set; }
  72
  73         public string StatusDescription { get; set; }
  74
  75         public long? RangeFrom { get; set; }
  76         public long? RangeTo { get; set; }
  77
  78         public int Retries { get; set; }
  79
  80         private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
  81         public Dictionary<string, string> Parameters
  82         {
  83             get
  84             {
  85                 Contract.Ensures(_parameters!=null);
  86                 return _parameters;
  87             }
  88         }
  89
  90
  91         [ContractInvariantMethod]
  92         private void Invariants()
  93         {
  94             Contract.Invariant(Headers!=null);
  95         }
  96
  97         public RestClient():base()
  98         {
  99             //The maximum error response must be large because missing server hashes are return as a Conflivt (409) error response
 100             //Any value above 2^21-1 will result in an empty response.
 101             //-1 essentially ignores the maximum length
 102             HttpWebRequest.DefaultMaximumErrorResponseLength = -1;
 103         }
 104
 105
 106         public RestClient(RestClient other)
 107             : base()
 108         {
 109             if (other==null)
 110                 //Log.ErrorFormat("[ERROR] No parameters provided to the rest client. \n{0}\n", other);
 111                 throw new ArgumentNullException("other");
 112             Contract.EndContractBlock();
 113
 114             //The maximum error response must be large because missing server hashes are return as a Conflivt (409) error response
 115             //Any value above 2^21-1 will result in an empty response.
 116             //-1 essentially ignores the maximum length
 117             HttpWebRequest.DefaultMaximumErrorResponseLength = -1;
 118
 119             CopyHeaders(other);
 120             Timeout = other.Timeout;
 121             Retries = other.Retries;
 122             BaseAddress = other.BaseAddress;
 123
 124             foreach (var parameter in other.Parameters)
 125             {
 126                 Parameters.Add(parameter.Key,parameter.Value);
 127             }
 128
 129             this.Proxy = other.Proxy;
 130         }
 131
 132
 133         protected override WebRequest GetWebRequest(Uri address)
 134         {
 135             TimedOut = false;
 136             var webRequest = base.GetWebRequest(address);
 137             var request = (HttpWebRequest)webRequest;
 138             request.ServicePoint.ConnectionLimit = 50;
 139             if (IfModifiedSince.HasValue)
 140                 request.IfModifiedSince = IfModifiedSince.Value;
 141             request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
 142             if(Timeout>0)
 143                 request.Timeout = Timeout;
 144
 145             if (RangeFrom.HasValue)
 146             {
 147                 if (RangeTo.HasValue)
 148                     request.AddRange(RangeFrom.Value, RangeTo.Value);
 149                 else
 150                     request.AddRange(RangeFrom.Value);
 151             }
 152             return request;
 153         }
 154
 155         public DateTime? IfModifiedSince { get; set; }
 156
 157         //Asynchronous version
 158         protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
 159         {
 160             Log.InfoFormat("[{0}] {1}", request.Method, request.RequestUri);
 161             HttpWebResponse response = null;
 162
 163             try
 164             {
 165                 response = (HttpWebResponse)base.GetWebResponse(request, result);
 166             }
 167             catch (WebException exc)
 168             {
 169                 if (!TryGetResponse(exc, request,out response))
 170                     throw;
 171             }
 172
 173             StatusCode = response.StatusCode;
 174             LastModified = response.LastModified;
 175             StatusDescription = response.StatusDescription;
 176             return response;
 177
 178         }
 179
 180
 181         //Synchronous version
 182         protected override WebResponse GetWebResponse(WebRequest request)
 183         {
 184             HttpWebResponse response = null;
 185             try
 186             {
 187                 Log.InfoFormat("[{0}] {1}",request.Method,request.RequestUri);
 188                 response = (HttpWebResponse)base.GetWebResponse(request);
 189             }
 190             catch (WebException exc)
 191             {
 192                 if (!TryGetResponse(exc, request,out response))
 193                     throw;
 194             }
 195
 196             StatusCode = response.StatusCode;
 197             LastModified = response.LastModified;
 198             StatusDescription = response.StatusDescription;
 199             return response;
 200         }
 201
 202         private bool TryGetResponse(WebException exc, WebRequest request,out HttpWebResponse response)
 203         {
 204             response = null;
 205             //Fail on empty response
 206             if (exc.Response == null)
 207             {
 208                 Log.WarnFormat("[{0}] {1} {2}", request.Method, exc.Status, request.RequestUri);
 209                 return false;
 210             }
 211
 212             response = (exc.Response as HttpWebResponse);
 213             var statusCode = (int)response.StatusCode;
 214             //Succeed on allowed status codes
 215             if (AllowedStatusCodes.Contains(response.StatusCode))
 216             {
 217                 if (Log.IsDebugEnabled)
 218                     Log.DebugFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 219                 return true;
 220             }
 221
 222             Log.WarnFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 223
 224             //Does the response have any content to log?
 225             if (exc.Response.ContentLength > 0)
 226             {
 227                 var content = LogContent(exc.Response);
 228                 Log.ErrorFormat(content);
 229             }
 230             return false;
 231         }
 232
 233         private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
 234
 235         public List<HttpStatusCode> AllowedStatusCodes
 236         {
 237             get
 238             {
 239                 return _allowedStatusCodes;
 240             }
 241         }
 242
 243         public DateTime LastModified { get; private set; }
 244
 245         private static string LogContent(WebResponse webResponse)
 246         {
 247             if (webResponse == null)
 248                 throw new ArgumentNullException("webResponse");
 249             Contract.EndContractBlock();
 250
 251             //The response stream must be copied to avoid affecting other code by disposing of the
 252             //original response stream.
 253             var stream = webResponse.GetResponseStream();
 254             using(var memStream=new MemoryStream())
 255             using (var reader = new StreamReader(memStream))
 256             {
 257                 stream.CopyTo(memStream);
 258                 string content = reader.ReadToEnd();
 259
 260                 stream.Seek(0,SeekOrigin.Begin);
 261                 return content;
 262             }
 263         }
 264
 265         public string DownloadStringWithRetry(string address,int retries=0)
 266         {
 267
 268             if (address == null)
 269                 throw new ArgumentNullException("address");
 270
 271             var actualAddress = GetActualAddress(address);
 272
 273             TraceStart("GET",actualAddress);
 274
 275             var actualRetries = (retries == 0) ? Retries : retries;
 276
 277             var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
 278
 279             var task = Retry(() =>
 280             {
 281                 var content = base.DownloadString(uriString);
 282
 283                 if (StatusCode == HttpStatusCode.NoContent)
 284                     return String.Empty;
 285                 return content;
 286
 287             }, actualRetries);
 288
 289             try
 290             {
 291                     var result = task.Result;
 292                 return result;
 293
 294             }
 295             catch (AggregateException exc)
 296             {
 297                 //If the task fails, propagate the original exception
 298                 if (exc.InnerException!=null)
 299                     throw exc.InnerException;
 300                 throw;
 301             }
 302         }
 303
 304         public void Head(string address,int retries=0)
 305         {
 306             AllowedStatusCodes.Add(HttpStatusCode.NotFound);
 307             RetryWithoutContent(address, retries, "HEAD");
 308         }
 309
 310         public void PutWithRetry(string address, int retries = 0, string contentType=null)
 311         {
 312             RetryWithoutContent(address, retries, "PUT",contentType);
 313         }
 314
 315         public void PostWithRetry(string address,string contentType)
 316         {
 317             RetryWithoutContent(address, 0, "POST",contentType);
 318         }
 319
 320         public void DeleteWithRetry(string address,int retries=0)
 321         {
 322             RetryWithoutContent(address, retries, "DELETE");
 323         }
 324
 325         public string GetHeaderValue(string headerName,bool optional=false)
 326         {
 327             if (this.ResponseHeaders==null)
 328                 throw new InvalidOperationException("ResponseHeaders are null");
 329             Contract.EndContractBlock();
 330
 331             var values=this.ResponseHeaders.GetValues(headerName);
 332             if (values != null)
 333                 return values[0];
 334
 335             if (optional)
 336                 return null;
 337             //A required header was not found
 338             throw new WebException(String.Format("The {0}  header is missing", headerName));
 339         }
 340
 341         public void SetNonEmptyHeaderValue(string headerName, string value)
 342         {
 343             if (String.IsNullOrWhiteSpace(value))
 344                 return;
 345             Headers.Add(headerName,value);
 346         }
 347
 348         private void RetryWithoutContent(string address, int retries, string method,string contentType=null)
 349         {
 350             if (address == null)
 351                 throw new ArgumentNullException("address");
 352
 353             var actualAddress = GetActualAddress(address);
 354             var actualRetries = (retries == 0) ? Retries : retries;
 355
 356             var task = Retry(() =>
 357             {
 358                 var uriString = String.Join("/",BaseAddress ,actualAddress);
 359                 var uri = new Uri(uriString);
 360                 var request =  GetWebRequest(uri);
 361                 if (contentType!=null)
 362                 {
 363                     request.ContentType = contentType;
 364                     request.ContentLength = 0;
 365                 }
 366                 request.Method = method;
 367                 if (ResponseHeaders!=null)
 368                     ResponseHeaders.Clear();
 369
 370                 TraceStart(method, uriString);
 371                 if (method == "PUT")
 372                     request.ContentLength = 0;
 373
 374                 //Have to use try/finally instead of using here, because WebClient needs a valid WebResponse object
 375                 //in order to return response headers
 376                 var response = (HttpWebResponse)GetWebResponse(request);
 377                 try
 378                 {
 379                     LastModified = response.LastModified;
 380                     StatusCode = response.StatusCode;
 381                     StatusDescription = response.StatusDescription;
 382                 }
 383                 finally
 384                 {
 385                     response.Close();
 386                 }
 387
 388
 389                 return 0;
 390             }, actualRetries);
 391
 392             try
 393             {
 394                 task.Wait();
 395             }
 396             catch (AggregateException ex)
 397             {
 398                 var exc = ex.InnerException;
 399                 if (exc is RetryException)
 400                 {
 401                     Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
 402                 }
 403                 else
 404                 {
 405                     Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
 406                 }
 407                 throw exc;
 408
 409             }
 410             catch(Exception ex)
 411             {
 412                 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
 413                 throw;
 414             }
 415         }
 416
 417         private static void TraceStart(string method, string actualAddress)
 418         {
 419             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
 420         }
 421
 422         private string GetActualAddress(string address)
 423         {
 424             if (Parameters.Count == 0)
 425                 return address;
 426             var addressBuilder=new StringBuilder(address);
 427
 428             bool isFirst = true;
 429             foreach (var parameter in Parameters)
 430             {
 431                 if(isFirst)
 432                     addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
 433                 else
 434                     addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
 435                 isFirst = false;
 436             }
 437             return addressBuilder.ToString();
 438         }
 439
 440         public string DownloadStringWithRetry(Uri address,int retries=0)
 441         {
 442             if (address == null)
 443                 throw new ArgumentNullException("address");
 444
 445             var actualRetries = (retries == 0) ? Retries : retries;
 446             var task = Retry(() =>
 447             {
 448                 var content = base.DownloadString(address);
 449
 450                 if (StatusCode == HttpStatusCode.NoContent)
 451                     return String.Empty;
 452                 return content;
 453
 454             }, actualRetries);
 455
 456             var result = task.Result;
 457             return result;
 458         }
 459
 460
 461         /// <summary>
 462         /// Copies headers from another RestClient
 463         /// </summary>
 464         /// <param name="source">The RestClient from which the headers are copied</param>
 465         public void CopyHeaders(RestClient source)
 466         {
 467             if (source == null)
 468                 throw new ArgumentNullException("source", "source can't be null");
 469             Contract.EndContractBlock();
 470             //The Headers getter initializes the property, it is never null
 471             Contract.Assume(Headers!=null);
 472
 473             CopyHeaders(source.Headers,Headers);
 474         }
 475
 476         /// <summary>
 477         /// Copies headers from one header collection to another
 478         /// </summary>
 479         /// <param name="source">The source collection from which the headers are copied</param>
 480         /// <param name="target">The target collection to which the headers are copied</param>
 481         public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
 482         {
 483             if (source == null)
 484                 throw new ArgumentNullException("source", "source can't be null");
 485             if (target == null)
 486                 throw new ArgumentNullException("target", "target can't be null");
 487             Contract.EndContractBlock();
 488
 489             for (int i = 0; i < source.Count; i++)
 490             {
 491                 target.Add(source.GetKey(i), source[i]);
 492             }
 493         }
 494
 495         public void AssertStatusOK(string message)
 496         {
 497             if (StatusCode >= HttpStatusCode.BadRequest)
 498                 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
 499         }
 500
 501
 502         private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
 503         {
 504             if (original==null)
 505                 throw new ArgumentNullException("original");
 506             Contract.EndContractBlock();
 507
 508             if (tcs == null)
 509                 tcs = new TaskCompletionSource<T>();
 510             Task.Factory.StartNew(original).ContinueWith(_original =>
 511                 {
 512                     if (!_original.IsFaulted)
 513                         tcs.SetFromTask(_original);
 514                     else
 515                     {
 516                         var e = _original.Exception.InnerException;
 517                         var we = (e as WebException);
 518                         if (we==null)
 519                             tcs.SetException(e);
 520                         else
 521                         {
 522                             var statusCode = GetStatusCode(we);
 523
 524                             //Return null for 404
 525                             if (statusCode == HttpStatusCode.NotFound)
 526                                 tcs.SetResult(default(T));
 527                             //Retry for timeouts and service unavailable
 528                             else if (we.Status == WebExceptionStatus.Timeout ||
 529                                 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
 530                             {
 531                                 TimedOut = true;
 532                                 if (retryCount == 0)
 533                                 {
 534                                     Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
 535                                     tcs.SetException(new RetryException("Timed out too many times.", e));
 536                                 }
 537                                 else
 538                                 {
 539                                     Log.ErrorFormat(
 540                                         "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
 541                                         retryCount, e);
 542                                     Retry(original, retryCount - 1, tcs);
 543                                 }
 544                             }
 545                             else
 546                                 tcs.SetException(e);
 547                         }
 548                     };
 549                 });
 550             return tcs.Task;
 551         }
 552
 553         private HttpStatusCode GetStatusCode(WebException we)
 554         {
 555             if (we==null)
 556                 throw new ArgumentNullException("we");
 557             var statusCode = HttpStatusCode.RequestTimeout;
 558             if (we.Response != null)
 559             {
 560                 statusCode = ((HttpWebResponse) we.Response).StatusCode;
 561                 this.StatusCode = statusCode;
 562             }
 563             return statusCode;
 564         }
 565
 566         public UriBuilder GetAddressBuilder(string container, string objectName)
 567         {
 568             var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
 569             return builder;
 570         }
 571
 572         public Dictionary<string, string> GetMeta(string metaPrefix)
 573         {
 574             if (String.IsNullOrWhiteSpace(metaPrefix))
 575                 throw new ArgumentNullException("metaPrefix");
 576             Contract.EndContractBlock();
 577
 578             var keys = ResponseHeaders.AllKeys.AsQueryable();
 579             var dict = (from key in keys
 580                         where key.StartsWith(metaPrefix)
 581                         let name = key.Substring(metaPrefix.Length)
 582                         select new { Name = name, Value = ResponseHeaders[key] })
 583                         .ToDictionary(t => t.Name, t => t.Value);
 584             return dict;
 585         }
 586
 587     }
 588
 589     public class RetryException:Exception
 590     {
 591         public RetryException()
 592             :base()
 593         {
 594
 595         }
 596
 597         public RetryException(string message)
 598             :base(message)
 599         {
 600
 601         }
 602
 603         public RetryException(string message,Exception innerException)
 604             :base(message,innerException)
 605         {
 606
 607         }
 608
 609         public RetryException(SerializationInfo info,StreamingContext context)
 610             :base(info,context)
 611         {
 612
 613         }
 614     }
 615 }