code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/RestClient.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="RestClient.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System.Collections.Specialized;
  43 using System.Diagnostics;
  44 using System.Diagnostics.Contracts;
  45 using System.IO;
  46 using System.Net;
  47 using System.Reflection;
  48 using System.Runtime.Serialization;
  49 using System.Threading.Tasks;
  50 using log4net;
  51
  52
  53 namespace Pithos.Network
  54 {
  55     using System;
  56     using System.Collections.Generic;
  57     using System.Linq;
  58     using System.Text;
  59
  60     /// <summary>
  61     /// TODO: Update summary.
  62     /// </summary>
  63     public class RestClient:WebClient
  64     {
  65         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  66
  67         public int Timeout { get; set; }
  68
  69         public bool TimedOut { get; set; }
  70
  71         public HttpStatusCode StatusCode { get; private set; }
  72
  73         public string StatusDescription { get; set; }
  74
  75         public long? RangeFrom { get; set; }
  76         public long? RangeTo { get; set; }
  77
  78         public int Retries { get; set; }
  79
  80         private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
  81         public Dictionary<string, string> Parameters
  82         {
  83             get
  84             {
  85                 Contract.Ensures(_parameters!=null);
  86                 return _parameters;
  87             }
  88         }
  89
  90
  91         [ContractInvariantMethod]
  92         private void Invariants()
  93         {
  94             Contract.Invariant(Headers!=null);
  95         }
  96
  97         public RestClient():base()\r
  98         {\r
  99             HttpWebRequest.DefaultMaximumErrorResponseLength = 67108864;\r
 100         }
 101
 102
 103         public RestClient(RestClient other)
 104             : base()
 105         {
 106             if (other==null)
 107                 throw new ArgumentNullException("other");
 108             Contract.EndContractBlock();
 109
 110             CopyHeaders(other);
 111             Timeout = other.Timeout;
 112             Retries = other.Retries;
 113             BaseAddress = other.BaseAddress;
 114
 115             foreach (var parameter in other.Parameters)
 116             {
 117                 Parameters.Add(parameter.Key,parameter.Value);
 118             }
 119
 120             this.Proxy = other.Proxy;
 121         }
 122
 123
 124         protected override WebRequest GetWebRequest(Uri address)
 125         {
 126             TimedOut = false;
 127             var webRequest = base.GetWebRequest(address);
 128             var request = (HttpWebRequest)webRequest;
 129             request.ServicePoint.ConnectionLimit = 50;
 130             if (IfModifiedSince.HasValue)
 131                 request.IfModifiedSince = IfModifiedSince.Value;
 132             request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
 133             if(Timeout>0)
 134                 request.Timeout = Timeout;
 135
 136             if (RangeFrom.HasValue)
 137             {
 138                 if (RangeTo.HasValue)
 139                     request.AddRange(RangeFrom.Value, RangeTo.Value);
 140                 else
 141                     request.AddRange(RangeFrom.Value);
 142             }
 143             return request;
 144         }
 145
 146         public DateTime? IfModifiedSince { get; set; }
 147
 148         //Asynchronous version
 149         protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
 150         {
 151             Log.InfoFormat("[{0}] {1}", request.Method, request.RequestUri);
 152             HttpWebResponse response = null;
 153
 154             try
 155             {
 156                 response = (HttpWebResponse)base.GetWebResponse(request, result);
 157             }
 158             catch (WebException exc)
 159             {
 160                 if (!TryGetResponse(exc, request,out response))
 161                     throw;
 162             }
 163
 164             StatusCode = response.StatusCode;
 165             LastModified = response.LastModified;
 166             StatusDescription = response.StatusDescription;
 167             return response;
 168
 169         }
 170
 171
 172         //Synchronous version
 173         protected override WebResponse GetWebResponse(WebRequest request)
 174         {
 175             HttpWebResponse response = null;
 176             try
 177             {
 178                 Log.InfoFormat("[{0}] {1}",request.Method,request.RequestUri);
 179                 response = (HttpWebResponse)base.GetWebResponse(request);
 180             }
 181             catch (WebException exc)
 182             {
 183                 if (!TryGetResponse(exc, request,out response))
 184                     throw;
 185             }
 186
 187             StatusCode = response.StatusCode;
 188             LastModified = response.LastModified;
 189             StatusDescription = response.StatusDescription;
 190             return response;
 191         }
 192
 193         private bool TryGetResponse(WebException exc, WebRequest request,out HttpWebResponse response)
 194         {
 195             response = null;
 196             //Fail on empty response
 197             if (exc.Response == null)
 198             {
 199                 Log.WarnFormat("[{0}] {1} {2}", request.Method, exc.Status, request.RequestUri);
 200                 return false;
 201             }
 202
 203             response = (exc.Response as HttpWebResponse);
 204             var statusCode = (int)response.StatusCode;
 205             //Succeed on allowed status codes
 206             if (AllowedStatusCodes.Contains(response.StatusCode))
 207             {
 208                 if (Log.IsDebugEnabled)
 209                     Log.DebugFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 210                 return true;
 211             }
 212
 213             Log.WarnFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 214
 215             //Does the response have any content to log?
 216             if (exc.Response.ContentLength > 0)
 217             {
 218                 var content = LogContent(exc.Response);
 219                 Log.ErrorFormat(content);
 220             }
 221             return false;
 222         }
 223
 224         private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
 225
 226         public List<HttpStatusCode> AllowedStatusCodes
 227         {
 228             get
 229             {
 230                 return _allowedStatusCodes;
 231             }
 232         }
 233
 234         public DateTime LastModified { get; private set; }
 235
 236         private static string LogContent(WebResponse webResponse)
 237         {
 238             if (webResponse == null)
 239                 throw new ArgumentNullException("webResponse");
 240             Contract.EndContractBlock();
 241
 242             //The response stream must be copied to avoid affecting other code by disposing of the
 243             //original response stream.
 244             var stream = webResponse.GetResponseStream();
 245             using(var memStream=new MemoryStream())
 246             using (var reader = new StreamReader(memStream))
 247             {
 248                 stream.CopyTo(memStream);
 249                 string content = reader.ReadToEnd();
 250
 251                 stream.Seek(0,SeekOrigin.Begin);
 252                 return content;
 253             }
 254         }
 255
 256         public string DownloadStringWithRetry(string address,int retries=0)
 257         {
 258
 259             if (address == null)
 260                 throw new ArgumentNullException("address");
 261
 262             var actualAddress = GetActualAddress(address);
 263
 264             TraceStart("GET",actualAddress);
 265
 266             var actualRetries = (retries == 0) ? Retries : retries;
 267
 268             var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
 269
 270             var task = Retry(() =>
 271             {
 272                 var content = base.DownloadString(uriString);
 273
 274                 if (StatusCode == HttpStatusCode.NoContent)
 275                     return String.Empty;
 276                 return content;
 277
 278             }, actualRetries);
 279
 280             try
 281             {
 282                 var result = task.Result;
 283                 return result;
 284
 285             }
 286             catch (AggregateException exc)
 287             {
 288                 //If the task fails, propagate the original exception
 289                 if (exc.InnerException!=null)
 290                     throw exc.InnerException;
 291                 throw;
 292             }
 293         }
 294
 295         public void Head(string address,int retries=0)
 296         {
 297             AllowedStatusCodes.Add(HttpStatusCode.NotFound);
 298             RetryWithoutContent(address, retries, "HEAD");
 299         }
 300
 301         public void PutWithRetry(string address, int retries = 0)
 302         {
 303             RetryWithoutContent(address, retries, "PUT");
 304         }
 305
 306         public void DeleteWithRetry(string address,int retries=0)
 307         {
 308             RetryWithoutContent(address, retries, "DELETE");
 309         }
 310
 311         public string GetHeaderValue(string headerName,bool optional=false)
 312         {
 313             if (this.ResponseHeaders==null)
 314                 throw new InvalidOperationException("ResponseHeaders are null");
 315             Contract.EndContractBlock();
 316
 317             var values=this.ResponseHeaders.GetValues(headerName);
 318             if (values != null)
 319                 return values[0];
 320
 321             if (optional)
 322                 return null;
 323             //A required header was not found
 324             throw new WebException(String.Format("The {0}  header is missing", headerName));
 325         }
 326
 327         public void SetNonEmptyHeaderValue(string headerName, string value)
 328         {
 329             if (String.IsNullOrWhiteSpace(value))
 330                 return;
 331             Headers.Add(headerName,value);
 332         }
 333
 334         private void RetryWithoutContent(string address, int retries, string method)
 335         {
 336             if (address == null)
 337                 throw new ArgumentNullException("address");
 338
 339             var actualAddress = GetActualAddress(address);
 340             var actualRetries = (retries == 0) ? Retries : retries;
 341
 342             var task = Retry(() =>
 343             {
 344                 var uriString = String.Join("/",BaseAddress ,actualAddress);
 345                 var uri = new Uri(uriString);
 346                 var request =  GetWebRequest(uri);
 347                 request.Method = method;
 348                 if (ResponseHeaders!=null)
 349                     ResponseHeaders.Clear();
 350
 351                 TraceStart(method, uriString);
 352                 if (method == "PUT")
 353                     request.ContentLength = 0;
 354
 355                 //Have to use try/finally instead of using here, because WebClient needs a valid WebResponse object
 356                 //in order to return response headers
 357                 var response = (HttpWebResponse)GetWebResponse(request);
 358                 try
 359                 {
 360                     LastModified = response.LastModified;
 361                     StatusCode = response.StatusCode;
 362                     StatusDescription = response.StatusDescription;
 363                 }
 364                 finally
 365                 {
 366                     response.Close();
 367                 }
 368
 369
 370                 return 0;
 371             }, actualRetries);
 372
 373             try
 374             {
 375                 task.Wait();
 376             }
 377             catch (AggregateException ex)
 378             {
 379                 var exc = ex.InnerException;
 380                 if (exc is RetryException)
 381                 {
 382                     Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
 383                 }
 384                 else
 385                 {
 386                     Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
 387                 }
 388                 throw exc;
 389
 390             }
 391             catch(Exception ex)
 392             {
 393                 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
 394                 throw;
 395             }
 396         }
 397
 398         private static void TraceStart(string method, string actualAddress)
 399         {
 400             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
 401         }
 402
 403         private string GetActualAddress(string address)
 404         {
 405             if (Parameters.Count == 0)
 406                 return address;
 407             var addressBuilder=new StringBuilder(address);
 408
 409             bool isFirst = true;
 410             foreach (var parameter in Parameters)
 411             {
 412                 if(isFirst)
 413                     addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
 414                 else
 415                     addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
 416                 isFirst = false;
 417             }
 418             return addressBuilder.ToString();
 419         }
 420
 421         public string DownloadStringWithRetry(Uri address,int retries=0)
 422         {
 423             if (address == null)
 424                 throw new ArgumentNullException("address");
 425
 426             var actualRetries = (retries == 0) ? Retries : retries;
 427             var task = Retry(() =>
 428             {
 429                 var content = base.DownloadString(address);
 430
 431                 if (StatusCode == HttpStatusCode.NoContent)
 432                     return String.Empty;
 433                 return content;
 434
 435             }, actualRetries);
 436
 437             var result = task.Result;
 438             return result;
 439         }
 440
 441
 442         /// <summary>
 443         /// Copies headers from another RestClient
 444         /// </summary>
 445         /// <param name="source">The RestClient from which the headers are copied</param>
 446         public void CopyHeaders(RestClient source)
 447         {
 448             if (source == null)
 449                 throw new ArgumentNullException("source", "source can't be null");
 450             Contract.EndContractBlock();
 451             //The Headers getter initializes the property, it is never null
 452             Contract.Assume(Headers!=null);
 453
 454             CopyHeaders(source.Headers,Headers);
 455         }
 456
 457         /// <summary>
 458         /// Copies headers from one header collection to another
 459         /// </summary>
 460         /// <param name="source">The source collection from which the headers are copied</param>
 461         /// <param name="target">The target collection to which the headers are copied</param>
 462         public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
 463         {
 464             if (source == null)
 465                 throw new ArgumentNullException("source", "source can't be null");
 466             if (target == null)
 467                 throw new ArgumentNullException("target", "target can't be null");
 468             Contract.EndContractBlock();
 469
 470             for (int i = 0; i < source.Count; i++)
 471             {
 472                 target.Add(source.GetKey(i), source[i]);
 473             }
 474         }
 475
 476         public void AssertStatusOK(string message)
 477         {
 478             if (StatusCode >= HttpStatusCode.BadRequest)
 479                 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
 480         }
 481
 482
 483         private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
 484         {
 485             if (original==null)
 486                 throw new ArgumentNullException("original");
 487             Contract.EndContractBlock();
 488
 489             if (tcs == null)
 490                 tcs = new TaskCompletionSource<T>();
 491             Task.Factory.StartNew(original).ContinueWith(_original =>
 492                 {
 493                     if (!_original.IsFaulted)
 494                         tcs.SetFromTask(_original);
 495                     else
 496                     {
 497                         var e = _original.Exception.InnerException;
 498                         var we = (e as WebException);
 499                         if (we==null)
 500                             tcs.SetException(e);
 501                         else
 502                         {
 503                             var statusCode = GetStatusCode(we);
 504
 505                             //Return null for 404
 506                             if (statusCode == HttpStatusCode.NotFound)
 507                                 tcs.SetResult(default(T));
 508                             //Retry for timeouts and service unavailable
 509                             else if (we.Status == WebExceptionStatus.Timeout ||
 510                                 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
 511                             {
 512                                 TimedOut = true;
 513                                 if (retryCount == 0)
 514                                 {
 515                                     Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
 516                                     tcs.SetException(new RetryException("Timed out too many times.", e));
 517                                 }
 518                                 else
 519                                 {
 520                                     Log.ErrorFormat(
 521                                         "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
 522                                         retryCount, e);
 523                                     Retry(original, retryCount - 1, tcs);
 524                                 }
 525                             }
 526                             else
 527                                 tcs.SetException(e);
 528                         }
 529                     };
 530                 });
 531             return tcs.Task;
 532         }
 533
 534         private HttpStatusCode GetStatusCode(WebException we)
 535         {
 536             if (we==null)
 537                 throw new ArgumentNullException("we");
 538             var statusCode = HttpStatusCode.RequestTimeout;
 539             if (we.Response != null)
 540             {
 541                 statusCode = ((HttpWebResponse) we.Response).StatusCode;
 542                 this.StatusCode = statusCode;
 543             }
 544             return statusCode;
 545         }
 546
 547         public UriBuilder GetAddressBuilder(string container, string objectName)
 548         {
 549             var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
 550             return builder;
 551         }
 552
 553         public Dictionary<string, string> GetMeta(string metaPrefix)
 554         {
 555             if (String.IsNullOrWhiteSpace(metaPrefix))
 556                 throw new ArgumentNullException("metaPrefix");
 557             Contract.EndContractBlock();
 558
 559             var keys = ResponseHeaders.AllKeys.AsQueryable();
 560             var dict = (from key in keys
 561                         where key.StartsWith(metaPrefix)
 562                         let name = key.Substring(metaPrefix.Length)
 563                         select new { Name = name, Value = ResponseHeaders[key] })
 564                         .ToDictionary(t => t.Name, t => t.Value);
 565             return dict;
 566         }
 567     }
 568
 569     public class RetryException:Exception
 570     {
 571         public RetryException()
 572             :base()
 573         {
 574
 575         }
 576
 577         public RetryException(string message)
 578             :base(message)
 579         {
 580
 581         }
 582
 583         public RetryException(string message,Exception innerException)
 584             :base(message,innerException)
 585         {
 586
 587         }
 588
 589         public RetryException(SerializationInfo info,StreamingContext context)
 590             :base(info,context)
 591         {
 592
 593         }
 594     }
 595 }