code.grnet.gr Git - pithos-ms-client/blob - trunk%2FPithos.Network%2FRestClient.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="RestClient.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System.Collections.Specialized;
  43 using System.Diagnostics;
  44 using System.Diagnostics.Contracts;
  45 using System.IO;
  46 using System.Net;
  47 using System.Reflection;
  48 using System.Runtime.Serialization;
  49 using System.Threading.Tasks;
  50 using log4net;
  51
  52
  53 namespace Pithos.Network
  54 {
  55     using System;
  56     using System.Collections.Generic;
  57     using System.Linq;
  58     using System.Text;
  59
  60     /// <summary>
  61     /// TODO: Update summary.
  62     /// </summary>
  63     public class RestClient:WebClient
  64     {
  65         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  66
  67         public int Timeout { get; set; }
  68
  69         public bool TimedOut { get; set; }
  70
  71         public HttpStatusCode StatusCode { get; private set; }
  72
  73         public string StatusDescription { get; set; }
  74
  75         public long? RangeFrom { get; set; }
  76         public long? RangeTo { get; set; }
  77
  78         public int Retries { get; set; }
  79
  80         private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
  81         public Dictionary<string, string> Parameters
  82         {
  83             get
  84             {
  85                 Contract.Ensures(_parameters!=null);
  86                 return _parameters;
  87             }
  88         }
  89
  90
  91         [ContractInvariantMethod]
  92         private void Invariants()
  93         {
  94             Contract.Invariant(Headers!=null);
  95         }
  96
  97         public RestClient():base()
  98         {
  99             //The maximum error response must be large because missing server hashes are return as a Conflivt (409) error response
 100             HttpWebRequest.DefaultMaximumErrorResponseLength = 16*1024*1024;
 101         }
 102
 103
 104         public RestClient(RestClient other)
 105             : base()
 106         {
 107             if (other==null)
 108                 throw new ArgumentNullException("other");
 109             Contract.EndContractBlock();
 110
 111             HttpWebRequest.DefaultMaximumErrorResponseLength = 16 * 1024 * 1024;
 112
 113             CopyHeaders(other);
 114             Timeout = other.Timeout;
 115             Retries = other.Retries;
 116             BaseAddress = other.BaseAddress;
 117
 118             foreach (var parameter in other.Parameters)
 119             {
 120                 Parameters.Add(parameter.Key,parameter.Value);
 121             }
 122
 123             this.Proxy = other.Proxy;
 124         }
 125
 126
 127         protected override WebRequest GetWebRequest(Uri address)
 128         {
 129             TimedOut = false;
 130             var webRequest = base.GetWebRequest(address);
 131             var request = (HttpWebRequest)webRequest;
 132             request.ServicePoint.ConnectionLimit = 50;
 133             if (IfModifiedSince.HasValue)
 134                 request.IfModifiedSince = IfModifiedSince.Value;
 135             request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
 136             if(Timeout>0)
 137                 request.Timeout = Timeout;
 138
 139             if (RangeFrom.HasValue)
 140             {
 141                 if (RangeTo.HasValue)
 142                     request.AddRange(RangeFrom.Value, RangeTo.Value);
 143                 else
 144                     request.AddRange(RangeFrom.Value);
 145             }
 146             return request;
 147         }
 148
 149         public DateTime? IfModifiedSince { get; set; }
 150
 151         //Asynchronous version
 152         protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
 153         {
 154             Log.InfoFormat("[{0}] {1}", request.Method, request.RequestUri);
 155             HttpWebResponse response = null;
 156
 157             try
 158             {
 159                 response = (HttpWebResponse)base.GetWebResponse(request, result);
 160             }
 161             catch (WebException exc)
 162             {
 163                 if (!TryGetResponse(exc, request,out response))
 164                     throw;
 165             }
 166
 167             StatusCode = response.StatusCode;
 168             LastModified = response.LastModified;
 169             StatusDescription = response.StatusDescription;
 170             return response;
 171
 172         }
 173
 174
 175         //Synchronous version
 176         protected override WebResponse GetWebResponse(WebRequest request)
 177         {
 178             HttpWebResponse response = null;
 179             try
 180             {
 181                 Log.InfoFormat("[{0}] {1}",request.Method,request.RequestUri);
 182                 response = (HttpWebResponse)base.GetWebResponse(request);
 183             }
 184             catch (WebException exc)
 185             {
 186                 if (!TryGetResponse(exc, request,out response))
 187                     throw;
 188             }
 189
 190             StatusCode = response.StatusCode;
 191             LastModified = response.LastModified;
 192             StatusDescription = response.StatusDescription;
 193             return response;
 194         }
 195
 196         private bool TryGetResponse(WebException exc, WebRequest request,out HttpWebResponse response)
 197         {
 198             response = null;
 199             //Fail on empty response
 200             if (exc.Response == null)
 201             {
 202                 Log.WarnFormat("[{0}] {1} {2}", request.Method, exc.Status, request.RequestUri);
 203                 return false;
 204             }
 205
 206             response = (exc.Response as HttpWebResponse);
 207             var statusCode = (int)response.StatusCode;
 208             //Succeed on allowed status codes
 209             if (AllowedStatusCodes.Contains(response.StatusCode))
 210             {
 211                 if (Log.IsDebugEnabled)
 212                     Log.DebugFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 213                 return true;
 214             }
 215
 216             Log.WarnFormat("[{0}] {1} {2}", request.Method, statusCode, request.RequestUri);
 217
 218             //Does the response have any content to log?
 219             if (exc.Response.ContentLength > 0)
 220             {
 221                 var content = LogContent(exc.Response);
 222                 Log.ErrorFormat(content);
 223             }
 224             return false;
 225         }
 226
 227         private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
 228
 229         public List<HttpStatusCode> AllowedStatusCodes
 230         {
 231             get
 232             {
 233                 return _allowedStatusCodes;
 234             }
 235         }
 236
 237         public DateTime LastModified { get; private set; }
 238
 239         private static string LogContent(WebResponse webResponse)
 240         {
 241             if (webResponse == null)
 242                 throw new ArgumentNullException("webResponse");
 243             Contract.EndContractBlock();
 244
 245             //The response stream must be copied to avoid affecting other code by disposing of the
 246             //original response stream.
 247             var stream = webResponse.GetResponseStream();
 248             using(var memStream=new MemoryStream())
 249             using (var reader = new StreamReader(memStream))
 250             {
 251                 stream.CopyTo(memStream);
 252                 string content = reader.ReadToEnd();
 253
 254                 stream.Seek(0,SeekOrigin.Begin);
 255                 return content;
 256             }
 257         }
 258
 259         public string DownloadStringWithRetry(string address,int retries=0)
 260         {
 261
 262             if (address == null)
 263                 throw new ArgumentNullException("address");
 264
 265             var actualAddress = GetActualAddress(address);
 266
 267             TraceStart("GET",actualAddress);
 268
 269             var actualRetries = (retries == 0) ? Retries : retries;
 270
 271             var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
 272
 273             var task = Retry(() =>
 274             {
 275                 var content = base.DownloadString(uriString);
 276
 277                 if (StatusCode == HttpStatusCode.NoContent)
 278                     return String.Empty;
 279                 return content;
 280
 281             }, actualRetries);
 282
 283             try
 284             {
 285                 var result = task.Result;
 286                 return result;
 287
 288             }
 289             catch (AggregateException exc)
 290             {
 291                 //If the task fails, propagate the original exception
 292                 if (exc.InnerException!=null)
 293                     throw exc.InnerException;
 294                 throw;
 295             }
 296         }
 297
 298         public void Head(string address,int retries=0)
 299         {
 300             AllowedStatusCodes.Add(HttpStatusCode.NotFound);
 301             RetryWithoutContent(address, retries, "HEAD");
 302         }
 303
 304         public void PutWithRetry(string address, int retries = 0)
 305         {
 306             RetryWithoutContent(address, retries, "PUT");
 307         }
 308
 309         public void DeleteWithRetry(string address,int retries=0)
 310         {
 311             RetryWithoutContent(address, retries, "DELETE");
 312         }
 313
 314         public string GetHeaderValue(string headerName,bool optional=false)
 315         {
 316             if (this.ResponseHeaders==null)
 317                 throw new InvalidOperationException("ResponseHeaders are null");
 318             Contract.EndContractBlock();
 319
 320             var values=this.ResponseHeaders.GetValues(headerName);
 321             if (values != null)
 322                 return values[0];
 323
 324             if (optional)
 325                 return null;
 326             //A required header was not found
 327             throw new WebException(String.Format("The {0}  header is missing", headerName));
 328         }
 329
 330         public void SetNonEmptyHeaderValue(string headerName, string value)
 331         {
 332             if (String.IsNullOrWhiteSpace(value))
 333                 return;
 334             Headers.Add(headerName,value);
 335         }
 336
 337         private void RetryWithoutContent(string address, int retries, string method)
 338         {
 339             if (address == null)
 340                 throw new ArgumentNullException("address");
 341
 342             var actualAddress = GetActualAddress(address);
 343             var actualRetries = (retries == 0) ? Retries : retries;
 344
 345             var task = Retry(() =>
 346             {
 347                 var uriString = String.Join("/",BaseAddress ,actualAddress);
 348                 var uri = new Uri(uriString);
 349                 var request =  GetWebRequest(uri);
 350                 request.Method = method;
 351                 if (ResponseHeaders!=null)
 352                     ResponseHeaders.Clear();
 353
 354                 TraceStart(method, uriString);
 355                 if (method == "PUT")
 356                     request.ContentLength = 0;
 357
 358                 //Have to use try/finally instead of using here, because WebClient needs a valid WebResponse object
 359                 //in order to return response headers
 360                 var response = (HttpWebResponse)GetWebResponse(request);
 361                 try
 362                 {
 363                     LastModified = response.LastModified;
 364                     StatusCode = response.StatusCode;
 365                     StatusDescription = response.StatusDescription;
 366                 }
 367                 finally
 368                 {
 369                     response.Close();
 370                 }
 371
 372
 373                 return 0;
 374             }, actualRetries);
 375
 376             try
 377             {
 378                 task.Wait();
 379             }
 380             catch (AggregateException ex)
 381             {
 382                 var exc = ex.InnerException;
 383                 if (exc is RetryException)
 384                 {
 385                     Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
 386                 }
 387                 else
 388                 {
 389                     Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
 390                 }
 391                 throw exc;
 392
 393             }
 394             catch(Exception ex)
 395             {
 396                 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
 397                 throw;
 398             }
 399         }
 400
 401         private static void TraceStart(string method, string actualAddress)
 402         {
 403             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
 404         }
 405
 406         private string GetActualAddress(string address)
 407         {
 408             if (Parameters.Count == 0)
 409                 return address;
 410             var addressBuilder=new StringBuilder(address);
 411
 412             bool isFirst = true;
 413             foreach (var parameter in Parameters)
 414             {
 415                 if(isFirst)
 416                     addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
 417                 else
 418                     addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
 419                 isFirst = false;
 420             }
 421             return addressBuilder.ToString();
 422         }
 423
 424         public string DownloadStringWithRetry(Uri address,int retries=0)
 425         {
 426             if (address == null)
 427                 throw new ArgumentNullException("address");
 428
 429             var actualRetries = (retries == 0) ? Retries : retries;
 430             var task = Retry(() =>
 431             {
 432                 var content = base.DownloadString(address);
 433
 434                 if (StatusCode == HttpStatusCode.NoContent)
 435                     return String.Empty;
 436                 return content;
 437
 438             }, actualRetries);
 439
 440             var result = task.Result;
 441             return result;
 442         }
 443
 444
 445         /// <summary>
 446         /// Copies headers from another RestClient
 447         /// </summary>
 448         /// <param name="source">The RestClient from which the headers are copied</param>
 449         public void CopyHeaders(RestClient source)
 450         {
 451             if (source == null)
 452                 throw new ArgumentNullException("source", "source can't be null");
 453             Contract.EndContractBlock();
 454             //The Headers getter initializes the property, it is never null
 455             Contract.Assume(Headers!=null);
 456
 457             CopyHeaders(source.Headers,Headers);
 458         }
 459
 460         /// <summary>
 461         /// Copies headers from one header collection to another
 462         /// </summary>
 463         /// <param name="source">The source collection from which the headers are copied</param>
 464         /// <param name="target">The target collection to which the headers are copied</param>
 465         public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
 466         {
 467             if (source == null)
 468                 throw new ArgumentNullException("source", "source can't be null");
 469             if (target == null)
 470                 throw new ArgumentNullException("target", "target can't be null");
 471             Contract.EndContractBlock();
 472
 473             for (int i = 0; i < source.Count; i++)
 474             {
 475                 target.Add(source.GetKey(i), source[i]);
 476             }
 477         }
 478
 479         public void AssertStatusOK(string message)
 480         {
 481             if (StatusCode >= HttpStatusCode.BadRequest)
 482                 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
 483         }
 484
 485
 486         private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
 487         {
 488             if (original==null)
 489                 throw new ArgumentNullException("original");
 490             Contract.EndContractBlock();
 491
 492             if (tcs == null)
 493                 tcs = new TaskCompletionSource<T>();
 494             Task.Factory.StartNew(original).ContinueWith(_original =>
 495                 {
 496                     if (!_original.IsFaulted)
 497                         tcs.SetFromTask(_original);
 498                     else
 499                     {
 500                         var e = _original.Exception.InnerException;
 501                         var we = (e as WebException);
 502                         if (we==null)
 503                             tcs.SetException(e);
 504                         else
 505                         {
 506                             var statusCode = GetStatusCode(we);
 507
 508                             //Return null for 404
 509                             if (statusCode == HttpStatusCode.NotFound)
 510                                 tcs.SetResult(default(T));
 511                             //Retry for timeouts and service unavailable
 512                             else if (we.Status == WebExceptionStatus.Timeout ||
 513                                 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
 514                             {
 515                                 TimedOut = true;
 516                                 if (retryCount == 0)
 517                                 {
 518                                     Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
 519                                     tcs.SetException(new RetryException("Timed out too many times.", e));
 520                                 }
 521                                 else
 522                                 {
 523                                     Log.ErrorFormat(
 524                                         "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
 525                                         retryCount, e);
 526                                     Retry(original, retryCount - 1, tcs);
 527                                 }
 528                             }
 529                             else
 530                                 tcs.SetException(e);
 531                         }
 532                     };
 533                 });
 534             return tcs.Task;
 535         }
 536
 537         private HttpStatusCode GetStatusCode(WebException we)
 538         {
 539             if (we==null)
 540                 throw new ArgumentNullException("we");
 541             var statusCode = HttpStatusCode.RequestTimeout;
 542             if (we.Response != null)
 543             {
 544                 statusCode = ((HttpWebResponse) we.Response).StatusCode;
 545                 this.StatusCode = statusCode;
 546             }
 547             return statusCode;
 548         }
 549
 550         public UriBuilder GetAddressBuilder(string container, string objectName)
 551         {
 552             var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
 553             return builder;
 554         }
 555
 556         public Dictionary<string, string> GetMeta(string metaPrefix)
 557         {
 558             if (String.IsNullOrWhiteSpace(metaPrefix))
 559                 throw new ArgumentNullException("metaPrefix");
 560             Contract.EndContractBlock();
 561
 562             var keys = ResponseHeaders.AllKeys.AsQueryable();
 563             var dict = (from key in keys
 564                         where key.StartsWith(metaPrefix)
 565                         let name = key.Substring(metaPrefix.Length)
 566                         select new { Name = name, Value = ResponseHeaders[key] })
 567                         .ToDictionary(t => t.Name, t => t.Value);
 568             return dict;
 569         }
 570     }
 571
 572     public class RetryException:Exception
 573     {
 574         public RetryException()
 575             :base()
 576         {
 577
 578         }
 579
 580         public RetryException(string message)
 581             :base(message)
 582         {
 583
 584         }
 585
 586         public RetryException(string message,Exception innerException)
 587             :base(message,innerException)
 588         {
 589
 590         }
 591
 592         public RetryException(SerializationInfo info,StreamingContext context)
 593             :base(info,context)
 594         {
 595
 596         }
 597     }
 598 }