code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/RestClient.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="RestClient.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System.Collections.Specialized;
  43 using System.Diagnostics;
  44 using System.Diagnostics.Contracts;
  45 using System.IO;
  46 using System.Net;
  47 using System.Reflection;
  48 using System.Runtime.Serialization;
  49 using System.Threading.Tasks;
  50 using log4net;
  51
  52
  53 namespace Pithos.Network
  54 {
  55     using System;
  56     using System.Collections.Generic;
  57     using System.Linq;
  58     using System.Text;
  59
  60     /// <summary>
  61     /// TODO: Update summary.
  62     /// </summary>
  63     public class RestClient:WebClient
  64     {
  65         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  66
  67         public int Timeout { get; set; }
  68
  69         public bool TimedOut { get; set; }
  70
  71         public HttpStatusCode StatusCode { get; private set; }
  72
  73         public string StatusDescription { get; set; }
  74
  75         public long? RangeFrom { get; set; }
  76         public long? RangeTo { get; set; }
  77
  78         public int Retries { get; set; }
  79
  80         private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
  81         public Dictionary<string, string> Parameters
  82         {
  83             get
  84             {
  85                 Contract.Ensures(_parameters!=null);
  86                 return _parameters;
  87             }
  88         }
  89
  90
  91         [ContractInvariantMethod]
  92         private void Invariants()
  93         {
  94             Contract.Invariant(Headers!=null);
  95         }
  96
  97         public RestClient():base()
  98         {
  99
 100         }
 101
 102
 103         public RestClient(RestClient other)
 104             : base()
 105         {
 106             if (other==null)
 107                 throw new ArgumentNullException("other");
 108             Contract.EndContractBlock();
 109
 110             CopyHeaders(other);
 111             Timeout = other.Timeout;
 112             Retries = other.Retries;
 113             BaseAddress = other.BaseAddress;
 114
 115             foreach (var parameter in other.Parameters)
 116             {
 117                 Parameters.Add(parameter.Key,parameter.Value);
 118             }
 119
 120             this.Proxy = other.Proxy;
 121         }
 122
 123
 124         protected override WebRequest GetWebRequest(Uri address)
 125         {
 126             TimedOut = false;
 127             var webRequest = base.GetWebRequest(address);
 128             var request = (HttpWebRequest)webRequest;
 129             request.ServicePoint.ConnectionLimit = 50;
 130             if (IfModifiedSince.HasValue)
 131                 request.IfModifiedSince = IfModifiedSince.Value;
 132             request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
 133             if(Timeout>0)
 134                 request.Timeout = Timeout;
 135
 136             if (RangeFrom.HasValue)
 137             {
 138                 if (RangeTo.HasValue)
 139                     request.AddRange(RangeFrom.Value, RangeTo.Value);
 140                 else
 141                     request.AddRange(RangeFrom.Value);
 142             }
 143             return request;
 144         }
 145
 146         public DateTime? IfModifiedSince { get; set; }
 147
 148         //Asynchronous version
 149         protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
 150         {
 151             Log.InfoFormat("ASYNC [{0}] {1}",request.Method, request.RequestUri);
 152             HttpWebResponse response = null;
 153
 154             try
 155             {
 156                 response = (HttpWebResponse)base.GetWebResponse(request, result);
 157             }
 158             catch (WebException exc)
 159             {
 160                 if (!TryGetResponse(exc, out response))
 161                     throw;
 162             }
 163
 164             StatusCode = response.StatusCode;
 165             LastModified = response.LastModified;
 166             StatusDescription = response.StatusDescription;
 167             return response;
 168
 169         }
 170
 171
 172         //Synchronous version
 173         protected override WebResponse GetWebResponse(WebRequest request)
 174         {
 175             HttpWebResponse response = null;
 176             try
 177             {
 178                 response = (HttpWebResponse)base.GetWebResponse(request);
 179             }
 180             catch (WebException exc)
 181             {
 182                 if (!TryGetResponse(exc, out response))
 183                     throw;
 184             }
 185
 186             StatusCode = response.StatusCode;
 187             LastModified = response.LastModified;
 188             StatusDescription = response.StatusDescription;
 189             return response;
 190         }
 191
 192         private bool TryGetResponse(WebException exc, out HttpWebResponse response)
 193         {
 194             response = null;
 195             //Fail on empty response
 196             if (exc.Response == null)
 197                 return false;
 198
 199             response = (exc.Response as HttpWebResponse);
 200             //Succeed on allowed status codes
 201             if (AllowedStatusCodes.Contains(response.StatusCode))
 202                 return true;
 203
 204             //Does the response have any content to log?
 205             if (exc.Response.ContentLength > 0)
 206             {
 207                 var content = LogContent(exc.Response);
 208                 Log.ErrorFormat(content);
 209             }
 210             return false;
 211         }
 212
 213         private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
 214
 215         public List<HttpStatusCode> AllowedStatusCodes
 216         {
 217             get
 218             {
 219                 return _allowedStatusCodes;
 220             }
 221         }
 222
 223         public DateTime LastModified { get; private set; }
 224
 225         private static string LogContent(WebResponse webResponse)
 226         {
 227             if (webResponse == null)
 228                 throw new ArgumentNullException("webResponse");
 229             Contract.EndContractBlock();
 230
 231             //The response stream must be copied to avoid affecting other code by disposing of the
 232             //original response stream.
 233             var stream = webResponse.GetResponseStream();
 234             using(var memStream=new MemoryStream())
 235             using (var reader = new StreamReader(memStream))
 236             {
 237                 stream.CopyTo(memStream);
 238                 string content = reader.ReadToEnd();
 239
 240                 stream.Seek(0,SeekOrigin.Begin);
 241                 return content;
 242             }
 243         }
 244
 245         public string DownloadStringWithRetry(string address,int retries=0)
 246         {
 247
 248             if (address == null)
 249                 throw new ArgumentNullException("address");
 250
 251             var actualAddress = GetActualAddress(address);
 252
 253             TraceStart("GET",actualAddress);
 254
 255             var actualRetries = (retries == 0) ? Retries : retries;
 256
 257             var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
 258
 259             var task = Retry(() =>
 260             {
 261                 var content = base.DownloadString(uriString);
 262
 263                 if (StatusCode == HttpStatusCode.NoContent)
 264                     return String.Empty;
 265                 return content;
 266
 267             }, actualRetries);
 268
 269             try
 270             {
 271                 var result = task.Result;
 272                 return result;
 273
 274             }
 275             catch (AggregateException exc)
 276             {
 277                 //If the task fails, propagate the original exception
 278                 if (exc.InnerException!=null)
 279                     throw exc.InnerException;
 280                 throw;
 281             }
 282         }
 283
 284         public void Head(string address,int retries=0)
 285         {
 286             AllowedStatusCodes.Add(HttpStatusCode.NotFound);
 287             RetryWithoutContent(address, retries, "HEAD");
 288         }
 289
 290         public void PutWithRetry(string address, int retries = 0)
 291         {
 292             RetryWithoutContent(address, retries, "PUT");
 293         }
 294
 295         public void DeleteWithRetry(string address,int retries=0)
 296         {
 297             RetryWithoutContent(address, retries, "DELETE");
 298         }
 299
 300         public string GetHeaderValue(string headerName,bool optional=false)
 301         {
 302             if (this.ResponseHeaders==null)
 303                 throw new InvalidOperationException("ResponseHeaders are null");
 304             Contract.EndContractBlock();
 305
 306             var values=this.ResponseHeaders.GetValues(headerName);
 307             if (values != null)
 308                 return values[0];
 309
 310             if (optional)
 311                 return null;
 312             //A required header was not found
 313             throw new WebException(String.Format("The {0}  header is missing", headerName));
 314         }
 315
 316         public void SetNonEmptyHeaderValue(string headerName, string value)
 317         {
 318             if (String.IsNullOrWhiteSpace(value))
 319                 return;
 320             Headers.Add(headerName,value);
 321         }
 322
 323         private void RetryWithoutContent(string address, int retries, string method)
 324         {
 325             if (address == null)
 326                 throw new ArgumentNullException("address");
 327
 328             var actualAddress = GetActualAddress(address);
 329             var actualRetries = (retries == 0) ? Retries : retries;
 330
 331             var task = Retry(() =>
 332             {
 333                 var uriString = String.Join("/",BaseAddress ,actualAddress);
 334                 var uri = new Uri(uriString);
 335                 var request =  GetWebRequest(uri);
 336                 request.Method = method;
 337                 if (ResponseHeaders!=null)
 338                     ResponseHeaders.Clear();
 339
 340                 TraceStart(method, uriString);
 341                 if (method == "PUT")
 342                     request.ContentLength = 0;
 343
 344                 //Have to use try/finally instead of using here, because WebClient needs a valid WebResponse object
 345                 //in order to return response headers
 346                 var response = (HttpWebResponse)GetWebResponse(request);
 347                 try
 348                 {
 349                     LastModified = response.LastModified;
 350                     StatusCode = response.StatusCode;
 351                     StatusDescription = response.StatusDescription;
 352                 }
 353                 finally
 354                 {
 355                     response.Close();
 356                 }
 357
 358
 359                 return 0;
 360             }, actualRetries);
 361
 362             try
 363             {
 364                 task.Wait();
 365             }
 366             catch (AggregateException ex)
 367             {
 368                 var exc = ex.InnerException;
 369                 if (exc is RetryException)
 370                 {
 371                     Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
 372                 }
 373                 else
 374                 {
 375                     Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
 376                 }
 377                 throw exc;
 378
 379             }
 380             catch(Exception ex)
 381             {
 382                 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
 383                 throw;
 384             }
 385         }
 386
 387         private static void TraceStart(string method, string actualAddress)
 388         {
 389             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
 390         }
 391
 392         private string GetActualAddress(string address)
 393         {
 394             if (Parameters.Count == 0)
 395                 return address;
 396             var addressBuilder=new StringBuilder(address);
 397
 398             bool isFirst = true;
 399             foreach (var parameter in Parameters)
 400             {
 401                 if(isFirst)
 402                     addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
 403                 else
 404                     addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
 405                 isFirst = false;
 406             }
 407             return addressBuilder.ToString();
 408         }
 409
 410         public string DownloadStringWithRetry(Uri address,int retries=0)
 411         {
 412             if (address == null)
 413                 throw new ArgumentNullException("address");
 414
 415             var actualRetries = (retries == 0) ? Retries : retries;
 416             var task = Retry(() =>
 417             {
 418                 var content = base.DownloadString(address);
 419
 420                 if (StatusCode == HttpStatusCode.NoContent)
 421                     return String.Empty;
 422                 return content;
 423
 424             }, actualRetries);
 425
 426             var result = task.Result;
 427             return result;
 428         }
 429
 430
 431         /// <summary>
 432         /// Copies headers from another RestClient
 433         /// </summary>
 434         /// <param name="source">The RestClient from which the headers are copied</param>
 435         public void CopyHeaders(RestClient source)
 436         {
 437             if (source == null)
 438                 throw new ArgumentNullException("source", "source can't be null");
 439             Contract.EndContractBlock();
 440             //The Headers getter initializes the property, it is never null
 441             Contract.Assume(Headers!=null);
 442
 443             CopyHeaders(source.Headers,Headers);
 444         }
 445
 446         /// <summary>
 447         /// Copies headers from one header collection to another
 448         /// </summary>
 449         /// <param name="source">The source collection from which the headers are copied</param>
 450         /// <param name="target">The target collection to which the headers are copied</param>
 451         public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
 452         {
 453             if (source == null)
 454                 throw new ArgumentNullException("source", "source can't be null");
 455             if (target == null)
 456                 throw new ArgumentNullException("target", "target can't be null");
 457             Contract.EndContractBlock();
 458
 459             for (int i = 0; i < source.Count; i++)
 460             {
 461                 target.Add(source.GetKey(i), source[i]);
 462             }
 463         }
 464
 465         public void AssertStatusOK(string message)
 466         {
 467             if (StatusCode >= HttpStatusCode.BadRequest)
 468                 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
 469         }
 470
 471
 472         private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
 473         {
 474             if (original==null)
 475                 throw new ArgumentNullException("original");
 476             Contract.EndContractBlock();
 477
 478             if (tcs == null)
 479                 tcs = new TaskCompletionSource<T>();
 480             Task.Factory.StartNew(original).ContinueWith(_original =>
 481                 {
 482                     if (!_original.IsFaulted)
 483                         tcs.SetFromTask(_original);
 484                     else
 485                     {
 486                         var e = _original.Exception.InnerException;
 487                         var we = (e as WebException);
 488                         if (we==null)
 489                             tcs.SetException(e);
 490                         else
 491                         {
 492                             var statusCode = GetStatusCode(we);
 493
 494                             //Return null for 404
 495                             if (statusCode == HttpStatusCode.NotFound)
 496                                 tcs.SetResult(default(T));
 497                             //Retry for timeouts and service unavailable
 498                             else if (we.Status == WebExceptionStatus.Timeout ||
 499                                 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
 500                             {
 501                                 TimedOut = true;
 502                                 if (retryCount == 0)
 503                                 {
 504                                     Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
 505                                     tcs.SetException(new RetryException("Timed out too many times.", e));
 506                                 }
 507                                 else
 508                                 {
 509                                     Log.ErrorFormat(
 510                                         "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
 511                                         retryCount, e);
 512                                     Retry(original, retryCount - 1, tcs);
 513                                 }
 514                             }
 515                             else
 516                                 tcs.SetException(e);
 517                         }
 518                     };
 519                 });
 520             return tcs.Task;
 521         }
 522
 523         private HttpStatusCode GetStatusCode(WebException we)
 524         {
 525             if (we==null)
 526                 throw new ArgumentNullException("we");
 527             var statusCode = HttpStatusCode.RequestTimeout;
 528             if (we.Response != null)
 529             {
 530                 statusCode = ((HttpWebResponse) we.Response).StatusCode;
 531                 this.StatusCode = statusCode;
 532             }
 533             return statusCode;
 534         }
 535
 536         public UriBuilder GetAddressBuilder(string container, string objectName)
 537         {
 538             var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
 539             return builder;
 540         }
 541
 542         public Dictionary<string, string> GetMeta(string metaPrefix)
 543         {
 544             if (String.IsNullOrWhiteSpace(metaPrefix))
 545                 throw new ArgumentNullException("metaPrefix");
 546             Contract.EndContractBlock();
 547
 548             var keys = ResponseHeaders.AllKeys.AsQueryable();
 549             var dict = (from key in keys
 550                         where key.StartsWith(metaPrefix)
 551                         let name = key.Substring(metaPrefix.Length)
 552                         select new { Name = name, Value = ResponseHeaders[key] })
 553                         .ToDictionary(t => t.Name, t => t.Value);
 554             return dict;
 555         }
 556     }
 557
 558     public class RetryException:Exception
 559     {
 560         public RetryException()
 561             :base()
 562         {
 563
 564         }
 565
 566         public RetryException(string message)
 567             :base(message)
 568         {
 569
 570         }
 571
 572         public RetryException(string message,Exception innerException)
 573             :base(message,innerException)
 574         {
 575
 576         }
 577
 578         public RetryException(SerializationInfo info,StreamingContext context)
 579             :base(info,context)
 580         {
 581
 582         }
 583     }
 584 }