code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/RestClient.cs

   1 // -----------------------------------------------------------------------
   2 // <copyright file="RestClient.cs" company="GRNet">
   3 // Copyright 2011 GRNET S.A. All rights reserved.
   4 //
   5 // Redistribution and use in source and binary forms, with or
   6 // without modification, are permitted provided that the following
   7 // conditions are met:
   8 //
   9 //   1. Redistributions of source code must retain the above
  10 //      copyright notice, this list of conditions and the following
  11 //      disclaimer.
  12 //
  13 //   2. Redistributions in binary form must reproduce the above
  14 //      copyright notice, this list of conditions and the following
  15 //      disclaimer in the documentation and/or other materials
  16 //      provided with the distribution.
  17 //
  18 // THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  19 // OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  21 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  22 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  25 // USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  26 // AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27 // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  28 // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29 // POSSIBILITY OF SUCH DAMAGE.
  30 //
  31 // The views and conclusions contained in the software and
  32 // documentation are those of the authors and should not be
  33 // interpreted as representing official policies, either expressed
  34 // or implied, of GRNET S.A.
  35 // </copyright>
  36 // -----------------------------------------------------------------------
  37
  38 using System.Collections.Specialized;
  39 using System.Diagnostics;
  40 using System.Diagnostics.Contracts;
  41 using System.IO;
  42 using System.Net;
  43 using System.Runtime.Serialization;
  44 using System.Threading.Tasks;
  45 using log4net;
  46
  47
  48 namespace Pithos.Network
  49 {
  50     using System;
  51     using System.Collections.Generic;
  52     using System.Linq;
  53     using System.Text;
  54
  55     /// <summary>
  56     /// TODO: Update summary.
  57     /// </summary>
  58     public class RestClient:WebClient
  59     {
  60         public int Timeout { get; set; }
  61
  62         public bool TimedOut { get; set; }
  63
  64         public HttpStatusCode StatusCode { get; private set; }
  65
  66         public string StatusDescription { get; set; }
  67
  68         public long? RangeFrom { get; set; }
  69         public long? RangeTo { get; set; }
  70
  71         public int Retries { get; set; }
  72
  73         private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
  74         public Dictionary<string, string> Parameters
  75         {
  76             get
  77             {
  78                 Contract.Ensures(_parameters!=null);
  79                 return _parameters;
  80             }
  81         }
  82
  83         private static readonly ILog Log = LogManager.GetLogger("RestClient");
  84
  85
  86         [ContractInvariantMethod]
  87         private void Invariants()
  88         {
  89             Contract.Invariant(Headers!=null);
  90         }
  91
  92         public RestClient():base()
  93         {
  94
  95         }
  96
  97
  98         public RestClient(RestClient other)
  99             : base()
 100         {
 101             if (other==null)
 102                 throw new ArgumentNullException("other");
 103             Contract.EndContractBlock();
 104
 105             CopyHeaders(other);
 106             Timeout = other.Timeout;
 107             Retries = other.Retries;
 108             BaseAddress = other.BaseAddress;
 109
 110             foreach (var parameter in other.Parameters)
 111             {
 112                 Parameters.Add(parameter.Key,parameter.Value);
 113             }
 114
 115             this.Proxy = other.Proxy;
 116         }
 117
 118
 119         protected override WebRequest GetWebRequest(Uri address)
 120         {
 121             TimedOut = false;
 122             var webRequest = base.GetWebRequest(address);
 123             var request = (HttpWebRequest)webRequest;
 124             request.ServicePoint.ConnectionLimit = 10;
 125             if (IfModifiedSince.HasValue)
 126                 request.IfModifiedSince = IfModifiedSince.Value;
 127             request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
 128             if(Timeout>0)
 129                 request.Timeout = Timeout;
 130
 131             if (RangeFrom.HasValue)
 132             {
 133                 if (RangeTo.HasValue)
 134                     request.AddRange(RangeFrom.Value, RangeTo.Value);
 135                 else
 136                     request.AddRange(RangeFrom.Value);
 137             }
 138             return request;
 139         }
 140
 141         public DateTime? IfModifiedSince { get; set; }
 142
 143         //Asynchronous version
 144         protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
 145         {
 146             Log.InfoFormat("ASYNC [{0}] {1}",request.Method, request.RequestUri);
 147             HttpWebResponse response = null;
 148
 149             try
 150             {
 151                 response = (HttpWebResponse)base.GetWebResponse(request, result);
 152             }
 153             catch (WebException exc)
 154             {
 155                 if (!TryGetResponse(exc, out response))
 156                     throw;
 157             }
 158
 159             StatusCode = response.StatusCode;
 160             LastModified = response.LastModified;
 161             StatusDescription = response.StatusDescription;
 162             return response;
 163
 164         }
 165
 166
 167         //Synchronous version
 168         protected override WebResponse GetWebResponse(WebRequest request)
 169         {
 170             HttpWebResponse response = null;
 171             try
 172             {
 173                 response = (HttpWebResponse)base.GetWebResponse(request);
 174             }
 175             catch (WebException exc)
 176             {
 177                 if (!TryGetResponse(exc, out response))
 178                     throw;
 179             }
 180
 181             StatusCode = response.StatusCode;
 182             LastModified = response.LastModified;
 183             StatusDescription = response.StatusDescription;
 184             return response;
 185         }
 186
 187         private bool TryGetResponse(WebException exc, out HttpWebResponse response)
 188         {
 189             response = null;
 190             //Fail on empty response
 191             if (exc.Response == null)
 192                 return false;
 193
 194             response = (exc.Response as HttpWebResponse);
 195             //Succeed on allowed status codes
 196             if (AllowedStatusCodes.Contains(response.StatusCode))
 197                 return true;
 198
 199             //Does the response have any content to log?
 200             if (exc.Response.ContentLength > 0)
 201             {
 202                 var content = LogContent(exc.Response);
 203                 Log.ErrorFormat(content);
 204             }
 205             return false;
 206         }
 207
 208         private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
 209
 210         public List<HttpStatusCode> AllowedStatusCodes
 211         {
 212             get
 213             {
 214                 return _allowedStatusCodes;
 215             }
 216         }
 217
 218         public DateTime LastModified { get; private set; }
 219
 220         private static string LogContent(WebResponse webResponse)
 221         {
 222             if (webResponse == null)
 223                 throw new ArgumentNullException("webResponse");
 224             Contract.EndContractBlock();
 225
 226             //The response stream must be copied to avoid affecting other code by disposing of the
 227             //original response stream.
 228             var stream = webResponse.GetResponseStream();
 229             using(var memStream=new MemoryStream((int) stream.Length))
 230             using (var reader = new StreamReader(memStream))
 231             {
 232                 stream.CopyTo(memStream);
 233                 string content = reader.ReadToEnd();
 234
 235                 stream.Seek(0,SeekOrigin.Begin);
 236                 return content;
 237             }
 238         }
 239
 240         public string DownloadStringWithRetry(string address,int retries=0)
 241         {
 242
 243             if (address == null)
 244                 throw new ArgumentNullException("address");
 245
 246             var actualAddress = GetActualAddress(address);
 247
 248             TraceStart("GET",actualAddress);
 249
 250             var actualRetries = (retries == 0) ? Retries : retries;
 251
 252             var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
 253
 254             var task = Retry(() =>
 255             {
 256                 var content = base.DownloadString(uriString);
 257
 258                 if (StatusCode == HttpStatusCode.NoContent)
 259                     return String.Empty;
 260                 return content;
 261
 262             }, actualRetries);
 263
 264             var result = task.Result;
 265             return result;
 266         }
 267
 268         public void Head(string address,int retries=0)
 269         {
 270             AllowedStatusCodes.Add(HttpStatusCode.NotFound);
 271             RetryWithoutContent(address, retries, "HEAD");
 272         }
 273
 274         public void PutWithRetry(string address, int retries = 0)
 275         {
 276             RetryWithoutContent(address, retries, "PUT");
 277         }
 278
 279         public void DeleteWithRetry(string address,int retries=0)
 280         {
 281             RetryWithoutContent(address, retries, "DELETE");
 282         }
 283
 284         public string GetHeaderValue(string headerName,bool optional=false)
 285         {
 286             if (this.ResponseHeaders==null)
 287                 throw new InvalidOperationException("ResponseHeaders are null");
 288             Contract.EndContractBlock();
 289
 290             var values=this.ResponseHeaders.GetValues(headerName);
 291             if (values != null)
 292                 return values[0];
 293
 294             if (optional)
 295                 return null;
 296             //A required header was not found
 297             throw new WebException(String.Format("The {0}  header is missing", headerName));
 298         }
 299
 300         public void SetNonEmptyHeaderValue(string headerName, string value)
 301         {
 302             if (String.IsNullOrWhiteSpace(value))
 303                 return;
 304             Headers.Add(headerName,value);
 305         }
 306
 307         private void RetryWithoutContent(string address, int retries, string method)
 308         {
 309             if (address == null)
 310                 throw new ArgumentNullException("address");
 311
 312             var actualAddress = GetActualAddress(address);
 313             var actualRetries = (retries == 0) ? Retries : retries;
 314
 315             var task = Retry(() =>
 316             {
 317                 var uriString = String.Join("/",BaseAddress ,actualAddress);
 318                 var uri = new Uri(uriString);
 319                 var request =  GetWebRequest(uri);
 320                 request.Method = method;
 321                 if (ResponseHeaders!=null)
 322                     ResponseHeaders.Clear();
 323
 324                 TraceStart(method, uriString);
 325                 if (method == "PUT")
 326                     request.ContentLength = 0;
 327
 328                 //Have to use try/finally instead of using here, because WebClient needs a valid WebResponse object
 329                 //in order to return response headers
 330                 var response = (HttpWebResponse)GetWebResponse(request);
 331                 try
 332                 {
 333                     LastModified = response.LastModified;
 334                     StatusCode = response.StatusCode;
 335                     StatusDescription = response.StatusDescription;
 336                 }
 337                 finally
 338                 {
 339                     response.Close();
 340                 }
 341
 342
 343                 return 0;
 344             }, actualRetries);
 345
 346             try
 347             {
 348                 task.Wait();
 349             }
 350             catch (AggregateException ex)
 351             {
 352                 var exc = ex.InnerException;
 353                 if (exc is RetryException)
 354                 {
 355                     Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
 356                 }
 357                 else
 358                 {
 359                     Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
 360                 }
 361                 throw exc;
 362
 363             }
 364             catch(Exception ex)
 365             {
 366                 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
 367                 throw;
 368             }
 369         }
 370
 371         private static void TraceStart(string method, string actualAddress)
 372         {
 373             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
 374         }
 375
 376         private string GetActualAddress(string address)
 377         {
 378             if (Parameters.Count == 0)
 379                 return address;
 380             var addressBuilder=new StringBuilder(address);
 381
 382             bool isFirst = true;
 383             foreach (var parameter in Parameters)
 384             {
 385                 if(isFirst)
 386                     addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
 387                 else
 388                     addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
 389                 isFirst = false;
 390             }
 391             return addressBuilder.ToString();
 392         }
 393
 394         public string DownloadStringWithRetry(Uri address,int retries=0)
 395         {
 396             if (address == null)
 397                 throw new ArgumentNullException("address");
 398
 399             var actualRetries = (retries == 0) ? Retries : retries;
 400             var task = Retry(() =>
 401             {
 402                 var content = base.DownloadString(address);
 403
 404                 if (StatusCode == HttpStatusCode.NoContent)
 405                     return String.Empty;
 406                 return content;
 407
 408             }, actualRetries);
 409
 410             var result = task.Result;
 411             return result;
 412         }
 413
 414
 415         /// <summary>
 416         /// Copies headers from another RestClient
 417         /// </summary>
 418         /// <param name="source">The RestClient from which the headers are copied</param>
 419         public void CopyHeaders(RestClient source)
 420         {
 421             if (source == null)
 422                 throw new ArgumentNullException("source", "source can't be null");
 423             Contract.EndContractBlock();
 424             //The Headers getter initializes the property, it is never null
 425             Contract.Assume(Headers!=null);
 426
 427             CopyHeaders(source.Headers,Headers);
 428         }
 429
 430         /// <summary>
 431         /// Copies headers from one header collection to another
 432         /// </summary>
 433         /// <param name="source">The source collection from which the headers are copied</param>
 434         /// <param name="target">The target collection to which the headers are copied</param>
 435         public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
 436         {
 437             if (source == null)
 438                 throw new ArgumentNullException("source", "source can't be null");
 439             if (target == null)
 440                 throw new ArgumentNullException("target", "target can't be null");
 441             Contract.EndContractBlock();
 442
 443             for (int i = 0; i < source.Count; i++)
 444             {
 445                 target.Add(source.GetKey(i), source[i]);
 446             }
 447         }
 448
 449         public void AssertStatusOK(string message)
 450         {
 451             if (StatusCode >= HttpStatusCode.BadRequest)
 452                 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
 453         }
 454
 455
 456         private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
 457         {
 458             if (original==null)
 459                 throw new ArgumentNullException("original");
 460             Contract.EndContractBlock();
 461
 462             if (tcs == null)
 463                 tcs = new TaskCompletionSource<T>();
 464             Task.Factory.StartNew(original).ContinueWith(_original =>
 465                 {
 466                     if (!_original.IsFaulted)
 467                         tcs.SetFromTask(_original);
 468                     else
 469                     {
 470                         var e = _original.Exception.InnerException;
 471                         var we = (e as WebException);
 472                         if (we==null)
 473                             tcs.SetException(e);
 474                         else
 475                         {
 476                             var statusCode = GetStatusCode(we);
 477
 478                             //Return null for 404
 479                             if (statusCode == HttpStatusCode.NotFound)
 480                                 tcs.SetResult(default(T));
 481                             //Retry for timeouts and service unavailable
 482                             else if (we.Status == WebExceptionStatus.Timeout ||
 483                                 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
 484                             {
 485                                 TimedOut = true;
 486                                 if (retryCount == 0)
 487                                 {
 488                                     Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
 489                                     tcs.SetException(new RetryException("Timed out too many times.", e));
 490                                 }
 491                                 else
 492                                 {
 493                                     Log.ErrorFormat(
 494                                         "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
 495                                         retryCount, e);
 496                                     Retry(original, retryCount - 1, tcs);
 497                                 }
 498                             }
 499                             else
 500                                 tcs.SetException(e);
 501                         }
 502                     };
 503                 });
 504             return tcs.Task;
 505         }
 506
 507         private HttpStatusCode GetStatusCode(WebException we)
 508         {
 509             if (we==null)
 510                 throw new ArgumentNullException("we");
 511             var statusCode = HttpStatusCode.RequestTimeout;
 512             if (we.Response != null)
 513             {
 514                 statusCode = ((HttpWebResponse) we.Response).StatusCode;
 515                 this.StatusCode = statusCode;
 516             }
 517             return statusCode;
 518         }
 519
 520         public UriBuilder GetAddressBuilder(string container, string objectName)
 521         {
 522             var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
 523             return builder;
 524         }
 525
 526         public Dictionary<string, string> GetMeta(string metaPrefix)
 527         {
 528             if (String.IsNullOrWhiteSpace(metaPrefix))
 529                 throw new ArgumentNullException("metaPrefix");
 530             Contract.EndContractBlock();
 531
 532             var keys = ResponseHeaders.AllKeys.AsQueryable();
 533             var dict = (from key in keys
 534                         where key.StartsWith(metaPrefix)
 535                         let name = key.Substring(metaPrefix.Length)
 536                         select new { Name = name, Value = ResponseHeaders[key] })
 537                         .ToDictionary(t => t.Name, t => t.Value);
 538             return dict;
 539         }
 540     }
 541
 542     public class RetryException:Exception
 543     {
 544         public RetryException()
 545             :base()
 546         {
 547
 548         }
 549
 550         public RetryException(string message)
 551             :base(message)
 552         {
 553
 554         }
 555
 556         public RetryException(string message,Exception innerException)
 557             :base(message,innerException)
 558         {
 559
 560         }
 561
 562         public RetryException(SerializationInfo info,StreamingContext context)
 563             :base(info,context)
 564         {
 565
 566         }
 567     }
 568 }