code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/RestClient.cs

   1 // -----------------------------------------------------------------------
   2 // <copyright file="RestClient.cs" company="Microsoft">
   3 // TODO: Update copyright text.
   4 // </copyright>
   5 // -----------------------------------------------------------------------
   6
   7 using System.Collections.Specialized;
   8 using System.Diagnostics;
   9 using System.Diagnostics.Contracts;
  10 using System.IO;
  11 using System.Net;
  12 using System.Runtime.Serialization;
  13 using System.Threading.Tasks;
  14 using log4net;
  15
  16
  17 namespace Pithos.Network
  18 {
  19     using System;
  20     using System.Collections.Generic;
  21     using System.Linq;
  22     using System.Text;
  23
  24     /// <summary>
  25     /// TODO: Update summary.
  26     /// </summary>
  27     public class RestClient:WebClient
  28     {
  29         public int Timeout { get; set; }
  30
  31         public bool TimedOut { get; set; }
  32
  33         public HttpStatusCode StatusCode { get; private set; }
  34
  35         public string StatusDescription { get; set; }
  36
  37         public long? RangeFrom { get; set; }
  38         public long? RangeTo { get; set; }
  39
  40         public int Retries { get; set; }
  41
  42         private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
  43         public Dictionary<string, string> Parameters
  44         {
  45             get
  46             {
  47                 Contract.Ensures(_parameters!=null);
  48                 return _parameters;
  49             }
  50         }
  51
  52         private static readonly ILog Log = LogManager.GetLogger("RestClient");
  53
  54
  55         [ContractInvariantMethod]
  56         private void Invariants()
  57         {
  58             Contract.Invariant(Headers!=null);
  59         }
  60
  61         public RestClient():base()
  62         {
  63
  64         }
  65
  66
  67         public RestClient(RestClient other)
  68             : base()
  69         {
  70             if (other==null)
  71                 throw new ArgumentNullException("other");
  72             Contract.EndContractBlock();
  73
  74             CopyHeaders(other);
  75             Timeout = other.Timeout;
  76             Retries = other.Retries;
  77             BaseAddress = other.BaseAddress;
  78
  79             foreach (var parameter in other.Parameters)
  80             {
  81                 Parameters.Add(parameter.Key,parameter.Value);
  82             }
  83
  84             this.Proxy = other.Proxy;
  85         }
  86
  87         protected override WebRequest GetWebRequest(Uri address)
  88         {
  89             TimedOut = false;
  90             var webRequest = base.GetWebRequest(address);
  91             var request = (HttpWebRequest)webRequest;
  92             if (IfModifiedSince.HasValue)
  93                 request.IfModifiedSince = IfModifiedSince.Value;
  94             request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
  95             if(Timeout>0)
  96                 request.Timeout = Timeout;
  97
  98             if (RangeFrom.HasValue)
  99             {
 100                 if (RangeTo.HasValue)
 101                     request.AddRange(RangeFrom.Value, RangeTo.Value);
 102                 else
 103                     request.AddRange(RangeFrom.Value);
 104             }
 105             return request;
 106         }
 107
 108         public DateTime? IfModifiedSince { get; set; }
 109
 110         protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
 111         {
 112             return ProcessResponse(()=>base.GetWebResponse(request, result));
 113         }
 114
 115         protected override WebResponse GetWebResponse(WebRequest request)
 116         {
 117             return ProcessResponse(() => base.GetWebResponse(request));
 118         }
 119
 120         private WebResponse ProcessResponse(Func<WebResponse> getResponse)
 121         {
 122             try
 123             {
 124                 var response = (HttpWebResponse)getResponse();
 125                 StatusCode = response.StatusCode;
 126                 LastModified = response.LastModified;
 127                 StatusDescription = response.StatusDescription;
 128                 return response;
 129             }
 130             catch (WebException exc)
 131             {
 132                 if (exc.Response != null)
 133                 {
 134                     var response = (exc.Response as HttpWebResponse);
 135                     if (AllowedStatusCodes.Contains(response.StatusCode))
 136                     {
 137                         StatusCode = response.StatusCode;
 138                         LastModified = response.LastModified;
 139                         StatusDescription = response.StatusDescription;
 140
 141                         return response;
 142                     }
 143                     if (exc.Response.ContentLength > 0)
 144                     {
 145                         string content = GetContent(exc.Response);
 146                         Log.ErrorFormat(content);
 147                     }
 148                 }
 149                 throw;
 150             }
 151         }
 152
 153         private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
 154         public List<HttpStatusCode> AllowedStatusCodes
 155         {
 156             get
 157             {
 158                 return _allowedStatusCodes;
 159             }
 160         }
 161
 162         public DateTime LastModified { get; private set; }
 163
 164         private static string GetContent(WebResponse webResponse)
 165         {
 166             if (webResponse == null)
 167                 throw new ArgumentNullException("webResponse");
 168             Contract.EndContractBlock();
 169
 170             string content;
 171             using (var stream = webResponse.GetResponseStream())
 172             using (var reader = new StreamReader(stream))
 173             {
 174                 content = reader.ReadToEnd();
 175             }
 176             return content;
 177         }
 178
 179         public string DownloadStringWithRetry(string address,int retries=0)
 180         {
 181
 182             if (address == null)
 183                 throw new ArgumentNullException("address");
 184
 185             var actualAddress = GetActualAddress(address);
 186
 187             TraceStart("GET",actualAddress);
 188
 189             var actualRetries = (retries == 0) ? Retries : retries;
 190
 191
 192             var task = Retry(() =>
 193             {
 194                 var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
 195                 var content = base.DownloadString(uriString);
 196
 197                 if (StatusCode == HttpStatusCode.NoContent)
 198                     return String.Empty;
 199                 return content;
 200
 201             }, actualRetries);
 202
 203             var result = task.Result;
 204             return result;
 205         }
 206
 207         public void Head(string address,int retries=0)
 208         {
 209             AllowedStatusCodes.Add(HttpStatusCode.NotFound);
 210             RetryWithoutContent(address, retries, "HEAD");
 211         }
 212
 213         public void PutWithRetry(string address, int retries = 0)
 214         {
 215             RetryWithoutContent(address, retries, "PUT");
 216         }
 217
 218         public void DeleteWithRetry(string address,int retries=0)
 219         {
 220             RetryWithoutContent(address, retries, "DELETE");
 221         }
 222
 223         public string GetHeaderValue(string headerName,bool optional=false)
 224         {
 225             if (this.ResponseHeaders==null)
 226                 throw new InvalidOperationException("ResponseHeaders are null");
 227             Contract.EndContractBlock();
 228
 229             var values=this.ResponseHeaders.GetValues(headerName);
 230             if (values != null)
 231                 return values[0];
 232
 233             if (optional)
 234                 return null;
 235             //A required header was not found
 236             throw new WebException(String.Format("The {0}  header is missing", headerName));
 237         }
 238
 239         public void SetNonEmptyHeaderValue(string headerName, string value)
 240         {
 241             if (String.IsNullOrWhiteSpace(value))
 242                 return;
 243             Headers.Add(headerName,value);
 244         }
 245
 246         private void RetryWithoutContent(string address, int retries, string method)
 247         {
 248             if (address == null)
 249                 throw new ArgumentNullException("address");
 250
 251             var actualAddress = GetActualAddress(address);
 252             var actualRetries = (retries == 0) ? Retries : retries;
 253
 254             var task = Retry(() =>
 255             {
 256                 var uriString = String.Join("/",BaseAddress ,actualAddress);
 257                 var uri = new Uri(uriString);
 258                 var request =  GetWebRequest(uri);
 259                 request.Method = method;
 260                 if (ResponseHeaders!=null)
 261                     ResponseHeaders.Clear();
 262
 263                 TraceStart(method, uriString);
 264                 if (method == "PUT")
 265                     request.ContentLength = 0;
 266                 var response = (HttpWebResponse)GetWebResponse(request);
 267                 StatusCode = response.StatusCode;
 268                 StatusDescription = response.StatusDescription;
 269
 270
 271                 return 0;
 272             }, actualRetries);
 273
 274             try
 275             {
 276                 task.Wait();
 277             }
 278             catch (AggregateException ex)
 279             {
 280                 var exc = ex.InnerException;
 281                 if (exc is RetryException)
 282                 {
 283                     Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
 284                 }
 285                 else
 286                 {
 287                     Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
 288                 }
 289                 throw exc;
 290
 291             }
 292             catch(Exception ex)
 293             {
 294                 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
 295                 throw;
 296             }
 297         }
 298
 299         private static void TraceStart(string method, string actualAddress)
 300         {
 301             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
 302         }
 303
 304         private string GetActualAddress(string address)
 305         {
 306             if (Parameters.Count == 0)
 307                 return address;
 308             var addressBuilder=new StringBuilder(address);
 309
 310             bool isFirst = true;
 311             foreach (var parameter in Parameters)
 312             {
 313                 if(isFirst)
 314                     addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
 315                 else
 316                     addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
 317                 isFirst = false;
 318             }
 319             return addressBuilder.ToString();
 320         }
 321
 322         public string DownloadStringWithRetry(Uri address,int retries=0)
 323         {
 324             if (address == null)
 325                 throw new ArgumentNullException("address");
 326
 327             var actualRetries = (retries == 0) ? Retries : retries;
 328             var task = Retry(() =>
 329             {
 330                 var content = base.DownloadString(address);
 331
 332                 if (StatusCode == HttpStatusCode.NoContent)
 333                     return String.Empty;
 334                 return content;
 335
 336             }, actualRetries);
 337
 338             var result = task.Result;
 339             return result;
 340         }
 341
 342
 343         /// <summary>
 344         /// Copies headers from another RestClient
 345         /// </summary>
 346         /// <param name="source">The RestClient from which the headers are copied</param>
 347         public void CopyHeaders(RestClient source)
 348         {
 349             if (source == null)
 350                 throw new ArgumentNullException("source", "source can't be null");
 351             Contract.EndContractBlock();
 352             //The Headers getter initializes the property, it is never null
 353             Contract.Assume(Headers!=null);
 354
 355             CopyHeaders(source.Headers,Headers);
 356         }
 357
 358         /// <summary>
 359         /// Copies headers from one header collection to another
 360         /// </summary>
 361         /// <param name="source">The source collection from which the headers are copied</param>
 362         /// <param name="target">The target collection to which the headers are copied</param>
 363         public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
 364         {
 365             if (source == null)
 366                 throw new ArgumentNullException("source", "source can't be null");
 367             if (target == null)
 368                 throw new ArgumentNullException("target", "target can't be null");
 369             Contract.EndContractBlock();
 370
 371             for (int i = 0; i < source.Count; i++)
 372             {
 373                 target.Add(source.GetKey(i), source[i]);
 374             }
 375         }
 376
 377         public void AssertStatusOK(string message)
 378         {
 379             if (StatusCode >= HttpStatusCode.BadRequest)
 380                 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
 381         }
 382
 383
 384         private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
 385         {
 386             if (original==null)
 387                 throw new ArgumentNullException("original");
 388             Contract.EndContractBlock();
 389
 390             if (tcs == null)
 391                 tcs = new TaskCompletionSource<T>();
 392             Task.Factory.StartNew(original).ContinueWith(_original =>
 393                 {
 394                     if (!_original.IsFaulted)
 395                         tcs.SetFromTask(_original);
 396                     else
 397                     {
 398                         var e = _original.Exception.InnerException;
 399                         var we = (e as WebException);
 400                         if (we==null)
 401                             tcs.SetException(e);
 402                         else
 403                         {
 404                             var statusCode = GetStatusCode(we);
 405
 406                             //Return null for 404
 407                             if (statusCode == HttpStatusCode.NotFound)
 408                                 tcs.SetResult(default(T));
 409                             //Retry for timeouts and service unavailable
 410                             else if (we.Status == WebExceptionStatus.Timeout ||
 411                                 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
 412                             {
 413                                 TimedOut = true;
 414                                 if (retryCount == 0)
 415                                 {
 416                                     Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
 417                                     tcs.SetException(new RetryException("Timed out too many times.", e));
 418                                 }
 419                                 else
 420                                 {
 421                                     Log.ErrorFormat(
 422                                         "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
 423                                         retryCount, e);
 424                                     Retry(original, retryCount - 1, tcs);
 425                                 }
 426                             }
 427                             else
 428                                 tcs.SetException(e);
 429                         }
 430                     };
 431                 });
 432             return tcs.Task;
 433         }
 434
 435         private HttpStatusCode GetStatusCode(WebException we)
 436         {
 437             if (we==null)
 438                 throw new ArgumentNullException("we");
 439             var statusCode = HttpStatusCode.RequestTimeout;
 440             if (we.Response != null)
 441             {
 442                 statusCode = ((HttpWebResponse) we.Response).StatusCode;
 443                 this.StatusCode = statusCode;
 444             }
 445             return statusCode;
 446         }
 447
 448         public UriBuilder GetAddressBuilder(string container, string objectName)
 449         {
 450             var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
 451             return builder;
 452         }
 453     }
 454
 455     public class RetryException:Exception
 456     {
 457         public RetryException()
 458             :base()
 459         {
 460
 461         }
 462
 463         public RetryException(string message)
 464             :base(message)
 465         {
 466
 467         }
 468
 469         public RetryException(string message,Exception innerException)
 470             :base(message,innerException)
 471         {
 472
 473         }
 474
 475         public RetryException(SerializationInfo info,StreamingContext context)
 476             :base(info,context)
 477         {
 478
 479         }
 480     }
 481 }