code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/RestClient.cs

   1 // -----------------------------------------------------------------------
   2 // <copyright file="RestClient.cs" company="Microsoft">
   3 // TODO: Update copyright text.
   4 // </copyright>
   5 // -----------------------------------------------------------------------
   6
   7 using System.Collections.Specialized;
   8 using System.Diagnostics;
   9 using System.Diagnostics.Contracts;
  10 using System.IO;
  11 using System.Net;
  12 using System.Runtime.Serialization;
  13 using System.Threading.Tasks;
  14 using log4net;
  15
  16
  17 namespace Pithos.Network
  18 {
  19     using System;
  20     using System.Collections.Generic;
  21     using System.Linq;
  22     using System.Text;
  23
  24     /// <summary>
  25     /// TODO: Update summary.
  26     /// </summary>
  27     public class RestClient:WebClient
  28     {
  29         public int Timeout { get; set; }
  30
  31         public bool TimedOut { get; set; }
  32
  33         public HttpStatusCode StatusCode { get; private set; }
  34
  35         public string StatusDescription { get; set; }
  36
  37         public long? RangeFrom { get; set; }
  38         public long? RangeTo { get; set; }
  39
  40         public int Retries { get; set; }
  41
  42         private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
  43         public Dictionary<string, string> Parameters
  44         {
  45             get
  46             {
  47                 Contract.Ensures(_parameters!=null);
  48                 return _parameters;
  49             }
  50         }
  51
  52         private static readonly ILog Log = LogManager.GetLogger("RestClient");
  53
  54
  55         [ContractInvariantMethod]
  56         private void Invariants()
  57         {
  58             Contract.Invariant(Headers!=null);
  59         }
  60
  61         public RestClient():base()
  62         {
  63
  64         }
  65
  66
  67         public RestClient(RestClient other)
  68             : base()
  69         {
  70             if (other==null)
  71                 throw new ArgumentNullException("other");
  72             Contract.EndContractBlock();
  73
  74             CopyHeaders(other);
  75             Timeout = other.Timeout;
  76             Retries = other.Retries;
  77             BaseAddress = other.BaseAddress;
  78
  79             foreach (var parameter in other.Parameters)
  80             {
  81                 Parameters.Add(parameter.Key,parameter.Value);
  82             }
  83
  84             this.Proxy = other.Proxy;
  85         }
  86
  87
  88         protected override WebRequest GetWebRequest(Uri address)
  89         {
  90             TimedOut = false;
  91             var webRequest = base.GetWebRequest(address);
  92             var request = (HttpWebRequest)webRequest;
  93             if (IfModifiedSince.HasValue)
  94                 request.IfModifiedSince = IfModifiedSince.Value;
  95             request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
  96             if(Timeout>0)
  97                 request.Timeout = Timeout;
  98
  99             if (RangeFrom.HasValue)
 100             {
 101                 if (RangeTo.HasValue)
 102                     request.AddRange(RangeFrom.Value, RangeTo.Value);
 103                 else
 104                     request.AddRange(RangeFrom.Value);
 105             }
 106             return request;
 107         }
 108
 109         public DateTime? IfModifiedSince { get; set; }
 110
 111         //Asynchronous version
 112         protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
 113         {
 114             Log.InfoFormat("ASYNC [{0}] {1}",request.Method, request.RequestUri);
 115             HttpWebResponse response = null;
 116
 117             try
 118             {
 119                 response = (HttpWebResponse)base.GetWebResponse(request, result);
 120             }
 121             catch (WebException exc)
 122             {
 123                 if (!TryGetResponse(exc, out response))
 124                     throw;
 125             }
 126
 127             StatusCode = response.StatusCode;
 128             LastModified = response.LastModified;
 129             StatusDescription = response.StatusDescription;
 130             return response;
 131
 132         }
 133
 134
 135         //Synchronous version
 136         protected override WebResponse GetWebResponse(WebRequest request)
 137         {
 138             HttpWebResponse response = null;
 139             try
 140             {
 141                 response = (HttpWebResponse)base.GetWebResponse(request);
 142             }
 143             catch (WebException exc)
 144             {
 145                 if (!TryGetResponse(exc, out response))
 146                     throw;
 147             }
 148
 149             StatusCode = response.StatusCode;
 150             LastModified = response.LastModified;
 151             StatusDescription = response.StatusDescription;
 152             return response;
 153         }
 154
 155         private bool TryGetResponse(WebException exc, out HttpWebResponse response)
 156         {
 157             response = null;
 158             //Fail on empty response
 159             if (exc.Response == null)
 160                 return false;
 161
 162             response = (exc.Response as HttpWebResponse);
 163             //Succeed on allowed status codes
 164             if (AllowedStatusCodes.Contains(response.StatusCode))
 165                 return true;
 166
 167             //Does the response have any content to log?
 168             if (exc.Response.ContentLength > 0)
 169             {
 170                 var content = LogContent(exc.Response);
 171                 Log.ErrorFormat(content);
 172             }
 173             return false;
 174         }
 175
 176         private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
 177
 178         public List<HttpStatusCode> AllowedStatusCodes
 179         {
 180             get
 181             {
 182                 return _allowedStatusCodes;
 183             }
 184         }
 185
 186         public DateTime LastModified { get; private set; }
 187
 188         private static string LogContent(WebResponse webResponse)
 189         {
 190             if (webResponse == null)
 191                 throw new ArgumentNullException("webResponse");
 192             Contract.EndContractBlock();
 193
 194             //The response stream must be copied to avoid affecting other code by disposing of the
 195             //original response stream.
 196             var stream = webResponse.GetResponseStream();
 197             using(var memStream=new MemoryStream((int) stream.Length))
 198             using (var reader = new StreamReader(memStream))
 199             {
 200                 stream.CopyTo(memStream);
 201                 string content = reader.ReadToEnd();
 202
 203                 stream.Seek(0,SeekOrigin.Begin);
 204                 return content;
 205             }
 206         }
 207
 208         public string DownloadStringWithRetry(string address,int retries=0)
 209         {
 210
 211             if (address == null)
 212                 throw new ArgumentNullException("address");
 213
 214             var actualAddress = GetActualAddress(address);
 215
 216             TraceStart("GET",actualAddress);
 217
 218             var actualRetries = (retries == 0) ? Retries : retries;
 219
 220             var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
 221
 222             var task = Retry(() =>
 223             {
 224                 var content = base.DownloadString(uriString);
 225
 226                 if (StatusCode == HttpStatusCode.NoContent)
 227                     return String.Empty;
 228                 return content;
 229
 230             }, actualRetries);
 231
 232             var result = task.Result;
 233             return result;
 234         }
 235
 236         public void Head(string address,int retries=0)
 237         {
 238             AllowedStatusCodes.Add(HttpStatusCode.NotFound);
 239             RetryWithoutContent(address, retries, "HEAD");
 240         }
 241
 242         public void PutWithRetry(string address, int retries = 0)
 243         {
 244             RetryWithoutContent(address, retries, "PUT");
 245         }
 246
 247         public void DeleteWithRetry(string address,int retries=0)
 248         {
 249             RetryWithoutContent(address, retries, "DELETE");
 250         }
 251
 252         public string GetHeaderValue(string headerName,bool optional=false)
 253         {
 254             if (this.ResponseHeaders==null)
 255                 throw new InvalidOperationException("ResponseHeaders are null");
 256             Contract.EndContractBlock();
 257
 258             var values=this.ResponseHeaders.GetValues(headerName);
 259             if (values != null)
 260                 return values[0];
 261
 262             if (optional)
 263                 return null;
 264             //A required header was not found
 265             throw new WebException(String.Format("The {0}  header is missing", headerName));
 266         }
 267
 268         public void SetNonEmptyHeaderValue(string headerName, string value)
 269         {
 270             if (String.IsNullOrWhiteSpace(value))
 271                 return;
 272             Headers.Add(headerName,value);
 273         }
 274
 275         private void RetryWithoutContent(string address, int retries, string method)
 276         {
 277             if (address == null)
 278                 throw new ArgumentNullException("address");
 279
 280             var actualAddress = GetActualAddress(address);
 281             var actualRetries = (retries == 0) ? Retries : retries;
 282
 283             var task = Retry(() =>
 284             {
 285                 var uriString = String.Join("/",BaseAddress ,actualAddress);
 286                 var uri = new Uri(uriString);
 287                 var request =  GetWebRequest(uri);
 288                 request.Method = method;
 289                 if (ResponseHeaders!=null)
 290                     ResponseHeaders.Clear();
 291
 292                 TraceStart(method, uriString);
 293                 if (method == "PUT")
 294                     request.ContentLength = 0;
 295
 296                 //Have to use try/finally instead of using here, because WebClient needs a valid WebResponse object
 297                 //in order to return response headers
 298                 var response = (HttpWebResponse)GetWebResponse(request);
 299                 try
 300                 {
 301                     LastModified = response.LastModified;
 302                     StatusCode = response.StatusCode;
 303                     StatusDescription = response.StatusDescription;
 304                 }
 305                 finally
 306                 {
 307                     response.Close();
 308                 }
 309
 310
 311                 return 0;
 312             }, actualRetries);
 313
 314             try
 315             {
 316                 task.Wait();
 317             }
 318             catch (AggregateException ex)
 319             {
 320                 var exc = ex.InnerException;
 321                 if (exc is RetryException)
 322                 {
 323                     Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
 324                 }
 325                 else
 326                 {
 327                     Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
 328                 }
 329                 throw exc;
 330
 331             }
 332             catch(Exception ex)
 333             {
 334                 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
 335                 throw;
 336             }
 337         }
 338
 339         private static void TraceStart(string method, string actualAddress)
 340         {
 341             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
 342         }
 343
 344         private string GetActualAddress(string address)
 345         {
 346             if (Parameters.Count == 0)
 347                 return address;
 348             var addressBuilder=new StringBuilder(address);
 349
 350             bool isFirst = true;
 351             foreach (var parameter in Parameters)
 352             {
 353                 if(isFirst)
 354                     addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
 355                 else
 356                     addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
 357                 isFirst = false;
 358             }
 359             return addressBuilder.ToString();
 360         }
 361
 362         public string DownloadStringWithRetry(Uri address,int retries=0)
 363         {
 364             if (address == null)
 365                 throw new ArgumentNullException("address");
 366
 367             var actualRetries = (retries == 0) ? Retries : retries;
 368             var task = Retry(() =>
 369             {
 370                 var content = base.DownloadString(address);
 371
 372                 if (StatusCode == HttpStatusCode.NoContent)
 373                     return String.Empty;
 374                 return content;
 375
 376             }, actualRetries);
 377
 378             var result = task.Result;
 379             return result;
 380         }
 381
 382
 383         /// <summary>
 384         /// Copies headers from another RestClient
 385         /// </summary>
 386         /// <param name="source">The RestClient from which the headers are copied</param>
 387         public void CopyHeaders(RestClient source)
 388         {
 389             if (source == null)
 390                 throw new ArgumentNullException("source", "source can't be null");
 391             Contract.EndContractBlock();
 392             //The Headers getter initializes the property, it is never null
 393             Contract.Assume(Headers!=null);
 394
 395             CopyHeaders(source.Headers,Headers);
 396         }
 397
 398         /// <summary>
 399         /// Copies headers from one header collection to another
 400         /// </summary>
 401         /// <param name="source">The source collection from which the headers are copied</param>
 402         /// <param name="target">The target collection to which the headers are copied</param>
 403         public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
 404         {
 405             if (source == null)
 406                 throw new ArgumentNullException("source", "source can't be null");
 407             if (target == null)
 408                 throw new ArgumentNullException("target", "target can't be null");
 409             Contract.EndContractBlock();
 410
 411             for (int i = 0; i < source.Count; i++)
 412             {
 413                 target.Add(source.GetKey(i), source[i]);
 414             }
 415         }
 416
 417         public void AssertStatusOK(string message)
 418         {
 419             if (StatusCode >= HttpStatusCode.BadRequest)
 420                 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
 421         }
 422
 423
 424         private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
 425         {
 426             if (original==null)
 427                 throw new ArgumentNullException("original");
 428             Contract.EndContractBlock();
 429
 430             if (tcs == null)
 431                 tcs = new TaskCompletionSource<T>();
 432             Task.Factory.StartNew(original).ContinueWith(_original =>
 433                 {
 434                     if (!_original.IsFaulted)
 435                         tcs.SetFromTask(_original);
 436                     else
 437                     {
 438                         var e = _original.Exception.InnerException;
 439                         var we = (e as WebException);
 440                         if (we==null)
 441                             tcs.SetException(e);
 442                         else
 443                         {
 444                             var statusCode = GetStatusCode(we);
 445
 446                             //Return null for 404
 447                             if (statusCode == HttpStatusCode.NotFound)
 448                                 tcs.SetResult(default(T));
 449                             //Retry for timeouts and service unavailable
 450                             else if (we.Status == WebExceptionStatus.Timeout ||
 451                                 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
 452                             {
 453                                 TimedOut = true;
 454                                 if (retryCount == 0)
 455                                 {
 456                                     Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
 457                                     tcs.SetException(new RetryException("Timed out too many times.", e));
 458                                 }
 459                                 else
 460                                 {
 461                                     Log.ErrorFormat(
 462                                         "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
 463                                         retryCount, e);
 464                                     Retry(original, retryCount - 1, tcs);
 465                                 }
 466                             }
 467                             else
 468                                 tcs.SetException(e);
 469                         }
 470                     };
 471                 });
 472             return tcs.Task;
 473         }
 474
 475         private HttpStatusCode GetStatusCode(WebException we)
 476         {
 477             if (we==null)
 478                 throw new ArgumentNullException("we");
 479             var statusCode = HttpStatusCode.RequestTimeout;
 480             if (we.Response != null)
 481             {
 482                 statusCode = ((HttpWebResponse) we.Response).StatusCode;
 483                 this.StatusCode = statusCode;
 484             }
 485             return statusCode;
 486         }
 487
 488         public UriBuilder GetAddressBuilder(string container, string objectName)
 489         {
 490             var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
 491             return builder;
 492         }
 493
 494         public Dictionary<string, string> GetMeta(string metaPrefix)
 495         {
 496             if (String.IsNullOrWhiteSpace(metaPrefix))
 497                 throw new ArgumentNullException("metaPrefix");
 498             Contract.EndContractBlock();
 499
 500             var keys = ResponseHeaders.AllKeys.AsQueryable();
 501             var dict = (from key in keys
 502                         where key.StartsWith(metaPrefix)
 503                         let name = key.Substring(metaPrefix.Length)
 504                         select new { Name = name, Value = ResponseHeaders[key] })
 505                         .ToDictionary(t => t.Name, t => t.Value);
 506             return dict;
 507         }
 508     }
 509
 510     public class RetryException:Exception
 511     {
 512         public RetryException()
 513             :base()
 514         {
 515
 516         }
 517
 518         public RetryException(string message)
 519             :base(message)
 520         {
 521
 522         }
 523
 524         public RetryException(string message,Exception innerException)
 525             :base(message,innerException)
 526         {
 527
 528         }
 529
 530         public RetryException(SerializationInfo info,StreamingContext context)
 531             :base(info,context)
 532         {
 533
 534         }
 535     }
 536 }