code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/RestClient.cs

   1 // -----------------------------------------------------------------------
   2 // <copyright file="RestClient.cs" company="Microsoft">
   3 // TODO: Update copyright text.
   4 // </copyright>
   5 // -----------------------------------------------------------------------
   6
   7 using System.Collections.Specialized;
   8 using System.Diagnostics;
   9 using System.Diagnostics.Contracts;
  10 using System.IO;
  11 using System.Net;
  12 using System.Runtime.Serialization;
  13 using System.Threading.Tasks;
  14 using log4net;
  15
  16 namespace Pithos.Network
  17 {
  18     using System;
  19     using System.Collections.Generic;
  20     using System.Linq;
  21     using System.Text;
  22
  23     /// <summary>
  24     /// TODO: Update summary.
  25     /// </summary>
  26     public class RestClient:WebClient
  27     {
  28         public int Timeout { get; set; }
  29
  30         public bool TimedOut { get; set; }
  31
  32         public HttpStatusCode StatusCode { get; private set; }
  33
  34         public string StatusDescription { get; set; }
  35
  36         public long? RangeFrom { get; set; }
  37         public long? RangeTo { get; set; }
  38
  39         public int Retries { get; set; }
  40
  41         private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
  42         public Dictionary<string, string> Parameters
  43         {
  44             get
  45             {
  46                 Contract.Ensures(_parameters!=null);
  47                 return _parameters;
  48             }
  49         }
  50
  51         private static readonly ILog Log = LogManager.GetLogger("RestClient");
  52
  53
  54         [ContractInvariantMethod]
  55         private void Invariants()
  56         {
  57             Contract.Invariant(Headers!=null);
  58         }
  59
  60         public RestClient():base()
  61         {
  62
  63         }
  64
  65
  66         public RestClient(RestClient other)
  67             : base()
  68         {
  69             if (other==null)
  70                 throw new ArgumentNullException("other");
  71             Contract.EndContractBlock();
  72
  73             CopyHeaders(other);
  74             Timeout = other.Timeout;
  75             Retries = other.Retries;
  76             BaseAddress = other.BaseAddress;
  77
  78             foreach (var parameter in other.Parameters)
  79             {
  80                 Parameters.Add(parameter.Key,parameter.Value);
  81             }
  82
  83             this.Proxy = other.Proxy;
  84         }
  85
  86         protected override WebRequest GetWebRequest(Uri address)
  87         {
  88             TimedOut = false;
  89             var webRequest = base.GetWebRequest(address);
  90             var request = (HttpWebRequest)webRequest;
  91             if (IfModifiedSince.HasValue)
  92                 request.IfModifiedSince = IfModifiedSince.Value;
  93             request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
  94             if(Timeout>0)
  95                 request.Timeout = Timeout;
  96
  97             if (RangeFrom.HasValue)
  98             {
  99                 if (RangeTo.HasValue)
 100                     request.AddRange(RangeFrom.Value, RangeTo.Value);
 101                 else
 102                     request.AddRange(RangeFrom.Value);
 103             }
 104             return request;
 105         }
 106
 107         public DateTime? IfModifiedSince { get; set; }
 108
 109         protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
 110         {
 111             return ProcessResponse(()=>base.GetWebResponse(request, result));
 112         }
 113
 114         protected override WebResponse GetWebResponse(WebRequest request)
 115         {
 116             return ProcessResponse(() => base.GetWebResponse(request));
 117         }
 118
 119         private WebResponse ProcessResponse(Func<WebResponse> getResponse)
 120         {
 121             try
 122             {
 123                 var response = (HttpWebResponse)getResponse();
 124                 StatusCode = response.StatusCode;
 125                 LastModified = response.LastModified;
 126                 StatusDescription = response.StatusDescription;
 127                 return response;
 128             }
 129             catch (WebException exc)
 130             {
 131                 if (exc.Response != null)
 132                 {
 133                     var response = (exc.Response as HttpWebResponse);
 134                     if (AllowedStatusCodes.Contains(response.StatusCode))
 135                     {
 136                         StatusCode = response.StatusCode;
 137                         LastModified = response.LastModified;
 138                         StatusDescription = response.StatusDescription;
 139
 140                         return response;
 141                     }
 142                     if (exc.Response.ContentLength > 0)
 143                     {
 144                         string content = GetContent(exc.Response);
 145                         Log.ErrorFormat(content);
 146                     }
 147                 }
 148                 throw;
 149             }
 150         }
 151
 152         private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
 153         public List<HttpStatusCode> AllowedStatusCodes
 154         {
 155             get
 156             {
 157                 return _allowedStatusCodes;
 158             }
 159         }
 160
 161         public DateTime LastModified { get; private set; }
 162
 163         private static string GetContent(WebResponse webResponse)
 164         {
 165             if (webResponse == null)
 166                 throw new ArgumentNullException("webResponse");
 167             Contract.EndContractBlock();
 168
 169             string content;
 170             using (var stream = webResponse.GetResponseStream())
 171             using (var reader = new StreamReader(stream))
 172             {
 173                 content = reader.ReadToEnd();
 174             }
 175             return content;
 176         }
 177
 178         public string DownloadStringWithRetry(string address,int retries=0)
 179         {
 180             if (address == null)
 181                 throw new ArgumentNullException("address");
 182
 183             var actualAddress = GetActualAddress(address);
 184
 185             TraceStart("GET",actualAddress);
 186
 187             var actualRetries = (retries == 0) ? Retries : retries;
 188
 189
 190
 191             var task = Retry(() =>
 192             {
 193                 var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
 194                 var content = base.DownloadString(uriString);
 195
 196                 if (StatusCode == HttpStatusCode.NoContent)
 197                     return String.Empty;
 198                 return content;
 199
 200             }, actualRetries);
 201
 202             var result = task.Result;
 203             return result;
 204         }
 205
 206         public void Head(string address,int retries=0)
 207         {
 208             AllowedStatusCodes.Add(HttpStatusCode.NotFound);
 209             RetryWithoutContent(address, retries, "HEAD");
 210         }
 211
 212         public void PutWithRetry(string address, int retries = 0)
 213         {
 214             RetryWithoutContent(address, retries, "PUT");
 215         }
 216
 217         public void DeleteWithRetry(string address,int retries=0)
 218         {
 219             RetryWithoutContent(address, retries, "DELETE");
 220         }
 221
 222         public string GetHeaderValue(string headerName)
 223         {
 224             if (this.ResponseHeaders==null)
 225                 throw new InvalidOperationException("ResponseHeaders are null");
 226             Contract.EndContractBlock();
 227
 228             var values=this.ResponseHeaders.GetValues(headerName);
 229             if (values == null)
 230                 throw new WebException(String.Format("The {0}  header is missing", headerName));
 231             else
 232                 return values[0];
 233         }
 234
 235         private void RetryWithoutContent(string address, int retries, string method)
 236         {
 237             if (address == null)
 238                 throw new ArgumentNullException("address");
 239
 240             var actualAddress = GetActualAddress(address);
 241             var actualRetries = (retries == 0) ? Retries : retries;
 242
 243             var task = Retry(() =>
 244             {
 245                 var uriString = String.Join("/",BaseAddress ,actualAddress);
 246                 var uri = new Uri(uriString);
 247                 var request =  GetWebRequest(uri);
 248                 request.Method = method;
 249                 if (ResponseHeaders!=null)
 250                     ResponseHeaders.Clear();
 251
 252                 TraceStart(method, uriString);
 253                 if (method == "PUT")
 254                     request.ContentLength = 0;
 255                 var response = (HttpWebResponse)GetWebResponse(request);
 256                 StatusCode = response.StatusCode;
 257                 StatusDescription = response.StatusDescription;
 258
 259
 260                 return 0;
 261             }, actualRetries);
 262
 263             try
 264             {
 265                 task.Wait();
 266             }
 267             catch (AggregateException ex)
 268             {
 269                 var exc = ex.InnerException;
 270                 if (exc is RetryException)
 271                 {
 272                     Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
 273                 }
 274                 else
 275                 {
 276                     Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
 277                 }
 278                 throw exc;
 279
 280             }
 281             catch(Exception ex)
 282             {
 283                 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
 284                 throw;
 285             }
 286         }
 287
 288         private static void TraceStart(string method, string actualAddress)
 289         {
 290             Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
 291         }
 292
 293         private string GetActualAddress(string address)
 294         {
 295             if (Parameters.Count == 0)
 296                 return address;
 297             var addressBuilder=new StringBuilder(address);
 298
 299             bool isFirst = true;
 300             foreach (var parameter in Parameters)
 301             {
 302                 if(isFirst)
 303                     addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
 304                 else
 305                     addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
 306                 isFirst = false;
 307             }
 308             return addressBuilder.ToString();
 309         }
 310
 311         public string DownloadStringWithRetry(Uri address,int retries=0)
 312         {
 313             if (address == null)
 314                 throw new ArgumentNullException("address");
 315
 316             var actualRetries = (retries == 0) ? Retries : retries;
 317             var task = Retry(() =>
 318             {
 319                 var content = base.DownloadString(address);
 320
 321                 if (StatusCode == HttpStatusCode.NoContent)
 322                     return String.Empty;
 323                 return content;
 324
 325             }, actualRetries);
 326
 327             var result = task.Result;
 328             return result;
 329         }
 330
 331
 332         /// <summary>
 333         /// Copies headers from another RestClient
 334         /// </summary>
 335         /// <param name="source">The RestClient from which the headers are copied</param>
 336         public void CopyHeaders(RestClient source)
 337         {
 338             if (source == null)
 339                 throw new ArgumentNullException("source", "source can't be null");
 340             Contract.EndContractBlock();
 341             //The Headers getter initializes the property, it is never null
 342             Contract.Assume(Headers!=null);
 343
 344             CopyHeaders(source.Headers,Headers);
 345         }
 346
 347         /// <summary>
 348         /// Copies headers from one header collection to another
 349         /// </summary>
 350         /// <param name="source">The source collection from which the headers are copied</param>
 351         /// <param name="target">The target collection to which the headers are copied</param>
 352         public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
 353         {
 354             if (source == null)
 355                 throw new ArgumentNullException("source", "source can't be null");
 356             if (target == null)
 357                 throw new ArgumentNullException("target", "target can't be null");
 358             Contract.EndContractBlock();
 359
 360             for (int i = 0; i < source.Count; i++)
 361             {
 362                 target.Add(source.GetKey(i), source[i]);
 363             }
 364         }
 365
 366         public void AssertStatusOK(string message)
 367         {
 368             if (StatusCode >= HttpStatusCode.BadRequest)
 369                 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
 370         }
 371
 372
 373         private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
 374         {
 375             if (original==null)
 376                 throw new ArgumentNullException("original");
 377             Contract.EndContractBlock();
 378
 379             if (tcs == null)
 380                 tcs = new TaskCompletionSource<T>();
 381             Task.Factory.StartNew(original).ContinueWith(_original =>
 382                 {
 383                     if (!_original.IsFaulted)
 384                         tcs.SetFromTask(_original);
 385                     else
 386                     {
 387                         var e = _original.Exception.InnerException;
 388                         var we = (e as WebException);
 389                         if (we==null)
 390                             tcs.SetException(e);
 391                         else
 392                         {
 393                             var statusCode = GetStatusCode(we);
 394
 395                             //Return null for 404
 396                             if (statusCode == HttpStatusCode.NotFound)
 397                                 tcs.SetResult(default(T));
 398                             //Retry for timeouts and service unavailable
 399                             else if (we.Status == WebExceptionStatus.Timeout ||
 400                                 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
 401                             {
 402                                 TimedOut = true;
 403                                 if (retryCount == 0)
 404                                 {
 405                                     Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
 406                                     tcs.SetException(new RetryException("Timed out too many times.", e));
 407                                 }
 408                                 else
 409                                 {
 410                                     Log.ErrorFormat(
 411                                         "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
 412                                         retryCount, e);
 413                                     Retry(original, retryCount - 1, tcs);
 414                                 }
 415                             }
 416                             else
 417                                 tcs.SetException(e);
 418                         }
 419                     };
 420                 });
 421             return tcs.Task;
 422         }
 423
 424         private HttpStatusCode GetStatusCode(WebException we)
 425         {
 426             if (we==null)
 427                 throw new ArgumentNullException("we");
 428             var statusCode = HttpStatusCode.RequestTimeout;
 429             if (we.Response != null)
 430             {
 431                 statusCode = ((HttpWebResponse) we.Response).StatusCode;
 432                 this.StatusCode = statusCode;
 433             }
 434             return statusCode;
 435         }
 436
 437         public UriBuilder GetAddressBuilder(string container, string objectName)
 438         {
 439             var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
 440             return builder;
 441         }
 442     }
 443
 444     public class RetryException:Exception
 445     {
 446         public RetryException()
 447             :base()
 448         {
 449
 450         }
 451
 452         public RetryException(string message)
 453             :base(message)
 454         {
 455
 456         }
 457
 458         public RetryException(string message,Exception innerException)
 459             :base(message,innerException)
 460         {
 461
 462         }
 463
 464         public RetryException(SerializationInfo info,StreamingContext context)
 465             :base(info,context)
 466         {
 467
 468         }
 469     }
 470 }