1 // -----------------------------------------------------------------------
2 // <copyright file="RestClient.cs" company="GRNet">
3 // Copyright 2011-2012 GRNET S.A. All rights reserved.
5 // Redistribution and use in source and binary forms, with or
6 // without modification, are permitted provided that the following
9 // 1. Redistributions of source code must retain the above
10 // copyright notice, this list of conditions and the following
13 // 2. Redistributions in binary form must reproduce the above
14 // copyright notice, this list of conditions and the following
15 // disclaimer in the documentation and/or other materials
16 // provided with the distribution.
18 // THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 // OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 // USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 // AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 // POSSIBILITY OF SUCH DAMAGE.
31 // The views and conclusions contained in the software and
32 // documentation are those of the authors and should not be
33 // interpreted as representing official policies, either expressed
34 // or implied, of GRNET S.A.
36 // -----------------------------------------------------------------------
38 using System.Collections.Specialized;
39 using System.Diagnostics;
40 using System.Diagnostics.Contracts;
43 using System.Runtime.Serialization;
44 using System.Threading.Tasks;
48 namespace Pithos.Network
51 using System.Collections.Generic;
56 /// TODO: Update summary.
58 public class RestClient:WebClient
60 public int Timeout { get; set; }
62 public bool TimedOut { get; set; }
64 public HttpStatusCode StatusCode { get; private set; }
66 public string StatusDescription { get; set; }
68 public long? RangeFrom { get; set; }
69 public long? RangeTo { get; set; }
71 public int Retries { get; set; }
73 private readonly Dictionary<string, string> _parameters=new Dictionary<string, string>();
74 public Dictionary<string, string> Parameters
78 Contract.Ensures(_parameters!=null);
83 private static readonly ILog Log = LogManager.GetLogger("RestClient");
86 [ContractInvariantMethod]
87 private void Invariants()
89 Contract.Invariant(Headers!=null);
92 public RestClient():base()
98 public RestClient(RestClient other)
102 throw new ArgumentNullException("other");
103 Contract.EndContractBlock();
106 Timeout = other.Timeout;
107 Retries = other.Retries;
108 BaseAddress = other.BaseAddress;
110 foreach (var parameter in other.Parameters)
112 Parameters.Add(parameter.Key,parameter.Value);
115 this.Proxy = other.Proxy;
119 protected override WebRequest GetWebRequest(Uri address)
122 var webRequest = base.GetWebRequest(address);
123 var request = (HttpWebRequest)webRequest;
124 request.ServicePoint.ConnectionLimit = 50;
125 if (IfModifiedSince.HasValue)
126 request.IfModifiedSince = IfModifiedSince.Value;
127 request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
129 request.Timeout = Timeout;
131 if (RangeFrom.HasValue)
133 if (RangeTo.HasValue)
134 request.AddRange(RangeFrom.Value, RangeTo.Value);
136 request.AddRange(RangeFrom.Value);
141 public DateTime? IfModifiedSince { get; set; }
143 //Asynchronous version
144 protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
146 Log.InfoFormat("ASYNC [{0}] {1}",request.Method, request.RequestUri);
147 HttpWebResponse response = null;
151 response = (HttpWebResponse)base.GetWebResponse(request, result);
153 catch (WebException exc)
155 if (!TryGetResponse(exc, out response))
159 StatusCode = response.StatusCode;
160 LastModified = response.LastModified;
161 StatusDescription = response.StatusDescription;
167 //Synchronous version
168 protected override WebResponse GetWebResponse(WebRequest request)
170 HttpWebResponse response = null;
173 response = (HttpWebResponse)base.GetWebResponse(request);
175 catch (WebException exc)
177 if (!TryGetResponse(exc, out response))
181 StatusCode = response.StatusCode;
182 LastModified = response.LastModified;
183 StatusDescription = response.StatusDescription;
187 private bool TryGetResponse(WebException exc, out HttpWebResponse response)
190 //Fail on empty response
191 if (exc.Response == null)
194 response = (exc.Response as HttpWebResponse);
195 //Succeed on allowed status codes
196 if (AllowedStatusCodes.Contains(response.StatusCode))
199 //Does the response have any content to log?
200 if (exc.Response.ContentLength > 0)
202 var content = LogContent(exc.Response);
203 Log.ErrorFormat(content);
208 private readonly List<HttpStatusCode> _allowedStatusCodes=new List<HttpStatusCode>{HttpStatusCode.NotModified};
210 public List<HttpStatusCode> AllowedStatusCodes
214 return _allowedStatusCodes;
218 public DateTime LastModified { get; private set; }
220 private static string LogContent(WebResponse webResponse)
222 if (webResponse == null)
223 throw new ArgumentNullException("webResponse");
224 Contract.EndContractBlock();
226 //The response stream must be copied to avoid affecting other code by disposing of the
227 //original response stream.
228 var stream = webResponse.GetResponseStream();
229 using(var memStream=new MemoryStream())
230 using (var reader = new StreamReader(memStream))
232 stream.CopyTo(memStream);
233 string content = reader.ReadToEnd();
235 stream.Seek(0,SeekOrigin.Begin);
240 public string DownloadStringWithRetry(string address,int retries=0)
244 throw new ArgumentNullException("address");
246 var actualAddress = GetActualAddress(address);
248 TraceStart("GET",actualAddress);
250 var actualRetries = (retries == 0) ? Retries : retries;
252 var uriString = String.Join("/", BaseAddress.TrimEnd('/'), actualAddress);
254 var task = Retry(() =>
256 var content = base.DownloadString(uriString);
258 if (StatusCode == HttpStatusCode.NoContent)
266 var result = task.Result;
270 catch (AggregateException exc)
272 //If the task fails, propagate the original exception
273 if (exc.InnerException!=null)
274 throw exc.InnerException;
279 public void Head(string address,int retries=0)
281 AllowedStatusCodes.Add(HttpStatusCode.NotFound);
282 RetryWithoutContent(address, retries, "HEAD");
285 public void PutWithRetry(string address, int retries = 0)
287 RetryWithoutContent(address, retries, "PUT");
290 public void DeleteWithRetry(string address,int retries=0)
292 RetryWithoutContent(address, retries, "DELETE");
295 public string GetHeaderValue(string headerName,bool optional=false)
297 if (this.ResponseHeaders==null)
298 throw new InvalidOperationException("ResponseHeaders are null");
299 Contract.EndContractBlock();
301 var values=this.ResponseHeaders.GetValues(headerName);
307 //A required header was not found
308 throw new WebException(String.Format("The {0} header is missing", headerName));
311 public void SetNonEmptyHeaderValue(string headerName, string value)
313 if (String.IsNullOrWhiteSpace(value))
315 Headers.Add(headerName,value);
318 private void RetryWithoutContent(string address, int retries, string method)
321 throw new ArgumentNullException("address");
323 var actualAddress = GetActualAddress(address);
324 var actualRetries = (retries == 0) ? Retries : retries;
326 var task = Retry(() =>
328 var uriString = String.Join("/",BaseAddress ,actualAddress);
329 var uri = new Uri(uriString);
330 var request = GetWebRequest(uri);
331 request.Method = method;
332 if (ResponseHeaders!=null)
333 ResponseHeaders.Clear();
335 TraceStart(method, uriString);
337 request.ContentLength = 0;
339 //Have to use try/finally instead of using here, because WebClient needs a valid WebResponse object
340 //in order to return response headers
341 var response = (HttpWebResponse)GetWebResponse(request);
344 LastModified = response.LastModified;
345 StatusCode = response.StatusCode;
346 StatusDescription = response.StatusDescription;
361 catch (AggregateException ex)
363 var exc = ex.InnerException;
364 if (exc is RetryException)
366 Log.ErrorFormat("[{0}] RETRY FAILED for {1} after {2} retries",method,address,retries);
370 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
377 Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, ex);
382 private static void TraceStart(string method, string actualAddress)
384 Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
387 private string GetActualAddress(string address)
389 if (Parameters.Count == 0)
391 var addressBuilder=new StringBuilder(address);
394 foreach (var parameter in Parameters)
397 addressBuilder.AppendFormat("?{0}={1}", parameter.Key, parameter.Value);
399 addressBuilder.AppendFormat("&{0}={1}", parameter.Key, parameter.Value);
402 return addressBuilder.ToString();
405 public string DownloadStringWithRetry(Uri address,int retries=0)
408 throw new ArgumentNullException("address");
410 var actualRetries = (retries == 0) ? Retries : retries;
411 var task = Retry(() =>
413 var content = base.DownloadString(address);
415 if (StatusCode == HttpStatusCode.NoContent)
421 var result = task.Result;
427 /// Copies headers from another RestClient
429 /// <param name="source">The RestClient from which the headers are copied</param>
430 public void CopyHeaders(RestClient source)
433 throw new ArgumentNullException("source", "source can't be null");
434 Contract.EndContractBlock();
435 //The Headers getter initializes the property, it is never null
436 Contract.Assume(Headers!=null);
438 CopyHeaders(source.Headers,Headers);
442 /// Copies headers from one header collection to another
444 /// <param name="source">The source collection from which the headers are copied</param>
445 /// <param name="target">The target collection to which the headers are copied</param>
446 public static void CopyHeaders(WebHeaderCollection source,WebHeaderCollection target)
449 throw new ArgumentNullException("source", "source can't be null");
451 throw new ArgumentNullException("target", "target can't be null");
452 Contract.EndContractBlock();
454 for (int i = 0; i < source.Count; i++)
456 target.Add(source.GetKey(i), source[i]);
460 public void AssertStatusOK(string message)
462 if (StatusCode >= HttpStatusCode.BadRequest)
463 throw new WebException(String.Format("{0} with code {1} - {2}", message, StatusCode, StatusDescription));
467 private Task<T> Retry<T>(Func<T> original, int retryCount, TaskCompletionSource<T> tcs = null)
470 throw new ArgumentNullException("original");
471 Contract.EndContractBlock();
474 tcs = new TaskCompletionSource<T>();
475 Task.Factory.StartNew(original).ContinueWith(_original =>
477 if (!_original.IsFaulted)
478 tcs.SetFromTask(_original);
481 var e = _original.Exception.InnerException;
482 var we = (e as WebException);
487 var statusCode = GetStatusCode(we);
489 //Return null for 404
490 if (statusCode == HttpStatusCode.NotFound)
491 tcs.SetResult(default(T));
492 //Retry for timeouts and service unavailable
493 else if (we.Status == WebExceptionStatus.Timeout ||
494 (we.Status == WebExceptionStatus.ProtocolError && statusCode == HttpStatusCode.ServiceUnavailable))
499 Log.ErrorFormat("[ERROR] Timed out too many times. \n{0}\n",e);
500 tcs.SetException(new RetryException("Timed out too many times.", e));
505 "[RETRY] Timed out after {0} ms. Will retry {1} more times\n{2}", Timeout,
507 Retry(original, retryCount - 1, tcs);
518 private HttpStatusCode GetStatusCode(WebException we)
521 throw new ArgumentNullException("we");
522 var statusCode = HttpStatusCode.RequestTimeout;
523 if (we.Response != null)
525 statusCode = ((HttpWebResponse) we.Response).StatusCode;
526 this.StatusCode = statusCode;
531 public UriBuilder GetAddressBuilder(string container, string objectName)
533 var builder = new UriBuilder(String.Join("/", BaseAddress, container, objectName));
537 public Dictionary<string, string> GetMeta(string metaPrefix)
539 if (String.IsNullOrWhiteSpace(metaPrefix))
540 throw new ArgumentNullException("metaPrefix");
541 Contract.EndContractBlock();
543 var keys = ResponseHeaders.AllKeys.AsQueryable();
544 var dict = (from key in keys
545 where key.StartsWith(metaPrefix)
546 let name = key.Substring(metaPrefix.Length)
547 select new { Name = name, Value = ResponseHeaders[key] })
548 .ToDictionary(t => t.Name, t => t.Value);
553 public class RetryException:Exception
555 public RetryException()
561 public RetryException(string message)
567 public RetryException(string message,Exception innerException)
568 :base(message,innerException)
573 public RetryException(SerializationInfo info,StreamingContext context)