code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Network/WebExtensions.cs

   1 using System;\r
   2 using System.Collections.Generic;\r
   3 using System.ComponentModel;\r
   4 using System.Diagnostics.Contracts;\r
   5 using System.Linq;\r
   6 using System.Net.Http;\r
   7 using System.Net.Http.Headers;\r
   8 using System.Reflection;\r
   9 using System.Text;\r
  10 using System.Net;\r
  11 using System.IO;\r
  12 using System.Threading;\r
  13 using System.Threading.Tasks;\r
  14 using log4net;\r
  15 \r
  16 namespace Pithos.Network\r
  17 {\r
  18     public static class WebExtensions\r
  19     {\r
  20         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  21 \r
  22         public static string ReadToEnd(this HttpWebResponse response)\r
  23         {\r
  24             using (var stream = response.GetResponseStream())\r
  25             {\r
  26                 if (stream == null)\r
  27                     return null;\r
  28                 using (var reader = new StreamReader(stream))\r
  29                 {\r
  30                     var body = reader.ReadToEnd();\r
  31                     return body;\r
  32                 }\r
  33             }\r
  34         }\r
  35 \r
  36         public static void LogError(this ILog log, HttpWebResponse response)\r
  37         {\r
  38             if (log.IsDebugEnabled)\r
  39             {\r
  40                 if (response != null)\r
  41                 {\r
  42                     var body = response.ReadToEnd();\r
  43                     log.ErrorFormat("Headers:\n{0}\nBody:{1}", response.Headers, body);\r
  44                 }\r
  45             }\r
  46         }\r
  47 \r
  48         public static TextReader GetLoggedReader(this Stream stream, ILog log)\r
  49         {\r
  50             var reader = new StreamReader(stream);\r
  51             if (!log.IsDebugEnabled)\r
  52                 return reader;\r
  53 \r
  54             using (reader)\r
  55             {\r
  56                 var body = reader.ReadToEnd();\r
  57                 log.DebugFormat("JSON response: {0}", body);\r
  58                 return new StringReader(body);\r
  59             }\r
  60         }\r
  61 \r
  62 \r
  63         public static IEnumerable<T> Range<T>(this IList<T> list, int start, int end)\r
  64         {\r
  65             Contract.Requires(start >= 0);\r
  66             Contract.Requires(end < list.Count);\r
  67             Contract.Requires(start <= end);\r
  68 \r
  69             if (list == null)\r
  70                 yield break;\r
  71 \r
  72             for (var i = 0; i <= end; i++)\r
  73             {\r
  74                 yield return list[i];\r
  75             }\r
  76 \r
  77         }\r
  78 \r
  79         public static Task<byte[]> UploadDataTaskAsync(this WebClient webClient, Uri address, string method, byte[] data, CancellationToken cancellationToken, IProgress<UploadProgressChangedEventArgs> progress)\r
  80         {\r
  81             var tcs = new TaskCompletionSource<byte[]>(address);\r
  82             if (cancellationToken.IsCancellationRequested)\r
  83             {\r
  84                 tcs.TrySetCanceled();\r
  85             }\r
  86             else\r
  87             {\r
  88                 CancellationTokenRegistration ctr = cancellationToken.Register(()=>\r
  89                 {\r
  90                     webClient.CancelAsync();\r
  91                 });\r
  92                 UploadDataCompletedEventHandler completedHandler = null;\r
  93                 UploadProgressChangedEventHandler progressHandler = null;\r
  94                 if (progress != null)\r
  95                     progressHandler = (s, e) => PithosEAPCommon.HandleProgress(tcs, e, () => e, progress);\r
  96                 completedHandler =(sender, e) =>PithosEAPCommon.HandleCompletion(tcs, true, e,() => e.Result,() =>\r
  97                 { \r
  98                     ctr.Dispose();\r
  99                     webClient.UploadDataCompleted -= completedHandler;\r
 100                     webClient.UploadProgressChanged -= progressHandler;\r
 101                 });\r
 102                 webClient.UploadDataCompleted += completedHandler;\r
 103                 webClient.UploadProgressChanged += progressHandler;\r
 104                 try\r
 105                 {\r
 106                     webClient.UploadDataAsync(address, method, data, tcs);\r
 107                     if (cancellationToken.IsCancellationRequested)\r
 108                         webClient.CancelAsync();\r
 109                 }\r
 110                 catch\r
 111                 {\r
 112                     webClient.UploadDataCompleted -= completedHandler;\r
 113                     webClient.UploadProgressChanged -= progressHandler;\r
 114                     throw;\r
 115                 }\r
 116             }\r
 117             return tcs.Task;\r
 118         }\r
 119 \r
 120         public static async Task<T> WithRetries<T>(this Func<Task<T>> func, int retries)\r
 121         {\r
 122             while (retries > 0)\r
 123             {\r
 124                 try\r
 125                 {\r
 126                     var result = await func().ConfigureAwait(false);\r
 127                     return result;\r
 128                 }\r
 129                 catch (Exception exc)\r
 130                 {\r
 131                     if (--retries == 0)\r
 132                         throw new RetryException("Failed too many times", exc);\r
 133                 }\r
 134             }\r
 135             throw new RetryException();\r
 136         }\r
 137 \r
 138         public static async Task<HttpResponseMessage> WithRetriesForWeb(this Func<Task<HttpResponseMessage>> func, int retries)\r
 139         {\r
 140             var waitTime = TimeSpan.FromSeconds(10);\r
 141             var acceptedCodes = new[] { HttpStatusCode.Moved, HttpStatusCode.MovedPermanently, HttpStatusCode.Found, HttpStatusCode.Redirect,HttpStatusCode.SeeOther,\r
 142                 HttpStatusCode.RedirectMethod,HttpStatusCode.NotModified,HttpStatusCode.TemporaryRedirect,HttpStatusCode.RedirectKeepVerb};\r
 143             while (retries > 0)\r
 144             {\r
 145                 var result = await func().ConfigureAwait(false);\r
 146                 if (result.IsSuccessStatusCode || acceptedCodes.Contains(result.StatusCode))\r
 147                     return result;\r
 148                     \r
 149                 if (--retries == 0)\r
 150                     throw new RetryException("Failed too many times");\r
 151 \r
 152                 //Wait for service unavailable\r
 153                 if (result.StatusCode == HttpStatusCode.ServiceUnavailable)\r
 154                 {\r
 155                     Log.InfoFormat("[UNAVAILABLE] Waiting before retry: {0}",result.ReasonPhrase);\r
 156                     await TaskEx.Delay(waitTime).ConfigureAwait(false);\r
 157                     //increase the timeout for repeated timeouts\r
 158                     if (waitTime<TimeSpan.FromSeconds(10))\r
 159                         waitTime = waitTime.Add(TimeSpan.FromSeconds(10));\r
 160                 }                \r
 161                 //Throw in all other cases\r
 162                 else \r
 163                     result.EnsureSuccessStatusCode();\r
 164             }\r
 165             throw new RetryException();\r
 166         }\r
 167 \r
 168 \r
 169         public static async Task<string> GetStringAsyncWithRetries(this HttpClient client, Uri requestUri, int retries,DateTime? since=null)\r
 170         {                        \r
 171             var request = new HttpRequestMessage(HttpMethod.Get, requestUri);            \r
 172             if (since.HasValue)\r
 173             {\r
 174                 request.Headers.IfModifiedSince = since.Value;\r
 175             }\r
 176             //Func<Task<HttpResponseMessage>> call = () => _baseHttpClient.SendAsync(request);\r
 177             using (var response = await client.SendAsyncWithRetries(request,3).ConfigureAwait(false))\r
 178             {\r
 179                 if (response.StatusCode == HttpStatusCode.NoContent)\r
 180                     return String.Empty;\r
 181 \r
 182                 var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);\r
 183                 return content;\r
 184             }\r
 185 \r
 186         }\r
 187 \r
 188         public static Task<HttpResponseMessage> HeadAsyncWithRetries(this HttpClient client, Uri requestUri, int retries)\r
 189         {\r
 190             return client.HeadAsyncWithRetries(requestUri, retries, HttpCompletionOption.ResponseContentRead, CancellationToken.None);\r
 191         }\r
 192 \r
 193         public static Task<HttpResponseMessage> HeadAsyncWithRetries(this HttpClient client, Uri requestUri, int retries, HttpCompletionOption completionOption, CancellationToken cancellationToken)\r
 194         {\r
 195             return client.SendAsyncWithRetries(new HttpRequestMessage(HttpMethod.Head, requestUri), retries, completionOption, cancellationToken);\r
 196         }\r
 197 \r
 198         public static Task<HttpResponseMessage> GetAsyncWithRetries(this HttpClient client, Uri requestUri, int retries)\r
 199         {\r
 200             return client.GetAsyncWithRetries(requestUri, retries, HttpCompletionOption.ResponseContentRead, CancellationToken.None);\r
 201         }\r
 202 \r
 203         public static Task<HttpResponseMessage> GetAsyncWithRetries(this HttpClient client, Uri requestUri, int retries, HttpCompletionOption completionOption, CancellationToken cancellationToken)\r
 204         {\r
 205             return client.SendAsyncWithRetries(new HttpRequestMessage(HttpMethod.Get, requestUri), retries, completionOption, cancellationToken);\r
 206         }\r
 207 \r
 208 \r
 209         public static Task<HttpResponseMessage> SendAsyncWithRetries(this HttpClient client,HttpRequestMessage message, int retries)\r
 210         {\r
 211             return client.SendAsyncWithRetries(message, retries, HttpCompletionOption.ResponseContentRead, CancellationToken.None);\r
 212         }\r
 213 \r
 214         public static async Task<HttpResponseMessage> SendAsyncWithRetries(this HttpClient client,HttpRequestMessage message, int retries,HttpCompletionOption completionOption, CancellationToken cancellationToken)\r
 215         {\r
 216             var waitTime = TimeSpan.FromSeconds(10);\r
 217             var acceptedCodes = new[] { HttpStatusCode.Moved, HttpStatusCode.MovedPermanently, HttpStatusCode.Found, HttpStatusCode.Redirect,HttpStatusCode.SeeOther,\r
 218                 HttpStatusCode.RedirectMethod,HttpStatusCode.NotModified,HttpStatusCode.TemporaryRedirect,HttpStatusCode.RedirectKeepVerb,HttpStatusCode.Conflict};\r
 219             while (retries > 0)\r
 220             {\r
 221                 var timedOut = false;\r
 222                 if (Log.IsDebugEnabled)\r
 223                     Log.DebugFormat("[REQUEST] {0}", message);\r
 224                 HttpResponseMessage result=null;\r
 225                 try\r
 226                 {\r
 227                     result = await client.SendAsync(message, completionOption, cancellationToken).ConfigureAwait(false);\r
 228                 }\r
 229                 catch (WebException exc)\r
 230                 {\r
 231                     if (exc.Status != WebExceptionStatus.Timeout)\r
 232                         throw;\r
 233                     timedOut = true;\r
 234                     if (Log.IsDebugEnabled)\r
 235                         Log.DebugFormat("[RESPONSE] [{0}]:[{1}] FAIL WITH TIMEOUT", message.Method, message.RequestUri);\r
 236                 }\r
 237                 catch (Exception exc)\r
 238                 {\r
 239                     Log.FatalFormat("Unexpected error while sending:\n{0}\n{1}", message, exc);\r
 240                     throw;\r
 241                 }\r
 242 \r
 243                 if (timedOut)\r
 244                 {\r
 245                     if (--retries == 0)\r
 246                         throw new RetryException("Failed too many times");\r
 247                     continue;\r
 248                 }\r
 249 \r
 250                 if (result.IsSuccessStatusCode || acceptedCodes.Contains(result.StatusCode))\r
 251                 {\r
 252                     if (Log.IsDebugEnabled)\r
 253                         Log.DebugFormat("[RESPONSE] [{0}]:[{1}] OK: [{2}]", message.Method, message.RequestUri,\r
 254                                         result.StatusCode);\r
 255                     return result;\r
 256                 }\r
 257                 //Failed, will have to abort or retry\r
 258                 if (Log.IsDebugEnabled)\r
 259                     Log.DebugFormat("[RESPONSE] [{0}]:[{1}] FAIL: [{2}]\n{3}", message.Method, message.RequestUri,\r
 260                                     result.StatusCode, result);\r
 261 \r
 262                 if (--retries == 0)\r
 263                     throw new RetryException("Failed too many times");\r
 264 \r
 265                 //Wait for service unavailable\r
 266                 if (result.StatusCode == HttpStatusCode.ServiceUnavailable ||\r
 267                     result.StatusCode == HttpStatusCode.BadGateway)\r
 268                 {\r
 269 \r
 270                     Log.WarnFormat("[UNAVAILABLE] Waiting before retrying [{0}]:[{1}] due to [{2}]", message.Method,\r
 271                                    message.RequestUri, result.ReasonPhrase);\r
 272                     await TaskEx.Delay(waitTime).ConfigureAwait(false);\r
 273                     //increase the timeout for repeated timeouts\r
 274                     if (waitTime < TimeSpan.FromSeconds(10))\r
 275                         waitTime = waitTime.Add(TimeSpan.FromSeconds(10));\r
 276                 }\r
 277                     //Throw in all other cases\r
 278                 else\r
 279                     result.EnsureSuccessStatusCode();\r
 280             }\r
 281             throw new RetryException();\r
 282         }\r
 283 \r
 284         public static string GetFirstValue(this HttpResponseHeaders headers, string name)\r
 285         {\r
 286             if (headers==null)\r
 287                 throw new ArgumentNullException("headers");\r
 288             if (String.IsNullOrWhiteSpace(name))\r
 289                 throw new ArgumentNullException("name");\r
 290             Contract.EndContractBlock();\r
 291 \r
 292             IEnumerable<string> values;\r
 293             if (headers.TryGetValues(name, out values))\r
 294             {\r
 295                 return values.FirstOrDefault();\r
 296             }\r
 297             return null;\r
 298         }\r
 299 \r
 300         public static  Dictionary<string, string> GetMeta(this HttpResponseHeaders headers,string metaPrefix)\r
 301         {\r
 302             if (headers == null)\r
 303                 throw new ArgumentNullException("headers");\r
 304             if (String.IsNullOrWhiteSpace(metaPrefix))\r
 305                 throw new ArgumentNullException("metaPrefix");\r
 306             Contract.EndContractBlock();\r
 307 \r
 308             var dict = (from header in headers\r
 309                         where header.Key.StartsWith(metaPrefix)\r
 310                         let name = header.Key.Substring(metaPrefix.Length)\r
 311                         select new { Name = name, Value = String.Join(",",header.Value) })\r
 312                         .ToDictionary(t => t.Name, t => t.Value);\r
 313             return dict;\r
 314         }\r
 315 \r
 316     }\r
 317 \r
 318     internal static class PithosEAPCommon\r
 319     {\r
 320         internal static void HandleProgress<T, E>(TaskCompletionSource<T> tcs, ProgressChangedEventArgs eventArgs, Func<E> getProgress, IProgress<E> callback)\r
 321         {\r
 322             if (eventArgs.UserState != tcs)\r
 323                 return;\r
 324             callback.Report(getProgress());\r
 325         }\r
 326 \r
 327         internal static void HandleCompletion<T>(TaskCompletionSource<T> tcs, bool requireMatch, AsyncCompletedEventArgs e, Func<T> getResult, Action unregisterHandler)\r
 328         {\r
 329             if (requireMatch)\r
 330             {\r
 331                 if (e.UserState != tcs)\r
 332                     return;\r
 333             }\r
 334             try\r
 335             {\r
 336                 unregisterHandler();\r
 337             }\r
 338             finally\r
 339             {\r
 340                 if (e.Cancelled)\r
 341                     tcs.TrySetCanceled();\r
 342                 else if (e.Error != null)\r
 343                     tcs.TrySetException(e.Error);\r
 344                 else\r
 345                     tcs.TrySetResult(getResult());\r
 346             }\r
 347         }\r
 348     }\r
 349 \r
 350 }\r