root / trunk / Pithos.Network / WebExtensions.cs @ 2115e2a5
History | View | Annotate | Download (19.9 kB)
1 |
using System; |
---|---|
2 |
using System.Collections.Generic; |
3 |
using System.ComponentModel; |
4 |
using System.Diagnostics.Contracts; |
5 |
using System.Linq; |
6 |
using System.Net.Http; |
7 |
using System.Net.Http.Headers; |
8 |
using System.Reflection; |
9 |
using System.Text; |
10 |
using System.Net; |
11 |
using System.IO; |
12 |
using System.Threading; |
13 |
using System.Threading.Tasks; |
14 |
using Pithos.Interfaces; |
15 |
using log4net; |
16 |
using System.ServiceModel.Channels; |
17 |
using Microsoft.Win32; |
18 |
|
19 |
namespace Pithos.Network |
20 |
{ |
21 |
public static class WebExtensions |
22 |
{ |
23 |
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
24 |
|
25 |
public static string ReadToEnd(this HttpWebResponse response) |
26 |
{ |
27 |
using (var stream = response.GetResponseStream()) |
28 |
{ |
29 |
if (stream == null) |
30 |
return null; |
31 |
using (var reader = new StreamReader(stream)) |
32 |
{ |
33 |
var body = reader.ReadToEnd(); |
34 |
return body; |
35 |
} |
36 |
} |
37 |
} |
38 |
|
39 |
public static void LogError(this ILog log, HttpWebResponse response) |
40 |
{ |
41 |
if (log.IsDebugEnabled) |
42 |
{ |
43 |
if (response != null) |
44 |
{ |
45 |
var body = response.ReadToEnd(); |
46 |
log.ErrorFormat("Headers:\n{0}\nBody:{1}", response.Headers, body); |
47 |
} |
48 |
} |
49 |
} |
50 |
|
51 |
public static TextReader GetLoggedReader(this Stream stream, ILog log) |
52 |
{ |
53 |
var reader = new StreamReader(stream); |
54 |
if (!log.IsDebugEnabled) |
55 |
return reader; |
56 |
|
57 |
using (reader) |
58 |
{ |
59 |
var body = reader.ReadToEnd(); |
60 |
log.DebugFormat("JSON response: {0}", body); |
61 |
return new StringReader(body); |
62 |
} |
63 |
} |
64 |
|
65 |
|
66 |
public static IEnumerable<T> Range<T>(this IList<T> list, int start, int end) |
67 |
{ |
68 |
Contract.Requires(start >= 0); |
69 |
Contract.Requires(end < list.Count); |
70 |
Contract.Requires(start <= end); |
71 |
|
72 |
if (list == null) |
73 |
yield break; |
74 |
|
75 |
for (var i = 0; i <= end; i++) |
76 |
{ |
77 |
yield return list[i]; |
78 |
} |
79 |
|
80 |
} |
81 |
|
82 |
public static Task<byte[]> UploadDataTaskAsync(this WebClient webClient, Uri address, string method, byte[] data, CancellationToken cancellationToken, IProgress<UploadProgressChangedEventArgs> progress) |
83 |
{ |
84 |
var tcs = new TaskCompletionSource<byte[]>(address); |
85 |
if (cancellationToken.IsCancellationRequested) |
86 |
{ |
87 |
tcs.TrySetCanceled(); |
88 |
} |
89 |
else |
90 |
{ |
91 |
CancellationTokenRegistration ctr = cancellationToken.Register(()=> |
92 |
{ |
93 |
webClient.CancelAsync(); |
94 |
}); |
95 |
UploadDataCompletedEventHandler completedHandler = null; |
96 |
UploadProgressChangedEventHandler progressHandler = null; |
97 |
if (progress != null) |
98 |
progressHandler = (s, e) => PithosEAPCommon.HandleProgress(tcs, e, () => e, progress); |
99 |
completedHandler =(sender, e) =>PithosEAPCommon.HandleCompletion(tcs, true, e,() => e.Result,() => |
100 |
{ |
101 |
ctr.Dispose(); |
102 |
webClient.UploadDataCompleted -= completedHandler; |
103 |
webClient.UploadProgressChanged -= progressHandler; |
104 |
}); |
105 |
webClient.UploadDataCompleted += completedHandler; |
106 |
webClient.UploadProgressChanged += progressHandler; |
107 |
try |
108 |
{ |
109 |
webClient.UploadDataAsync(address, method, data, tcs); |
110 |
if (cancellationToken.IsCancellationRequested) |
111 |
webClient.CancelAsync(); |
112 |
} |
113 |
catch |
114 |
{ |
115 |
webClient.UploadDataCompleted -= completedHandler; |
116 |
webClient.UploadProgressChanged -= progressHandler; |
117 |
throw; |
118 |
} |
119 |
} |
120 |
return tcs.Task; |
121 |
} |
122 |
|
123 |
public static async Task<T> WithRetries<T>(this Func<Task<T>> func, int retries) |
124 |
{ |
125 |
while (retries > 0) |
126 |
{ |
127 |
try |
128 |
{ |
129 |
var result = await func().ConfigureAwait(false); |
130 |
return result; |
131 |
} |
132 |
catch (Exception exc) |
133 |
{ |
134 |
if (--retries == 0) |
135 |
throw new RetryException("Failed too many times", exc); |
136 |
} |
137 |
} |
138 |
throw new RetryException(); |
139 |
} |
140 |
|
141 |
public static async Task<HttpResponseMessage> WithRetriesForWeb(this Func<Task<HttpResponseMessage>> func, int retries) |
142 |
{ |
143 |
var waitTime = TimeSpan.FromSeconds(10); |
144 |
var acceptedCodes = new[] { HttpStatusCode.Moved, HttpStatusCode.MovedPermanently, HttpStatusCode.Found, HttpStatusCode.Redirect,HttpStatusCode.SeeOther, |
145 |
HttpStatusCode.RedirectMethod,HttpStatusCode.NotModified,HttpStatusCode.TemporaryRedirect,HttpStatusCode.RedirectKeepVerb}; |
146 |
while (retries > 0) |
147 |
{ |
148 |
var result = await func().ConfigureAwait(false); |
149 |
if (result.IsSuccessStatusCode || acceptedCodes.Contains(result.StatusCode)) |
150 |
return result; |
151 |
|
152 |
if (--retries == 0) |
153 |
throw new RetryException("Failed too many times"); |
154 |
|
155 |
//Wait for service unavailable |
156 |
if (result.StatusCode == HttpStatusCode.ServiceUnavailable) |
157 |
{ |
158 |
Log.InfoFormat("[UNAVAILABLE] Waiting before retry: {0}",result.ReasonPhrase); |
159 |
await TaskEx.Delay(waitTime).ConfigureAwait(false); |
160 |
//increase the timeout for repeated timeouts |
161 |
if (waitTime<TimeSpan.FromSeconds(10)) |
162 |
waitTime = waitTime.Add(TimeSpan.FromSeconds(10)); |
163 |
} |
164 |
//Throw in all other cases |
165 |
else |
166 |
result.EnsureSuccessStatusCode(); |
167 |
} |
168 |
throw new RetryException(); |
169 |
} |
170 |
|
171 |
|
172 |
public static async Task<string> GetStringAsyncWithRetries(this HttpClient client, Uri requestUri, int retries,DateTime? since=null) |
173 |
{ |
174 |
var request = new HttpRequestMessage(HttpMethod.Get, requestUri); |
175 |
if (since.HasValue) |
176 |
{ |
177 |
request.Headers.IfModifiedSince = since.Value; |
178 |
} |
179 |
//Func<Task<HttpResponseMessage>> call = () => _baseHttpClient.SendAsync(request); |
180 |
using (var response = await client.SendAsyncWithRetries(request,3).ConfigureAwait(false)) |
181 |
{ |
182 |
if (response.StatusCode == HttpStatusCode.NoContent) |
183 |
return String.Empty; |
184 |
|
185 |
var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); |
186 |
return content; |
187 |
} |
188 |
|
189 |
} |
190 |
|
191 |
public static Task<HttpResponseMessage> HeadAsyncWithRetries(this HttpClient client, Uri requestUri, int retries,bool acceptNotFound=false) |
192 |
{ |
193 |
return client.HeadAsyncWithRetries(requestUri, retries, acceptNotFound,HttpCompletionOption.ResponseContentRead, CancellationToken.None); |
194 |
} |
195 |
|
196 |
public static Task<HttpResponseMessage> HeadAsyncWithRetries(this HttpClient client, Uri requestUri, int retries, bool acceptNotFound,HttpCompletionOption completionOption, CancellationToken cancellationToken) |
197 |
{ |
198 |
return client.SendAsyncWithRetries(new HttpRequestMessage(HttpMethod.Head, requestUri), retries, acceptNotFound,completionOption, cancellationToken); |
199 |
} |
200 |
|
201 |
public static Task<HttpResponseMessage> GetAsyncWithRetries(this HttpClient client, Uri requestUri, int retries) |
202 |
{ |
203 |
return client.GetAsyncWithRetries(requestUri, retries, HttpCompletionOption.ResponseContentRead, CancellationToken.None); |
204 |
} |
205 |
|
206 |
public static Task<HttpResponseMessage> GetAsyncWithRetries(this HttpClient client, Uri requestUri, int retries, HttpCompletionOption completionOption, CancellationToken cancellationToken) |
207 |
{ |
208 |
return client.SendAsyncWithRetries(new HttpRequestMessage(HttpMethod.Get, requestUri), retries, false,completionOption, cancellationToken); |
209 |
} |
210 |
|
211 |
|
212 |
public static Task<HttpResponseMessage> SendAsyncWithRetries(this HttpClient client,HttpRequestMessage message, int retries) |
213 |
{ |
214 |
return client.SendAsyncWithRetries(message, retries,false, HttpCompletionOption.ResponseContentRead, CancellationToken.None); |
215 |
} |
216 |
|
217 |
public static async Task<HttpResponseMessage> SendAsyncWithRetries(this HttpClient client,HttpRequestMessage message, int retries,bool acceptNotFound,HttpCompletionOption completionOption, CancellationToken cancellationToken) |
218 |
{ |
219 |
var waitTime = TimeSpan.FromSeconds(10); |
220 |
var acceptedCodes = new HashSet<HttpStatusCode> { HttpStatusCode.Moved, HttpStatusCode.MovedPermanently, |
221 |
HttpStatusCode.Found, HttpStatusCode.Redirect,HttpStatusCode.SeeOther,HttpStatusCode.RedirectMethod, |
222 |
HttpStatusCode.NotModified,HttpStatusCode.TemporaryRedirect,HttpStatusCode.RedirectKeepVerb, |
223 |
HttpStatusCode.Conflict}; |
224 |
if (acceptNotFound) |
225 |
acceptedCodes.Add(HttpStatusCode.NotFound); |
226 |
|
227 |
while (retries > 0) |
228 |
{ |
229 |
var timedOut = false; |
230 |
if (Log.IsDebugEnabled) |
231 |
Log.DebugFormat("[REQUEST] {0}", message); |
232 |
HttpResponseMessage result=null; |
233 |
|
234 |
|
235 |
var msg = message.Clone(); |
236 |
Exception innerException=null; |
237 |
try |
238 |
{ |
239 |
result = await client.SendAsync(msg, completionOption, cancellationToken).ConfigureAwait(false); |
240 |
if (result.IsSuccessStatusCode || acceptedCodes.Contains(result.StatusCode)) |
241 |
{ |
242 |
if (Log.IsDebugEnabled) |
243 |
Log.DebugFormat("[RESPONSE] [{0}]:[{1}] OK: [{2}]", msg.Method, msg.RequestUri, |
244 |
result.StatusCode); |
245 |
return result; |
246 |
} |
247 |
|
248 |
} |
249 |
catch (HttpRequestException exc) |
250 |
{ |
251 |
//A timeout or other error caused a failure without receiving a response from the server |
252 |
if (Log.IsDebugEnabled) |
253 |
Log.DebugFormat("[RESPONSE] [{0}]:[{1}] HTTP FAIL :\n{2}", msg.Method, msg.RequestUri,exc); |
254 |
innerException = exc; |
255 |
} |
256 |
catch (WebException exc) |
257 |
{ |
258 |
if (exc.Status != WebExceptionStatus.Timeout) |
259 |
throw; |
260 |
timedOut = true; |
261 |
if (Log.IsDebugEnabled) |
262 |
Log.DebugFormat("[RESPONSE] [{0}]:[{1}] FAIL WITH TIMEOUT", msg.Method, msg.RequestUri); |
263 |
innerException = exc; |
264 |
} |
265 |
catch(TaskCanceledException exc) |
266 |
{ |
267 |
//If the task was cancelled due to a timeout, retry it |
268 |
if (!exc.CancellationToken.IsCancellationRequested) |
269 |
{ |
270 |
timedOut = true; |
271 |
if (Log.IsDebugEnabled) |
272 |
Log.DebugFormat("[RESPONSE] [{0}]:[{1}] FAIL WITH TIMEOUT", msg.Method, msg.RequestUri); |
273 |
} |
274 |
else |
275 |
{ |
276 |
throw; |
277 |
} |
278 |
innerException = exc; |
279 |
} |
280 |
catch (Exception exc) |
281 |
{ |
282 |
Log.FatalFormat("Unexpected error while sending:\n{0}\n{1}", msg, exc); |
283 |
throw; |
284 |
} |
285 |
|
286 |
//Report the failure |
287 |
var resultStatus = result.NullSafe(s => s.StatusCode); |
288 |
|
289 |
if (Log.IsDebugEnabled) |
290 |
{ |
291 |
if (timedOut) |
292 |
Log.DebugFormat("[TIMEOUT] [{0}]:[{1}]", msg.Method, msg.RequestUri); |
293 |
else |
294 |
Log.DebugFormat("[RESPONSE] [{0}]:[{1}] FAIL: [{2}]\n{3}", msg.Method, msg.RequestUri, |
295 |
resultStatus, result); |
296 |
} |
297 |
|
298 |
//Handle the failure |
299 |
|
300 |
if (--retries == 0) |
301 |
throw new RetryException("Failed too many times"); |
302 |
|
303 |
//Wait for service unavailable |
304 |
if (resultStatus == HttpStatusCode.ServiceUnavailable || |
305 |
resultStatus == HttpStatusCode.BadGateway || |
306 |
resultStatus == HttpStatusCode.InternalServerError ) |
307 |
{ |
308 |
|
309 |
Log.WarnFormat("[UNAVAILABLE] Waiting before retrying [{0}]:[{1}] due to [{2}]", msg.Method, |
310 |
msg.RequestUri, result.ReasonPhrase); |
311 |
await TaskEx.Delay(waitTime).ConfigureAwait(false); |
312 |
//increase the timeout for repeated timeouts |
313 |
if (waitTime < TimeSpan.FromSeconds(10)) |
314 |
waitTime = waitTime.Add(TimeSpan.FromSeconds(10)); |
315 |
} |
316 |
//Throw in all other cases, unless there was a client-side timeout |
317 |
else if (!timedOut) |
318 |
//Throw if there was no result (never received a reply) |
319 |
if (result == null) |
320 |
throw innerException ?? new RetryException(); |
321 |
else |
322 |
//Otherwise force the exception that corresponds to the response |
323 |
throw new HttpRequestWithStatusException(result.StatusCode,result.ReasonPhrase); |
324 |
} |
325 |
throw new RetryException(); |
326 |
} |
327 |
|
328 |
public static HttpRequestMessage Clone(this HttpRequestMessage message) |
329 |
{ |
330 |
var newMessage = new HttpRequestMessage(message.Method, message.RequestUri); |
331 |
foreach (var header in message.Headers) |
332 |
{ |
333 |
newMessage.Headers.Add(header.Key, header.Value); |
334 |
} |
335 |
newMessage.Content = message.Content.Clone(); |
336 |
|
337 |
foreach (var property in message.Properties) |
338 |
{ |
339 |
newMessage.Properties.Add(property.Key, property.Value); |
340 |
} |
341 |
return newMessage; |
342 |
} |
343 |
|
344 |
public static HttpContent Clone(this HttpContent content) |
345 |
{ |
346 |
if (content == null) |
347 |
return null; |
348 |
if (!(content is FileBlockContent || |
349 |
content is ByteArrayContentWithProgress || |
350 |
content is StringContent || |
351 |
content is ByteArrayContent)) |
352 |
throw new ArgumentException("content"); |
353 |
|
354 |
HttpContent newContent=null; |
355 |
if (content is FileBlockContent) |
356 |
{ |
357 |
newContent = (content as FileBlockContent).Clone(); |
358 |
} |
359 |
else if (content is ByteArrayContentWithProgress) |
360 |
{ |
361 |
newContent=(content as ByteArrayContentWithProgress).Clone(); |
362 |
} |
363 |
else if (content is StringContent) |
364 |
{ |
365 |
var fieldInfo = typeof(ByteArrayContent).GetField("content", |
366 |
BindingFlags.NonPublic | BindingFlags.Instance); |
367 |
var bytes = (byte[])fieldInfo.GetValue(content); |
368 |
var enc=Encoding.GetEncoding(content.Headers.ContentType.CharSet ?? "utf-8"); |
369 |
var stringContent=enc.GetString(bytes); |
370 |
newContent = new StringContent(stringContent); |
371 |
} |
372 |
else if (content is ByteArrayContent) |
373 |
{ |
374 |
var fieldInfo = typeof(ByteArrayContent).GetField("content", |
375 |
BindingFlags.NonPublic | BindingFlags.Instance); |
376 |
var bytes = (byte[])fieldInfo.GetValue(content); |
377 |
newContent = new ByteArrayContent(bytes); |
378 |
} |
379 |
|
380 |
|
381 |
foreach (var header in content.Headers) |
382 |
{ |
383 |
if (header.Key == "Content-Type") |
384 |
newContent.Headers.ContentType=content.Headers.ContentType; |
385 |
else if (newContent.Headers.Contains(header.Key)) |
386 |
newContent.Headers.Add(header.Key, header.Value); |
387 |
} |
388 |
|
389 |
return newContent; |
390 |
} |
391 |
|
392 |
public static string GetFirstValue(this HttpResponseHeaders headers, string name) |
393 |
{ |
394 |
if (headers==null) |
395 |
throw new ArgumentNullException("headers"); |
396 |
if (String.IsNullOrWhiteSpace(name)) |
397 |
throw new ArgumentNullException("name"); |
398 |
Contract.EndContractBlock(); |
399 |
|
400 |
IEnumerable<string> values; |
401 |
if (headers.TryGetValues(name, out values)) |
402 |
{ |
403 |
return values.FirstOrDefault(); |
404 |
} |
405 |
return null; |
406 |
} |
407 |
|
408 |
public static Dictionary<string, string> GetMeta(this HttpResponseHeaders headers,string metaPrefix) |
409 |
{ |
410 |
Contract.Requires<ArgumentNullException>(headers!=null,"headers"); |
411 |
Contract.Requires<ArgumentNullException>(!String.IsNullOrWhiteSpace(metaPrefix), "metaPrefix"); |
412 |
Contract.EndContractBlock(); |
413 |
|
414 |
var dict = (from header in headers |
415 |
where header.Key.StartsWith(metaPrefix) |
416 |
let name = header.Key.Substring(metaPrefix.Length) |
417 |
select new { Name = name, Value = String.Join(",",header.Value) }) |
418 |
.ToDictionary(t => t.Name, t => t.Value); |
419 |
return dict; |
420 |
} |
421 |
|
422 |
public static MediaTypeHeaderValue GetMimeType(this Uri uri) |
423 |
{ |
424 |
var extension=Path.GetExtension(uri.IsAbsoluteUri? uri.AbsolutePath:uri.OriginalString).ToLower(); |
425 |
|
426 |
string mimeType = "application/octet-stream"; |
427 |
|
428 |
var regKey = Registry.ClassesRoot.OpenSubKey(extension); |
429 |
|
430 |
if (regKey != null) |
431 |
{ |
432 |
object contentType = regKey.GetValue("Content Type"); |
433 |
|
434 |
if (contentType != null) |
435 |
mimeType = contentType.ToString(); |
436 |
} |
437 |
|
438 |
return new MediaTypeHeaderValue(mimeType) ; |
439 |
} |
440 |
|
441 |
} |
442 |
|
443 |
internal static class PithosEAPCommon |
444 |
{ |
445 |
internal static void HandleProgress<T, E>(TaskCompletionSource<T> tcs, ProgressChangedEventArgs eventArgs, Func<E> getProgress, IProgress<E> callback) |
446 |
{ |
447 |
if (eventArgs.UserState != tcs) |
448 |
return; |
449 |
callback.Report(getProgress()); |
450 |
} |
451 |
|
452 |
internal static void HandleCompletion<T>(TaskCompletionSource<T> tcs, bool requireMatch, AsyncCompletedEventArgs e, Func<T> getResult, Action unregisterHandler) |
453 |
{ |
454 |
if (requireMatch) |
455 |
{ |
456 |
if (e.UserState != tcs) |
457 |
return; |
458 |
} |
459 |
try |
460 |
{ |
461 |
unregisterHandler(); |
462 |
} |
463 |
finally |
464 |
{ |
465 |
if (e.Cancelled) |
466 |
tcs.TrySetCanceled(); |
467 |
else if (e.Error != null) |
468 |
tcs.TrySetException(e.Error); |
469 |
else |
470 |
tcs.TrySetResult(getResult()); |
471 |
} |
472 |
} |
473 |
} |
474 |
|
475 |
} |