root / trunk / Pithos.Network / CloudFilesClient.cs @ 84b880b5
History | View | Annotate | Download (88.4 kB)
1 |
#region |
---|---|
2 |
/* ----------------------------------------------------------------------- |
3 |
* <copyright file="CloudFilesClient.cs" company="GRNet"> |
4 |
* |
5 |
* Copyright 2011-2012 GRNET S.A. All rights reserved. |
6 |
* |
7 |
* Redistribution and use in source and binary forms, with or |
8 |
* without modification, are permitted provided that the following |
9 |
* conditions are met: |
10 |
* |
11 |
* 1. Redistributions of source code must retain the above |
12 |
* copyright notice, this list of conditions and the following |
13 |
* disclaimer. |
14 |
* |
15 |
* 2. Redistributions in binary form must reproduce the above |
16 |
* copyright notice, this list of conditions and the following |
17 |
* disclaimer in the documentation and/or other materials |
18 |
* provided with the distribution. |
19 |
* |
20 |
* |
21 |
* THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
22 |
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
23 |
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
24 |
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
25 |
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
26 |
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
27 |
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
28 |
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
29 |
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
30 |
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
31 |
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
32 |
* POSSIBILITY OF SUCH DAMAGE. |
33 |
* |
34 |
* The views and conclusions contained in the software and |
35 |
* documentation are those of the authors and should not be |
36 |
* interpreted as representing official policies, either expressed |
37 |
* or implied, of GRNET S.A. |
38 |
* </copyright> |
39 |
* ----------------------------------------------------------------------- |
40 |
*/ |
41 |
#endregion |
42 |
|
43 |
// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos |
44 |
// |
45 |
// The class provides methods to upload/download files, delete files, manage containers |
46 |
|
47 |
|
48 |
using System; |
49 |
using System.Collections.Generic; |
50 |
using System.Collections.Specialized; |
51 |
using System.ComponentModel.Composition; |
52 |
using System.Diagnostics; |
53 |
using System.Diagnostics.Contracts; |
54 |
using System.IO; |
55 |
using System.Linq; |
56 |
using System.Net; |
57 |
using System.Net.Http; |
58 |
using System.Net.Http.Headers; |
59 |
using System.Reflection; |
60 |
using System.ServiceModel.Channels; |
61 |
using System.Text; |
62 |
using System.Threading; |
63 |
using System.Threading.Tasks; |
64 |
using Newtonsoft.Json; |
65 |
using Newtonsoft.Json.Linq; |
66 |
using Pithos.Interfaces; |
67 |
using Pithos.Network; |
68 |
using log4net; |
69 |
|
70 |
namespace Pithos.Network |
71 |
{ |
72 |
|
73 |
[Export(typeof(ICloudClient))] |
74 |
public class CloudFilesClient:ICloudClient,IDisposable |
75 |
{ |
76 |
private const string TOKEN_HEADER = "X-Auth-Token"; |
77 |
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
78 |
|
79 |
//CloudFilesClient uses *_baseClient* internally to communicate with the server |
80 |
//RestClient provides a REST-friendly interface over the standard WebClient. |
81 |
private RestClient _baseClient; |
82 |
|
83 |
private HttpClient _baseHttpClient; |
84 |
private HttpClient _baseHttpClientNoTimeout; |
85 |
|
86 |
|
87 |
//During authentication the client provides a UserName |
88 |
public string UserName { get; set; } |
89 |
|
90 |
//and and ApiKey to the server |
91 |
public string ApiKey { get; set; } |
92 |
|
93 |
//And receives an authentication Token. This token must be provided in ALL other operations, |
94 |
//in the X-Auth-Token header |
95 |
private string _token; |
96 |
private readonly string _emptyGuid = Guid.Empty.ToString(); |
97 |
private readonly Uri _emptyUri = new Uri("",UriKind.Relative); |
98 |
|
99 |
private HttpClientHandler _httpClientHandler = new HttpClientHandler |
100 |
{ |
101 |
AllowAutoRedirect = true, |
102 |
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, |
103 |
UseCookies = true, |
104 |
}; |
105 |
|
106 |
|
107 |
public string Token |
108 |
{ |
109 |
get { return _token; } |
110 |
set |
111 |
{ |
112 |
_token = value; |
113 |
_baseClient.Headers[TOKEN_HEADER] = value; |
114 |
} |
115 |
} |
116 |
|
117 |
//The client also receives a StorageUrl after authentication. All subsequent operations must |
118 |
//use this url |
119 |
public Uri StorageUrl { get; set; } |
120 |
|
121 |
|
122 |
public Uri RootAddressUri |
123 |
{ |
124 |
get { return _rootAddressUri; } |
125 |
set { _rootAddressUri = value; } |
126 |
} |
127 |
|
128 |
|
129 |
public double DownloadPercentLimit { get; set; } |
130 |
public double UploadPercentLimit { get; set; } |
131 |
|
132 |
public string AuthenticationUrl { get; set; } |
133 |
|
134 |
|
135 |
public string VersionPath |
136 |
{ |
137 |
get { return UsePithos ? "v1" : "v1.0"; } |
138 |
} |
139 |
|
140 |
public bool UsePithos { get; set; } |
141 |
|
142 |
|
143 |
BufferManager _bufferManager=BufferManager.CreateBufferManager(TreeHash.DEFAULT_BLOCK_SIZE*4,(int)TreeHash.DEFAULT_BLOCK_SIZE); |
144 |
private string _userCatalogUrl; |
145 |
private Uri _rootAddressUri; |
146 |
|
147 |
public CloudFilesClient(string userName, string apiKey) |
148 |
{ |
149 |
UserName = userName; |
150 |
ApiKey = apiKey; |
151 |
_userCatalogUrl = "https://pithos.okeanos.io/user_catalogs"; |
152 |
} |
153 |
|
154 |
public CloudFilesClient(AccountInfo accountInfo) |
155 |
{ |
156 |
Contract.Requires<ArgumentNullException>(accountInfo!=null,"accountInfo is null"); |
157 |
Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); |
158 |
Contract.Ensures(StorageUrl != null); |
159 |
Contract.Ensures(_baseClient != null); |
160 |
Contract.Ensures(RootAddressUri != null); |
161 |
Contract.EndContractBlock(); |
162 |
|
163 |
_baseClient = new RestClient |
164 |
{ |
165 |
BaseAddress = accountInfo.StorageUri.ToString(), |
166 |
Timeout = 30000, |
167 |
Retries = 3, |
168 |
}; |
169 |
StorageUrl = accountInfo.StorageUri; |
170 |
Token = accountInfo.Token; |
171 |
UserName = accountInfo.UserName; |
172 |
_userCatalogUrl = "https://pithos.okeanos.io/user_catalogs"; |
173 |
//Get the root address (StorageUrl without the account) |
174 |
var storageUrl = StorageUrl.AbsoluteUri; |
175 |
var usernameIndex = storageUrl.LastIndexOf(UserName); |
176 |
var rootUrl = storageUrl.Substring(0, usernameIndex); |
177 |
RootAddressUri = new Uri(rootUrl); |
178 |
//RootAddressUri = new Uri(storageUrl); |
179 |
|
180 |
var httpClientHandler = new HttpClientHandler |
181 |
{ |
182 |
AllowAutoRedirect = true, |
183 |
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, |
184 |
UseCookies = true, |
185 |
}; |
186 |
|
187 |
|
188 |
_baseHttpClient = new HttpClient(httpClientHandler) |
189 |
{ |
190 |
BaseAddress = StorageUrl, |
191 |
Timeout = TimeSpan.FromSeconds(30) |
192 |
}; |
193 |
_baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, Token); |
194 |
|
195 |
_baseHttpClientNoTimeout = new HttpClient(httpClientHandler) |
196 |
{ |
197 |
BaseAddress = StorageUrl, |
198 |
Timeout = TimeSpan.FromMilliseconds(-1) |
199 |
}; |
200 |
_baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, Token); |
201 |
|
202 |
|
203 |
} |
204 |
|
205 |
|
206 |
private static void AssertStatusOK(HttpResponseMessage response, string message) |
207 |
{ |
208 |
var statusCode = response.StatusCode; |
209 |
if (statusCode >= HttpStatusCode.BadRequest) |
210 |
throw new WebException(String.Format("{0} with code {1} - {2}", message, statusCode, response.ReasonPhrase)); |
211 |
} |
212 |
|
213 |
public async Task<AccountInfo> Authenticate() |
214 |
{ |
215 |
if (String.IsNullOrWhiteSpace(UserName)) |
216 |
throw new InvalidOperationException("UserName is empty"); |
217 |
if (String.IsNullOrWhiteSpace(ApiKey)) |
218 |
throw new InvalidOperationException("ApiKey is empty"); |
219 |
if (String.IsNullOrWhiteSpace(AuthenticationUrl)) |
220 |
throw new InvalidOperationException("AuthenticationUrl is empty"); |
221 |
//if (String.IsNullOrWhiteSpace(Token)) |
222 |
// throw new InvalidOperationException("Token is Empty"); |
223 |
|
224 |
//Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); |
225 |
//Contract.Ensures(StorageUrl != null); |
226 |
//Contract.Ensures(_baseClient != null); |
227 |
//Contract.Ensures(RootAddressUri != null); |
228 |
//Contract.EndContractBlock(); |
229 |
|
230 |
|
231 |
Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName); |
232 |
|
233 |
var groups = new List<Group>(); |
234 |
|
235 |
// using (var authClient = new HttpClient(_httpClientHandler,false){ BaseAddress = new Uri(AuthenticationUrl),Timeout=TimeSpan.FromSeconds(30) }) |
236 |
using (var authClient = new HttpClient(_httpClientHandler, false) { BaseAddress = new Uri(AuthenticationUrl), Timeout = TimeSpan.FromSeconds(30) }) |
237 |
{ |
238 |
|
239 |
authClient.DefaultRequestHeaders.Add("X-Auth-User", UserName); |
240 |
authClient.DefaultRequestHeaders.Add("X-Auth-Key", ApiKey); |
241 |
|
242 |
|
243 |
string token; |
244 |
string objectStorageService; |
245 |
|
246 |
//using (var response = await authClient.GetAsyncWithRetries(new Uri(VersionPath, UriKind.Relative),3).ConfigureAwait(false)) // .DownloadStringWithRetryRelative(new Uri(VersionPath, UriKind.Relative), 3); |
247 |
using (var response = await authClient.SendAsyncWithRetries(new HttpRequestMessage(HttpMethod.Post,new Uri("", UriKind.Relative)), 3).ConfigureAwait(false)) |
248 |
{ |
249 |
AssertStatusOK(response,"Authentication failed"); |
250 |
|
251 |
var body=await response.Content.ReadAsStringAsync(); |
252 |
JObject json = (JObject) JsonConvert.DeserializeObject(body); |
253 |
dynamic jsonD = json; |
254 |
|
255 |
objectStorageService = json["access"]["serviceCatalog"][9]["endpoints"][0]["publicURL"].ToString(); |
256 |
Log.InfoFormat("[ServicePoint] {0}", json["access"]["serviceCatalog"][0]["endpoints"][0]["publicURL"].ToString()); |
257 |
Log.InfoFormat("[ServicePoint] {0}", json["access"]["serviceCatalog"][1]["endpoints"][0]["publicURL"].ToString()); |
258 |
Log.InfoFormat("[ServicePoint] {0}", json["access"]["serviceCatalog"][2]["endpoints"][0]["publicURL"].ToString()); |
259 |
Log.InfoFormat("[ServicePoint] {0}", json["access"]["serviceCatalog"][3]["endpoints"][0]["publicURL"].ToString()); |
260 |
Log.InfoFormat("[ServicePoint] {0}", json["access"]["serviceCatalog"][4]["endpoints"][0]["publicURL"].ToString()); |
261 |
Log.InfoFormat("[ServicePoint] {0}", json["access"]["serviceCatalog"][5]["endpoints"][0]["publicURL"].ToString()); |
262 |
Log.InfoFormat("[ServicePoint] {0}", json["access"]["serviceCatalog"][6]["endpoints"][0]["publicURL"].ToString()); |
263 |
Log.InfoFormat("[ServicePoint] {0}", json["access"]["serviceCatalog"][7]["endpoints"][0]["publicURL"].ToString()); |
264 |
Log.InfoFormat("[ServicePoint] {0}", json["access"]["serviceCatalog"][8]["endpoints"][0]["publicURL"].ToString()); |
265 |
Log.InfoFormat("[ServicePoint] {0}", json["access"]["serviceCatalog"][9]["endpoints"][0]["publicURL"].ToString()); |
266 |
|
267 |
if (String.IsNullOrWhiteSpace(objectStorageService)) |
268 |
throw new InvalidOperationException("Failed to obtain storage url"); |
269 |
|
270 |
token = this.ApiKey;// response.Headers.GetFirstValue(TOKEN_HEADER); |
271 |
if (String.IsNullOrWhiteSpace(token)) |
272 |
throw new InvalidOperationException("Failed to obtain token url"); |
273 |
} |
274 |
|
275 |
string storageUrl = new Uri(objectStorageService).Combine(UserName).ToString(); |
276 |
_baseClient = new RestClient |
277 |
{ |
278 |
BaseAddress = storageUrl, |
279 |
Timeout = 30000, |
280 |
Retries = 3, |
281 |
}; |
282 |
|
283 |
StorageUrl = new Uri(storageUrl); |
284 |
Token = token; |
285 |
|
286 |
//Get the root address (StorageUrl without the account) |
287 |
var rootUrl = objectStorageService;//.Substring(0, usernameIndex); |
288 |
RootAddressUri = new Uri(rootUrl); |
289 |
|
290 |
|
291 |
_baseHttpClient = new HttpClient(_httpClientHandler,false) |
292 |
{ |
293 |
BaseAddress = StorageUrl, |
294 |
Timeout = TimeSpan.FromSeconds(30) |
295 |
}; |
296 |
_baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, token); |
297 |
|
298 |
_baseHttpClientNoTimeout = new HttpClient(_httpClientHandler,false) |
299 |
{ |
300 |
BaseAddress = StorageUrl, |
301 |
Timeout = TimeSpan.FromMilliseconds(-1) |
302 |
}; |
303 |
_baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, token); |
304 |
|
305 |
/* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable(); |
306 |
groups = (from key in keys |
307 |
where key.StartsWith("X-Account-Group-") |
308 |
let name = key.Substring(16) |
309 |
select new Group(name, authClient.ResponseHeaders[key])) |
310 |
.ToList(); |
311 |
|
312 |
*/ |
313 |
} |
314 |
|
315 |
Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName); |
316 |
Debug.Assert(_baseClient!=null); |
317 |
var displayName = UserName; |
318 |
Guid uuid; |
319 |
if (Guid.TryParse(UserName, out uuid)) |
320 |
{ |
321 |
displayName = await ResolveName(uuid); |
322 |
} |
323 |
return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName, DisplayName=displayName, Groups=groups}; |
324 |
|
325 |
} |
326 |
|
327 |
private static void TraceStart(string method, Uri actualAddress) |
328 |
{ |
329 |
Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress); |
330 |
} |
331 |
|
332 |
private async Task<string> GetStringAsync(Uri targetUri, string errorMessage,DateTimeOffset? since=null) |
333 |
{ |
334 |
TraceStart("GET",targetUri); |
335 |
var request = new HttpRequestMessage(HttpMethod.Get, targetUri); |
336 |
//request.Headers.Add("User-Agent", "Pithos+ Custom Header"); |
337 |
if (since.HasValue) |
338 |
{ |
339 |
request.Headers.IfModifiedSince = since.Value; |
340 |
} |
341 |
using (var response = await _baseHttpClient.SendAsyncWithRetries(request,3).ConfigureAwait(false)) |
342 |
{ |
343 |
AssertStatusOK(response, errorMessage); |
344 |
|
345 |
if (response.StatusCode == HttpStatusCode.NoContent) |
346 |
return String.Empty; |
347 |
|
348 |
var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); |
349 |
return content; |
350 |
} |
351 |
} |
352 |
|
353 |
public async Task<IList<ContainerInfo>> ListContainers(string account) |
354 |
{ |
355 |
|
356 |
var targetUrl = GetTargetUrl(account); |
357 |
var targetUri = new Uri(String.Format("{0}?format=json", targetUrl)); |
358 |
var result = await GetStringAsync(targetUri, "List Containers failed").ConfigureAwait(false); |
359 |
if (String.IsNullOrWhiteSpace(result)) |
360 |
return new List<ContainerInfo>(); |
361 |
var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(result); |
362 |
foreach (var info in infos) |
363 |
{ |
364 |
info.Account = account; |
365 |
} |
366 |
return infos; |
367 |
} |
368 |
|
369 |
|
370 |
private string GetAccountUrl(string account) |
371 |
{ |
372 |
return RootAddressUri.Combine(account).AbsoluteUri; |
373 |
} |
374 |
|
375 |
public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null) |
376 |
{ |
377 |
using (ThreadContext.Stacks["Share"].Push("List Accounts")) |
378 |
{ |
379 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
380 |
|
381 |
var targetUri = new Uri(String.Format("{0}?format=json", RootAddressUri), UriKind.Absolute); |
382 |
var content=TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListSharingAccounts failed", since).ConfigureAwait(false)).Result; |
383 |
|
384 |
//If the result is empty, return an empty list, |
385 |
var infos = String.IsNullOrWhiteSpace(content) |
386 |
? new List<ShareAccountInfo>() |
387 |
//Otherwise deserialize the account list into a list of ShareAccountInfos |
388 |
: JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content); |
389 |
|
390 |
Log.DebugFormat("END"); |
391 |
return infos; |
392 |
} |
393 |
} |
394 |
|
395 |
|
396 |
/// <summary> |
397 |
/// Request listing of all objects in a container modified since a specific time. |
398 |
/// If the *since* value is missing, return all objects |
399 |
/// </summary> |
400 |
/// <param name="knownContainers">Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new |
401 |
/// and should be polled anyway |
402 |
/// </param> |
403 |
/// <param name="since"></param> |
404 |
/// <returns></returns> |
405 |
public IList<ObjectInfo> ListSharedObjects(HashSet<string> knownContainers, DateTimeOffset? since) |
406 |
{ |
407 |
|
408 |
using (ThreadContext.Stacks["Share"].Push("List Objects")) |
409 |
{ |
410 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
411 |
//'since' is not used here because we need to have ListObjects return a NoChange result |
412 |
//for all shared accounts,containers |
413 |
|
414 |
Func<ContainerInfo, string> getKey = c => String.Format("{0}\\{1}", c.Account, c.Name); |
415 |
|
416 |
var containers = (from account in ListSharingAccounts() |
417 |
let conts = TaskEx.Run(async ()=>await ListContainers(account.name).ConfigureAwait(false)).Result |
418 |
from container in conts |
419 |
select container).ToList(); |
420 |
var items = from container in containers |
421 |
let actualSince=knownContainers.Contains(getKey(container))?since:null |
422 |
select ListObjects(container.Account , container.Name, actualSince); |
423 |
var objects=items.SelectMany(r=> r).ToList(); |
424 |
|
425 |
//For each object |
426 |
//Check parents recursively up to (but not including) the container. |
427 |
//If parents are missing, add them to the list |
428 |
//Need function to calculate all parent URLs |
429 |
objects = AddMissingParents(objects); |
430 |
|
431 |
//Store any new containers |
432 |
foreach (var container in containers) |
433 |
{ |
434 |
knownContainers.Add(getKey(container)); |
435 |
} |
436 |
|
437 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
438 |
return objects; |
439 |
} |
440 |
} |
441 |
|
442 |
private List<ObjectInfo> AddMissingParents(List<ObjectInfo> objects) |
443 |
{ |
444 |
//TODO: Remove short-circuit when we decide to use Missing Parents functionality |
445 |
//return objects; |
446 |
|
447 |
var existingUris = objects.ToDictionary(o => o.Uri, o => o); |
448 |
foreach (var objectInfo in objects) |
449 |
{ |
450 |
//Can be null when retrieving objects to show in selective sync |
451 |
if (objectInfo.Name == null) |
452 |
continue; |
453 |
|
454 |
//No need to unescape here, the parts will be used to create new ObjectInfos |
455 |
var parts = objectInfo.Name.ToString().Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries); |
456 |
//If there is no parent, skip |
457 |
if (parts.Length == 1) |
458 |
continue; |
459 |
var baseParts = new[] |
460 |
{ |
461 |
objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container.ToString() |
462 |
}; |
463 |
for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++) |
464 |
{ |
465 |
var nameparts = parts.Range(0, partIdx).ToArray(); |
466 |
var parentName= String.Join("/", nameparts); |
467 |
|
468 |
var parentParts = baseParts.Concat(nameparts); |
469 |
var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts); |
470 |
|
471 |
var parentUri = new Uri(parentUrl, UriKind.Absolute); |
472 |
|
473 |
ObjectInfo existingInfo; |
474 |
if (!existingUris.TryGetValue(parentUri,out existingInfo)) |
475 |
{ |
476 |
var h = parentUrl.GetHashCode(); |
477 |
var reverse = new string(parentUrl.Reverse().ToArray()); |
478 |
var rh = reverse.GetHashCode(); |
479 |
var b1 = BitConverter.GetBytes(h); |
480 |
var b2 = BitConverter.GetBytes(rh); |
481 |
var g = new Guid(0,0,0,b1.Concat(b2).ToArray()); |
482 |
|
483 |
existingUris[parentUri] = new ObjectInfo |
484 |
{ |
485 |
Account = objectInfo.Account, |
486 |
Container = objectInfo.Container, |
487 |
Content_Type = ObjectInfo.CONTENT_TYPE_DIRECTORY, |
488 |
ETag = Signature.MERKLE_EMPTY, |
489 |
X_Object_Hash = Signature.MERKLE_EMPTY, |
490 |
Name=new Uri(parentName,UriKind.Relative), |
491 |
StorageUri=objectInfo.StorageUri, |
492 |
Bytes = 0, |
493 |
UUID=g.ToString(), |
494 |
}; |
495 |
} |
496 |
} |
497 |
} |
498 |
return existingUris.Values.ToList(); |
499 |
} |
500 |
|
501 |
public void SetTags(ObjectInfo target,IDictionary<string,string> tags) |
502 |
{ |
503 |
Contract.Requires<InvalidOperationException>(!String.IsNullOrWhiteSpace(Token),"The Token is not set"); |
504 |
Contract.Requires<InvalidOperationException>(StorageUrl != null,"The StorageUrl is not set"); |
505 |
Contract.Requires<ArgumentNullException>(target != null,"target is null"); |
506 |
Contract.EndContractBlock(); |
507 |
|
508 |
using (ThreadContext.Stacks["Share"].Push("Share Object")) |
509 |
{ |
510 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
511 |
|
512 |
using (var client = new RestClient(_baseClient)) |
513 |
{ |
514 |
|
515 |
client.BaseAddress = GetAccountUrl(target.Account); |
516 |
|
517 |
client.Parameters.Clear(); |
518 |
client.Parameters.Add("update", ""); |
519 |
|
520 |
foreach (var tag in tags) |
521 |
{ |
522 |
var headerTag = String.Format("X-Object-Meta-{0}", tag.Key); |
523 |
client.Headers.Add(headerTag, tag.Value); |
524 |
} |
525 |
|
526 |
client.DownloadStringWithRetryRelative(target.Container, 3); |
527 |
|
528 |
|
529 |
client.AssertStatusOK("SetTags failed"); |
530 |
//If the status is NOT ACCEPTED we have a problem |
531 |
if (client.StatusCode != HttpStatusCode.Accepted) |
532 |
{ |
533 |
Log.Error("Failed to set tags"); |
534 |
throw new Exception("Failed to set tags"); |
535 |
} |
536 |
|
537 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
538 |
} |
539 |
} |
540 |
|
541 |
|
542 |
} |
543 |
|
544 |
public void ShareObject(string account, Uri container, Uri objectName, string shareTo, bool read, bool write) |
545 |
{ |
546 |
|
547 |
Contract.Requires<InvalidOperationException>(!String.IsNullOrWhiteSpace(Token), "The Token is not set"); |
548 |
Contract.Requires<InvalidOperationException>(StorageUrl != null, "The StorageUrl is not set"); |
549 |
Contract.Requires<ArgumentNullException>(container != null, "container is null"); |
550 |
Contract.Requires<ArgumentException>(!container.IsAbsoluteUri, "container is absolute"); |
551 |
Contract.Requires<ArgumentNullException>(objectName != null, "objectName is null"); |
552 |
Contract.Requires<ArgumentException>(!objectName.IsAbsoluteUri, "objectName is absolute"); |
553 |
Contract.Requires<ArgumentNullException>(!String.IsNullOrWhiteSpace(account), "account is not set"); |
554 |
Contract.Requires<ArgumentNullException>(!String.IsNullOrWhiteSpace(shareTo), "shareTo is not set"); |
555 |
Contract.EndContractBlock(); |
556 |
|
557 |
using (ThreadContext.Stacks["Share"].Push("Share Object")) |
558 |
{ |
559 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
560 |
|
561 |
using (var client = new RestClient(_baseClient)) |
562 |
{ |
563 |
|
564 |
client.BaseAddress = GetAccountUrl(account); |
565 |
|
566 |
client.Parameters.Clear(); |
567 |
client.Parameters.Add("format", "json"); |
568 |
|
569 |
string permission = ""; |
570 |
if (write) |
571 |
permission = String.Format("write={0}", shareTo); |
572 |
else if (read) |
573 |
permission = String.Format("read={0}", shareTo); |
574 |
client.Headers.Add("X-Object-Sharing", permission); |
575 |
|
576 |
var content = client.DownloadStringWithRetryRelative(container, 3); |
577 |
|
578 |
client.AssertStatusOK("ShareObject failed"); |
579 |
|
580 |
//If the result is empty, return an empty list, |
581 |
var infos = String.IsNullOrWhiteSpace(content) |
582 |
? new List<ObjectInfo>() |
583 |
//Otherwise deserialize the object list into a list of ObjectInfos |
584 |
: JsonConvert.DeserializeObject<IList<ObjectInfo>>(content); |
585 |
|
586 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
587 |
} |
588 |
} |
589 |
|
590 |
|
591 |
} |
592 |
|
593 |
public async Task<AccountInfo> GetAccountPolicies(AccountInfo accountInfo) |
594 |
{ |
595 |
if (accountInfo==null) |
596 |
throw new ArgumentNullException("accountInfo"); |
597 |
Contract.EndContractBlock(); |
598 |
|
599 |
using (ThreadContext.Stacks["Account"].Push("GetPolicies")) |
600 |
{ |
601 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
602 |
|
603 |
/* |
604 |
if (_baseClient == null) |
605 |
{ |
606 |
_baseClient = new RestClient |
607 |
{ |
608 |
BaseAddress = accountInfo.StorageUri.ToString(), |
609 |
Timeout = 30000, |
610 |
Retries = 3, |
611 |
}; |
612 |
} |
613 |
|
614 |
*/ |
615 |
var containerUri = GetTargetUri(accountInfo.UserName); |
616 |
var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute); |
617 |
using(var response=await _baseHttpClient.HeadAsyncWithRetries(targetUri,3).ConfigureAwait(false)) |
618 |
{ |
619 |
|
620 |
var quotaValue=response.Headers.GetFirstValue("X-Account-Policy-Quota"); |
621 |
var bytesValue = response.Headers.GetFirstValue("X-Account-Bytes-Used"); |
622 |
long quota, bytes; |
623 |
if (long.TryParse(quotaValue, out quota)) |
624 |
accountInfo.Quota = quota; |
625 |
if (long.TryParse(bytesValue, out bytes)) |
626 |
accountInfo.BytesUsed = bytes; |
627 |
|
628 |
return accountInfo; |
629 |
} |
630 |
|
631 |
|
632 |
//using (var client = new RestClient(_baseClient)) |
633 |
//{ |
634 |
// if (!String.IsNullOrWhiteSpace(accountInfo.UserName)) |
635 |
// client.BaseAddress = GetAccountUrl(accountInfo.UserName); |
636 |
|
637 |
// client.Parameters.Clear(); |
638 |
// client.Parameters.Add("format", "json"); |
639 |
// client.Head(_emptyUri, 3); |
640 |
|
641 |
// var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"]; |
642 |
// var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"]; |
643 |
|
644 |
// long quota, bytes; |
645 |
// if (long.TryParse(quotaValue, out quota)) |
646 |
// accountInfo.Quota = quota; |
647 |
// if (long.TryParse(bytesValue, out bytes)) |
648 |
// accountInfo.BytesUsed = bytes; |
649 |
|
650 |
// return accountInfo; |
651 |
|
652 |
//} |
653 |
|
654 |
} |
655 |
} |
656 |
|
657 |
public void UpdateMetadata(ObjectInfo objectInfo) |
658 |
{ |
659 |
Contract.Requires<ArgumentNullException>(objectInfo != null,"objectInfo is null"); |
660 |
Contract.EndContractBlock(); |
661 |
|
662 |
using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata")) |
663 |
{ |
664 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
665 |
|
666 |
|
667 |
using(var client=new RestClient(_baseClient)) |
668 |
{ |
669 |
|
670 |
client.BaseAddress = GetAccountUrl(objectInfo.Account); |
671 |
|
672 |
client.Parameters.Clear(); |
673 |
|
674 |
|
675 |
//Set Tags |
676 |
foreach (var tag in objectInfo.Tags) |
677 |
{ |
678 |
var headerTag = String.Format("X-Object-Meta-{0}", tag.Key); |
679 |
client.Headers.Add(headerTag, tag.Value); |
680 |
} |
681 |
|
682 |
//Set Permissions |
683 |
|
684 |
var permissions=objectInfo.GetPermissionString(); |
685 |
client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions); |
686 |
|
687 |
client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition); |
688 |
client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding); |
689 |
client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest); |
690 |
var isPublic = objectInfo.IsPublic.ToString().ToLower(); |
691 |
client.Headers.Add("X-Object-Public", isPublic); |
692 |
|
693 |
|
694 |
var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name); |
695 |
client.PostWithRetry(new Uri(address,UriKind.Relative),"application/xml"); |
696 |
|
697 |
client.AssertStatusOK("UpdateMetadata failed"); |
698 |
//If the status is NOT ACCEPTED or OK we have a problem |
699 |
if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK)) |
700 |
{ |
701 |
Log.Error("Failed to update metadata"); |
702 |
throw new Exception("Failed to update metadata"); |
703 |
} |
704 |
|
705 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
706 |
} |
707 |
} |
708 |
|
709 |
} |
710 |
|
711 |
public void UpdateMetadata(ContainerInfo containerInfo) |
712 |
{ |
713 |
if (containerInfo == null) |
714 |
throw new ArgumentNullException("containerInfo"); |
715 |
Contract.EndContractBlock(); |
716 |
|
717 |
using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata")) |
718 |
{ |
719 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
720 |
|
721 |
|
722 |
using(var client=new RestClient(_baseClient)) |
723 |
{ |
724 |
|
725 |
client.BaseAddress = GetAccountUrl(containerInfo.Account); |
726 |
|
727 |
client.Parameters.Clear(); |
728 |
|
729 |
|
730 |
//Set Tags |
731 |
foreach (var tag in containerInfo.Tags) |
732 |
{ |
733 |
var headerTag = String.Format("X-Container-Meta-{0}", tag.Key); |
734 |
client.Headers.Add(headerTag, tag.Value); |
735 |
} |
736 |
|
737 |
|
738 |
//Set Policies |
739 |
foreach (var policy in containerInfo.Policies) |
740 |
{ |
741 |
var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key); |
742 |
client.Headers.Add(headerPolicy, policy.Value); |
743 |
} |
744 |
|
745 |
|
746 |
var uriBuilder = client.GetAddressBuilder(containerInfo.Name,_emptyUri); |
747 |
var uri = uriBuilder.Uri; |
748 |
|
749 |
client.UploadValues(uri,new NameValueCollection()); |
750 |
|
751 |
|
752 |
client.AssertStatusOK("UpdateMetadata failed"); |
753 |
//If the status is NOT ACCEPTED or OK we have a problem |
754 |
if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK)) |
755 |
{ |
756 |
Log.Error("Failed to update metadata"); |
757 |
throw new Exception("Failed to update metadata"); |
758 |
} |
759 |
|
760 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
761 |
} |
762 |
} |
763 |
|
764 |
} |
765 |
|
766 |
|
767 |
|
768 |
|
769 |
public IList<ObjectInfo> ListObjects(string account, Uri container, DateTimeOffset? since = null) |
770 |
{ |
771 |
/* |
772 |
if (container==null) |
773 |
throw new ArgumentNullException("container"); |
774 |
if (container.IsAbsoluteUri) |
775 |
throw new ArgumentException("container"); |
776 |
Contract.EndContractBlock(); |
777 |
*/ |
778 |
|
779 |
using (ThreadContext.Stacks["Objects"].Push("List")) |
780 |
{ |
781 |
|
782 |
var containerUri = GetTargetUri(account).Combine(container); |
783 |
var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute); |
784 |
|
785 |
var content =TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result; |
786 |
|
787 |
//304 will result in an empty string. Empty containers return an empty json array |
788 |
if (String.IsNullOrWhiteSpace(content)) |
789 |
return new[] {new NoModificationInfo(account, container)}; |
790 |
|
791 |
//If the result is empty, return an empty list, |
792 |
var infos = String.IsNullOrWhiteSpace(content) |
793 |
? new List<ObjectInfo>() |
794 |
//Otherwise deserialize the object list into a list of ObjectInfos |
795 |
: JsonConvert.DeserializeObject<IList<ObjectInfo>>(content); |
796 |
|
797 |
foreach (var info in infos) |
798 |
{ |
799 |
info.Container = container; |
800 |
info.Account = account; |
801 |
info.StorageUri = StorageUrl; |
802 |
} |
803 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
804 |
return infos; |
805 |
} |
806 |
} |
807 |
|
808 |
public IList<ObjectInfo> ListObjects(string account, Uri container, Uri folder, DateTimeOffset? since = null) |
809 |
{ |
810 |
/* if (container==null) |
811 |
throw new ArgumentNullException("container"); |
812 |
if (container.IsAbsoluteUri) |
813 |
throw new ArgumentException("container"); |
814 |
Contract.EndContractBlock();*/ |
815 |
|
816 |
using (ThreadContext.Stacks["Objects"].Push("List")) |
817 |
{ |
818 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
819 |
|
820 |
var containerUri = GetTargetUri(account).Combine(container); |
821 |
var targetUri = new Uri(String.Format("{0}?format=json&path={1}", containerUri,folder), UriKind.Absolute); |
822 |
var content = TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result; |
823 |
|
824 |
//304 will result in an empty string. Empty containers return an empty json array |
825 |
if (String.IsNullOrWhiteSpace(content)) |
826 |
return new[] { new NoModificationInfo(account, container) }; |
827 |
|
828 |
|
829 |
var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content); |
830 |
foreach (var info in infos) |
831 |
{ |
832 |
info.Account = account; |
833 |
if (info.Container == null) |
834 |
info.Container = container; |
835 |
info.StorageUri = StorageUrl; |
836 |
} |
837 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
838 |
return infos; |
839 |
/* |
840 |
using (var client = new RestClient(_baseClient)) |
841 |
{ |
842 |
if (!String.IsNullOrWhiteSpace(account)) |
843 |
client.BaseAddress = GetAccountUrl(account); |
844 |
|
845 |
client.Parameters.Clear(); |
846 |
client.Parameters.Add("format", "json"); |
847 |
client.Parameters.Add("path", folder.ToString()); |
848 |
client.IfModifiedSince = since; |
849 |
var content = client.DownloadStringWithRetryRelative(container, 3); |
850 |
client.AssertStatusOK("ListObjects failed"); |
851 |
|
852 |
if (client.StatusCode==HttpStatusCode.NotModified) |
853 |
return new[]{new NoModificationInfo(account,container,folder)}; |
854 |
|
855 |
var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content); |
856 |
foreach (var info in infos) |
857 |
{ |
858 |
info.Account = account; |
859 |
if (info.Container == null) |
860 |
info.Container = container; |
861 |
info.StorageUri = StorageUrl; |
862 |
} |
863 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
864 |
return infos; |
865 |
} |
866 |
*/ |
867 |
} |
868 |
} |
869 |
|
870 |
|
871 |
public async Task<bool> ContainerExists(string account, Uri container) |
872 |
{ |
873 |
if (container==null) |
874 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
875 |
if (container.IsAbsoluteUri) |
876 |
throw new ArgumentException( "The container must be relative","container"); |
877 |
Contract.EndContractBlock(); |
878 |
|
879 |
using (ThreadContext.Stacks["Containters"].Push("Exists")) |
880 |
{ |
881 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
882 |
|
883 |
var targetUri = GetTargetUri(account).Combine(container); |
884 |
|
885 |
using (var response =await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3)) |
886 |
{ |
887 |
|
888 |
bool result; |
889 |
switch (response.StatusCode) |
890 |
{ |
891 |
case HttpStatusCode.OK: |
892 |
case HttpStatusCode.NoContent: |
893 |
result = true; |
894 |
break; |
895 |
case HttpStatusCode.NotFound: |
896 |
result = false; |
897 |
break; |
898 |
default: |
899 |
throw CreateWebException("ContainerExists", response.StatusCode); |
900 |
} |
901 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
902 |
|
903 |
return result; |
904 |
} |
905 |
|
906 |
} |
907 |
} |
908 |
|
909 |
private Uri GetTargetUri(string account) |
910 |
{ |
911 |
return new Uri(GetTargetUrl(account),UriKind.Absolute); |
912 |
} |
913 |
|
914 |
private string GetTargetUrl(string account) |
915 |
{ |
916 |
return String.IsNullOrWhiteSpace(account) |
917 |
? _baseHttpClient.BaseAddress.ToString() |
918 |
: GetAccountUrl(account); |
919 |
} |
920 |
|
921 |
public async Task<bool> ObjectExists(string account, Uri container, Uri objectName) |
922 |
{ |
923 |
if (container == null) |
924 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
925 |
if (container.IsAbsoluteUri) |
926 |
throw new ArgumentException("The container must be relative","container"); |
927 |
if (objectName == null) |
928 |
throw new ArgumentNullException("objectName", "The objectName property can't be empty"); |
929 |
if (objectName.IsAbsoluteUri) |
930 |
throw new ArgumentException("The objectName must be relative","objectName"); |
931 |
Contract.EndContractBlock(); |
932 |
|
933 |
var targetUri=GetTargetUri(account).Combine(container).Combine(objectName); |
934 |
|
935 |
using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).ConfigureAwait(false)) |
936 |
{ |
937 |
switch (response.StatusCode) |
938 |
{ |
939 |
case HttpStatusCode.OK: |
940 |
case HttpStatusCode.NoContent: |
941 |
return true; |
942 |
case HttpStatusCode.NotFound: |
943 |
return false; |
944 |
default: |
945 |
throw CreateWebException("ObjectExists", response.StatusCode); |
946 |
} |
947 |
} |
948 |
} |
949 |
|
950 |
public async Task<ObjectInfo> GetObjectInfo(string account, Uri container, Uri objectName) |
951 |
{ |
952 |
if (container == null) |
953 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
954 |
if (container.IsAbsoluteUri) |
955 |
throw new ArgumentException("The container must be relative", "container"); |
956 |
if (objectName == null) |
957 |
throw new ArgumentNullException("objectName", "The objectName property can't be empty"); |
958 |
if (objectName.IsAbsoluteUri) |
959 |
throw new ArgumentException("The objectName must be relative", "objectName"); |
960 |
Contract.EndContractBlock(); |
961 |
|
962 |
using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo")) |
963 |
{ |
964 |
|
965 |
var targetUri = GetTargetUri(account).Combine(container).Combine(objectName); |
966 |
try |
967 |
{ |
968 |
using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true)) |
969 |
{ |
970 |
switch (response.StatusCode) |
971 |
{ |
972 |
case HttpStatusCode.OK: |
973 |
case HttpStatusCode.NoContent: |
974 |
var tags = response.Headers.GetMeta("X-Object-Meta-"); |
975 |
var extensions = (from header in response.Headers |
976 |
where |
977 |
header.Key.StartsWith("X-Object-") && |
978 |
!header.Key.StartsWith("X-Object-Meta-") |
979 |
select new {Name = header.Key, Value = header.Value.FirstOrDefault()}) |
980 |
.ToDictionary(t => t.Name, t => t.Value); |
981 |
|
982 |
var permissions = response.Headers.GetFirstValue("X-Object-Sharing"); |
983 |
|
984 |
|
985 |
var info = new ObjectInfo |
986 |
{ |
987 |
Account = account, |
988 |
Container = container, |
989 |
Name = objectName, |
990 |
ETag = response.Headers.ETag.NullSafe(e=>e.Tag), |
991 |
UUID = response.Headers.GetFirstValue("X-Object-UUID"), |
992 |
X_Object_Hash = response.Headers.GetFirstValue("X-Object-Hash"), |
993 |
Content_Type = response.Headers.GetFirstValue("Content-Type"), |
994 |
Bytes = Convert.ToInt64(response.Content.Headers.ContentLength), |
995 |
Tags = tags, |
996 |
Last_Modified = response.Content.Headers.LastModified, |
997 |
Extensions = extensions, |
998 |
ContentEncoding = |
999 |
response.Content.Headers.ContentEncoding.FirstOrDefault(), |
1000 |
ContendDisposition = |
1001 |
response.Content.Headers.ContentDisposition.NullSafe(c=>c.ToString()), |
1002 |
Manifest = response.Headers.GetFirstValue("X-Object-Manifest"), |
1003 |
PublicUrl = response.Headers.GetFirstValue("X-Object-Public"), |
1004 |
StorageUri = StorageUrl, |
1005 |
}; |
1006 |
info.SetPermissions(permissions); |
1007 |
return info; |
1008 |
case HttpStatusCode.NotFound: |
1009 |
return ObjectInfo.Empty; |
1010 |
default: |
1011 |
throw new WebException( |
1012 |
String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", |
1013 |
objectName, response.StatusCode)); |
1014 |
} |
1015 |
} |
1016 |
} |
1017 |
catch (RetryException) |
1018 |
{ |
1019 |
Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.", objectName); |
1020 |
return ObjectInfo.Empty; |
1021 |
} |
1022 |
catch (WebException e) |
1023 |
{ |
1024 |
Log.Error( |
1025 |
String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status {1}", |
1026 |
objectName, e.Status), e); |
1027 |
throw; |
1028 |
} |
1029 |
} |
1030 |
} |
1031 |
|
1032 |
|
1033 |
|
1034 |
public async Task CreateFolder(string account, Uri container, Uri folder) |
1035 |
{ |
1036 |
if (container == null) |
1037 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1038 |
if (container.IsAbsoluteUri) |
1039 |
throw new ArgumentException("The container must be relative","container"); |
1040 |
if (folder == null) |
1041 |
throw new ArgumentNullException("folder", "The objectName property can't be empty"); |
1042 |
if (folder.IsAbsoluteUri) |
1043 |
throw new ArgumentException("The objectName must be relative","folder"); |
1044 |
Contract.EndContractBlock(); |
1045 |
|
1046 |
var folderUri=container.Combine(folder); |
1047 |
var targetUri = GetTargetUri(account).Combine(folderUri); |
1048 |
var message = new HttpRequestMessage(HttpMethod.Put, targetUri); |
1049 |
message.Content=new StringContent(""); |
1050 |
message.Content.Headers.ContentType = new MediaTypeHeaderValue(ObjectInfo.CONTENT_TYPE_DIRECTORY); |
1051 |
//message.Headers.Add("Content-Length", "0"); |
1052 |
using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false)) |
1053 |
{ |
1054 |
if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted) |
1055 |
throw CreateWebException("CreateFolder", response.StatusCode); |
1056 |
} |
1057 |
} |
1058 |
|
1059 |
private Dictionary<string, string> GetMeta(HttpResponseMessage response,string metaPrefix) |
1060 |
{ |
1061 |
Contract.Requires<ArgumentNullException>(!String.IsNullOrWhiteSpace(metaPrefix), "metaPrefix is empty"); |
1062 |
Contract.EndContractBlock(); |
1063 |
|
1064 |
var dict = (from header in response.Headers |
1065 |
where header.Key.StartsWith(metaPrefix) |
1066 |
select new { Name = header.Key, Value = String.Join(",", header.Value) }) |
1067 |
.ToDictionary(t => t.Name, t => t.Value); |
1068 |
|
1069 |
|
1070 |
return dict; |
1071 |
} |
1072 |
|
1073 |
|
1074 |
public async Task<ContainerInfo> GetContainerInfo(string account, Uri container) |
1075 |
{ |
1076 |
if (container == null) |
1077 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1078 |
if (container.IsAbsoluteUri) |
1079 |
throw new ArgumentException("The container must be relative","container"); |
1080 |
Contract.EndContractBlock(); |
1081 |
|
1082 |
var targetUri = GetTargetUri(account).Combine(container); |
1083 |
using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true).ConfigureAwait(false)) |
1084 |
{ |
1085 |
if (Log.IsDebugEnabled) |
1086 |
Log.DebugFormat("ContainerInfo data: {0}\n{1}",response,await response.Content.ReadAsStringAsync().ConfigureAwait(false)); |
1087 |
switch (response.StatusCode) |
1088 |
{ |
1089 |
case HttpStatusCode.OK: |
1090 |
case HttpStatusCode.NoContent: |
1091 |
var tags = GetMeta(response,"X-Container-Meta-"); |
1092 |
var policies = GetMeta(response,"X-Container-Policy-"); |
1093 |
|
1094 |
var containerInfo = new ContainerInfo |
1095 |
{ |
1096 |
Account = account, |
1097 |
Name = container, |
1098 |
StorageUrl = StorageUrl.ToString(), |
1099 |
Count =long.Parse(response.Headers.GetFirstValue("X-Container-Object-Count")), |
1100 |
Bytes = long.Parse(response.Headers.GetFirstValue("X-Container-Bytes-Used")), |
1101 |
BlockHash = response.Headers.GetFirstValue("X-Container-Block-Hash"), |
1102 |
BlockSize = |
1103 |
int.Parse(response.Headers.GetFirstValue("X-Container-Block-Size")), |
1104 |
Last_Modified = response.Content.Headers.LastModified, |
1105 |
Tags = tags, |
1106 |
Policies = policies |
1107 |
}; |
1108 |
|
1109 |
|
1110 |
return containerInfo; |
1111 |
case HttpStatusCode.NotFound: |
1112 |
return ContainerInfo.Empty; |
1113 |
default: |
1114 |
throw CreateWebException("GetContainerInfo", response.StatusCode); |
1115 |
} |
1116 |
} |
1117 |
} |
1118 |
|
1119 |
public async Task CreateContainer(string account, Uri container) |
1120 |
{ |
1121 |
if (container == null) |
1122 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1123 |
if (container.IsAbsoluteUri) |
1124 |
throw new ArgumentException("The container must be relative","container"); |
1125 |
Contract.EndContractBlock(); |
1126 |
|
1127 |
var targetUri=GetTargetUri(account).Combine(container); |
1128 |
var message = new HttpRequestMessage(HttpMethod.Put, targetUri); |
1129 |
|
1130 |
//message.Content.Headers.ContentLength = 0; |
1131 |
using (var response =await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false)) |
1132 |
{ |
1133 |
var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK}; |
1134 |
if (!expectedCodes.Contains(response.StatusCode)) |
1135 |
throw CreateWebException("CreateContainer", response.StatusCode); |
1136 |
} |
1137 |
} |
1138 |
|
1139 |
public async Task WipeContainer(string account, Uri container) |
1140 |
{ |
1141 |
if (container == null) |
1142 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1143 |
if (container.IsAbsoluteUri) |
1144 |
throw new ArgumentException("The container must be relative", "container"); |
1145 |
Contract.EndContractBlock(); |
1146 |
|
1147 |
await DeleteContainer(account, new Uri(String.Format("{0}?delimiter=/", container), UriKind.Relative)).ConfigureAwait(false); |
1148 |
} |
1149 |
|
1150 |
|
1151 |
public async Task DeleteContainer(string account, Uri container) |
1152 |
{ |
1153 |
if (container == null) |
1154 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1155 |
if (container.IsAbsoluteUri) |
1156 |
throw new ArgumentException("The container must be relative","container"); |
1157 |
Contract.EndContractBlock(); |
1158 |
|
1159 |
var targetUri = GetTargetUri(account).Combine(container); |
1160 |
var message = new HttpRequestMessage(HttpMethod.Delete, targetUri); |
1161 |
using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false)) |
1162 |
{ |
1163 |
var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent }; |
1164 |
if (!expectedCodes.Contains(response.StatusCode)) |
1165 |
throw CreateWebException("DeleteContainer", response.StatusCode); |
1166 |
} |
1167 |
|
1168 |
} |
1169 |
|
1170 |
/// <summary> |
1171 |
/// |
1172 |
/// </summary> |
1173 |
/// <param name="account"></param> |
1174 |
/// <param name="container"></param> |
1175 |
/// <param name="objectName"></param> |
1176 |
/// <param name="fileName"></param> |
1177 |
/// <param name="cancellationToken"> </param> |
1178 |
/// <returns></returns> |
1179 |
/// <remarks>This method should have no timeout or a very long one</remarks> |
1180 |
//Asynchronously download the object specified by *objectName* in a specific *container* to |
1181 |
// a local file |
1182 |
public async Task GetObject(string account, Uri container, Uri objectName, string fileName,CancellationToken cancellationToken) |
1183 |
{ |
1184 |
if (container == null) |
1185 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1186 |
if (container.IsAbsoluteUri) |
1187 |
throw new ArgumentException("The container must be relative","container"); |
1188 |
if (objectName == null) |
1189 |
throw new ArgumentNullException("objectName", "The objectName property can't be empty"); |
1190 |
if (objectName.IsAbsoluteUri) |
1191 |
throw new ArgumentException("The objectName must be relative","objectName"); |
1192 |
Contract.EndContractBlock(); |
1193 |
|
1194 |
|
1195 |
try |
1196 |
{ |
1197 |
//WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient |
1198 |
//object to avoid concurrency errors. |
1199 |
// |
1200 |
//Download operations take a long time therefore they have no timeout. |
1201 |
using(var client = new RestClient(_baseClient) { Timeout = 0 }) |
1202 |
{ |
1203 |
if (!String.IsNullOrWhiteSpace(account)) |
1204 |
client.BaseAddress = GetAccountUrl(account); |
1205 |
|
1206 |
//The container and objectName are relative names. They are joined with the client's |
1207 |
//BaseAddress to create the object's absolute address |
1208 |
var builder = client.GetAddressBuilder(container, objectName); |
1209 |
var uri = builder.Uri; |
1210 |
|
1211 |
//Download progress is reported to the Trace log |
1212 |
Log.InfoFormat("[GET] START {0}", objectName); |
1213 |
/*client.DownloadProgressChanged += (sender, args) => |
1214 |
Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}", |
1215 |
fileName, args.ProgressPercentage, |
1216 |
args.BytesReceived, |
1217 |
args.TotalBytesToReceive);*/ |
1218 |
var progress = new Progress<DownloadProgressChangedEventArgs>(args => |
1219 |
{ |
1220 |
Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}", |
1221 |
fileName, args.ProgressPercentage, |
1222 |
args.BytesReceived, |
1223 |
args.TotalBytesToReceive); |
1224 |
if (DownloadProgressChanged!=null) |
1225 |
DownloadProgressChanged(this, new DownloadArgs(args)); |
1226 |
}); |
1227 |
|
1228 |
//Start downloading the object asynchronously |
1229 |
await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false); |
1230 |
|
1231 |
//Once the download completes |
1232 |
//Delete the local client object |
1233 |
} |
1234 |
//And report failure or completion |
1235 |
} |
1236 |
catch (Exception exc) |
1237 |
{ |
1238 |
Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc); |
1239 |
throw; |
1240 |
} |
1241 |
|
1242 |
Log.InfoFormat("[GET] END {0}", objectName); |
1243 |
|
1244 |
|
1245 |
} |
1246 |
|
1247 |
public async Task<IList<string>> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash) |
1248 |
{ |
1249 |
if (container == null) |
1250 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1251 |
if (container.IsAbsoluteUri) |
1252 |
throw new ArgumentException("The container must be relative","container"); |
1253 |
if (objectName == null) |
1254 |
throw new ArgumentNullException("objectName", "The objectName property can't be empty"); |
1255 |
if (objectName.IsAbsoluteUri) |
1256 |
throw new ArgumentException("The objectName must be relative","objectName"); |
1257 |
if (hash == null) |
1258 |
throw new ArgumentNullException("hash"); |
1259 |
if (String.IsNullOrWhiteSpace(Token)) |
1260 |
throw new InvalidOperationException("Invalid Token"); |
1261 |
if (StorageUrl == null) |
1262 |
throw new InvalidOperationException("Invalid Storage Url"); |
1263 |
Contract.EndContractBlock(); |
1264 |
|
1265 |
|
1266 |
|
1267 |
//The container and objectName are relative names. They are joined with the client's |
1268 |
//BaseAddress to create the object's absolute address |
1269 |
|
1270 |
var targetUri = GetTargetUri(account).Combine(container).Combine(objectName); |
1271 |
|
1272 |
|
1273 |
var uri = new Uri(String.Format("{0}?format=json&hashmap",targetUri),UriKind.Absolute); |
1274 |
|
1275 |
|
1276 |
//Send the tree hash as Json to the server |
1277 |
var jsonHash = hash.ToJson(); |
1278 |
if (Log.IsDebugEnabled) |
1279 |
Log.DebugFormat("Hashes:\r\n{0}", jsonHash); |
1280 |
|
1281 |
var mimeType = objectName.GetMimeType(); |
1282 |
|
1283 |
var message = new HttpRequestMessage(HttpMethod.Put, uri) |
1284 |
{ |
1285 |
Content = new StringContent(jsonHash) |
1286 |
}; |
1287 |
message.Content.Headers.ContentType = mimeType; |
1288 |
message.Headers.Add("ETag",hash.TopHash.ToHashString()); |
1289 |
|
1290 |
|
1291 |
//Don't use a timeout because putting the hashmap may be a long process |
1292 |
|
1293 |
using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false)) |
1294 |
{ |
1295 |
var empty = (IList<string>)new List<string>(); |
1296 |
|
1297 |
switch (response.StatusCode) |
1298 |
{ |
1299 |
case HttpStatusCode.Created: |
1300 |
//The server will respond either with 201-created if all blocks were already on the server |
1301 |
return empty; |
1302 |
case HttpStatusCode.Conflict: |
1303 |
//or with a 409-conflict and return the list of missing parts |
1304 |
using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) |
1305 |
using(var reader=stream.GetLoggedReader(Log)) |
1306 |
{ |
1307 |
var serializer = new JsonSerializer(); |
1308 |
serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member); |
1309 |
var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>)); |
1310 |
return hashes; |
1311 |
} |
1312 |
default: |
1313 |
//All other cases are unexpected |
1314 |
//Ensure that failure codes raise exceptions |
1315 |
response.EnsureSuccessStatusCode(); |
1316 |
//And log any other codes as warngings, but continute processing |
1317 |
Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",response.StatusCode,response.ReasonPhrase); |
1318 |
return empty; |
1319 |
} |
1320 |
} |
1321 |
|
1322 |
} |
1323 |
|
1324 |
|
1325 |
public async Task<byte[]> GetBlock(string account, Uri container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken) |
1326 |
{ |
1327 |
if (String.IsNullOrWhiteSpace(Token)) |
1328 |
throw new InvalidOperationException("Invalid Token"); |
1329 |
if (StorageUrl == null) |
1330 |
throw new InvalidOperationException("Invalid Storage Url"); |
1331 |
if (container == null) |
1332 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1333 |
if (container.IsAbsoluteUri) |
1334 |
throw new ArgumentException("The container must be relative","container"); |
1335 |
if (relativeUrl == null) |
1336 |
throw new ArgumentNullException("relativeUrl"); |
1337 |
if (end.HasValue && end < 0) |
1338 |
throw new ArgumentOutOfRangeException("end"); |
1339 |
if (start < 0) |
1340 |
throw new ArgumentOutOfRangeException("start"); |
1341 |
Contract.EndContractBlock(); |
1342 |
|
1343 |
|
1344 |
var targetUri = GetTargetUri(account).Combine(container).Combine(relativeUrl); |
1345 |
var message = new HttpRequestMessage(HttpMethod.Get, targetUri); |
1346 |
//Don't add a range if start=0, end=null (empty files) |
1347 |
if (start!=0 || end!=null) |
1348 |
message.Headers.Range=new RangeHeaderValue(start,end); |
1349 |
|
1350 |
//Don't use a timeout because putting the hashmap may be a long process |
1351 |
|
1352 |
IProgress<DownloadArgs> progress = new Progress<DownloadArgs>(args => |
1353 |
{ |
1354 |
Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}", |
1355 |
targetUri.Segments.Last(), args.ProgressPercentage, |
1356 |
args.BytesReceived, |
1357 |
args.TotalBytesToReceive); |
1358 |
|
1359 |
if (DownloadProgressChanged!=null) |
1360 |
DownloadProgressChanged(this, args); |
1361 |
}); |
1362 |
|
1363 |
|
1364 |
using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false,HttpCompletionOption.ResponseHeadersRead, |
1365 |
cancellationToken).ConfigureAwait(false)) |
1366 |
using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) |
1367 |
{ |
1368 |
long totalSize = response.Content.Headers.ContentLength ?? 0; |
1369 |
byte[] buffer,streambuf; |
1370 |
lock (_bufferManager) |
1371 |
{ |
1372 |
buffer = _bufferManager.TakeBuffer(65536); |
1373 |
streambuf = _bufferManager.TakeBuffer((int)totalSize); |
1374 |
} |
1375 |
|
1376 |
using (var targetStream = new MemoryStream(streambuf)) |
1377 |
{ |
1378 |
|
1379 |
long total = 0; |
1380 |
try |
1381 |
{ |
1382 |
|
1383 |
int read; |
1384 |
while ((read = await stream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) != 0) |
1385 |
{ |
1386 |
total += read; |
1387 |
progress.Report(new DownloadArgs(total, totalSize)); |
1388 |
await targetStream.WriteAsync(buffer, 0, read).ConfigureAwait(false); |
1389 |
} |
1390 |
} |
1391 |
finally |
1392 |
{ |
1393 |
lock (_bufferManager) |
1394 |
{ |
1395 |
_bufferManager.ReturnBuffer(buffer); |
1396 |
_bufferManager.ReturnBuffer(streambuf); |
1397 |
} |
1398 |
} |
1399 |
var result = targetStream.ToArray(); |
1400 |
return result; |
1401 |
} |
1402 |
} |
1403 |
|
1404 |
} |
1405 |
|
1406 |
public event EventHandler<UploadArgs> UploadProgressChanged; |
1407 |
public event EventHandler<DownloadArgs> DownloadProgressChanged; |
1408 |
|
1409 |
|
1410 |
public async Task PostBlock(string account, Uri container, byte[] block, int offset, int count,string blockHash,CancellationToken token) |
1411 |
{ |
1412 |
if (container == null) |
1413 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1414 |
if (container.IsAbsoluteUri) |
1415 |
throw new ArgumentException("The container must be relative","container"); |
1416 |
if (block == null) |
1417 |
throw new ArgumentNullException("block"); |
1418 |
if (offset < 0 || offset >= block.Length) |
1419 |
throw new ArgumentOutOfRangeException("offset"); |
1420 |
if (count < 0 || count > block.Length) |
1421 |
throw new ArgumentOutOfRangeException("count"); |
1422 |
if (String.IsNullOrWhiteSpace(Token)) |
1423 |
throw new InvalidOperationException("Invalid Token"); |
1424 |
if (StorageUrl == null) |
1425 |
throw new InvalidOperationException("Invalid Storage Url"); |
1426 |
Contract.EndContractBlock(); |
1427 |
|
1428 |
|
1429 |
try |
1430 |
{ |
1431 |
var containerUri = GetTargetUri(account).Combine(container); |
1432 |
var targetUri = new Uri(String.Format("{0}?update", containerUri)); |
1433 |
|
1434 |
|
1435 |
//Don't use a timeout because putting the hashmap may be a long process |
1436 |
|
1437 |
|
1438 |
Log.InfoFormat("[BLOCK POST] START"); |
1439 |
|
1440 |
|
1441 |
var progress = new Progress<UploadArgs>(args => |
1442 |
{ |
1443 |
Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}", |
1444 |
args.ProgressPercentage, |
1445 |
args.BytesSent, |
1446 |
args.TotalBytesToSend); |
1447 |
if (UploadProgressChanged != null) |
1448 |
UploadProgressChanged(this,args); |
1449 |
}); |
1450 |
|
1451 |
var message = new HttpRequestMessage(HttpMethod.Post, targetUri) |
1452 |
{ |
1453 |
Content = new ByteArrayContentWithProgress(block, offset, count,progress) |
1454 |
}; |
1455 |
message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream"); |
1456 |
|
1457 |
//Send the block |
1458 |
using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3,false,HttpCompletionOption.ResponseContentRead,token).ConfigureAwait(false)) |
1459 |
{ |
1460 |
Log.InfoFormat("[BLOCK POST PROGRESS] Completed "); |
1461 |
response.EnsureSuccessStatusCode(); |
1462 |
var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false); |
1463 |
var cleanHash = responseHash.TrimEnd(); |
1464 |
Debug.Assert(blockHash==cleanHash); |
1465 |
if (!blockHash.Equals(cleanHash, StringComparison.OrdinalIgnoreCase)) |
1466 |
Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]", account, container, blockHash, cleanHash); |
1467 |
} |
1468 |
Log.InfoFormat("[BLOCK POST] END"); |
1469 |
} |
1470 |
catch (TaskCanceledException ) |
1471 |
{ |
1472 |
Log.Info("Aborting block"); |
1473 |
throw; |
1474 |
} |
1475 |
catch (Exception exc) |
1476 |
{ |
1477 |
Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc); |
1478 |
throw; |
1479 |
} |
1480 |
} |
1481 |
|
1482 |
public async Task PostBlock(string account, Uri container, string filePath, long offset, int count, string blockHash, CancellationToken token) |
1483 |
{ |
1484 |
if (container == null) |
1485 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1486 |
if (container.IsAbsoluteUri) |
1487 |
throw new ArgumentException("The container must be relative", "container"); |
1488 |
if (String.IsNullOrWhiteSpace(filePath)) |
1489 |
throw new ArgumentNullException("filePath"); |
1490 |
if (!File.Exists(filePath)) |
1491 |
throw new FileNotFoundException("Missing file","filePath"); |
1492 |
if (String.IsNullOrWhiteSpace(Token)) |
1493 |
throw new InvalidOperationException("Invalid Token"); |
1494 |
if (StorageUrl == null) |
1495 |
throw new InvalidOperationException("Invalid Storage Url"); |
1496 |
Contract.EndContractBlock(); |
1497 |
|
1498 |
|
1499 |
try |
1500 |
{ |
1501 |
var containerUri = GetTargetUri(account).Combine(container); |
1502 |
var targetUri = new Uri(String.Format("{0}?update", containerUri)); |
1503 |
|
1504 |
|
1505 |
//Don't use a timeout because putting the hashmap may be a long process |
1506 |
|
1507 |
|
1508 |
Log.InfoFormat("[BLOCK POST] START"); |
1509 |
|
1510 |
|
1511 |
var progress = new Progress<UploadArgs>(args => |
1512 |
{ |
1513 |
Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2} at {3:###,} Kbps ", |
1514 |
args.ProgressPercentage, |
1515 |
args.BytesSent, |
1516 |
args.TotalBytesToSend,args.Speed); |
1517 |
if (UploadProgressChanged != null) |
1518 |
UploadProgressChanged(this, args); |
1519 |
}); |
1520 |
|
1521 |
var message = new HttpRequestMessage(HttpMethod.Post, targetUri) |
1522 |
{ |
1523 |
Content = new FileBlockContent(filePath, offset, count, progress) |
1524 |
}; |
1525 |
message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream"); |
1526 |
|
1527 |
//Send the block |
1528 |
using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false, HttpCompletionOption.ResponseContentRead, token).ConfigureAwait(false)) |
1529 |
{ |
1530 |
Log.InfoFormat("[BLOCK POST PROGRESS] Completed "); |
1531 |
response.EnsureSuccessStatusCode(); |
1532 |
var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false); |
1533 |
var cleanHash = responseHash.TrimEnd(); |
1534 |
Debug.Assert(blockHash == cleanHash); |
1535 |
if (!blockHash.Equals(cleanHash, StringComparison.OrdinalIgnoreCase)) |
1536 |
Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]", account, container, blockHash, cleanHash); |
1537 |
} |
1538 |
Log.InfoFormat("[BLOCK POST] END"); |
1539 |
} |
1540 |
catch (TaskCanceledException) |
1541 |
{ |
1542 |
Log.Info("Aborting block"); |
1543 |
throw; |
1544 |
} |
1545 |
catch (Exception exc) |
1546 |
{ |
1547 |
Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc); |
1548 |
throw; |
1549 |
} |
1550 |
} |
1551 |
|
1552 |
|
1553 |
public async Task<TreeHash> GetHashMap(string account, Uri container, Uri objectName) |
1554 |
{ |
1555 |
if (container == null) |
1556 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1557 |
if (container.IsAbsoluteUri) |
1558 |
throw new ArgumentException("The container must be relative","container"); |
1559 |
if (objectName == null) |
1560 |
throw new ArgumentNullException("objectName", "The objectName property can't be empty"); |
1561 |
if (objectName.IsAbsoluteUri) |
1562 |
throw new ArgumentException("The objectName must be relative","objectName"); |
1563 |
if (String.IsNullOrWhiteSpace(Token)) |
1564 |
throw new InvalidOperationException("Invalid Token"); |
1565 |
if (StorageUrl == null) |
1566 |
throw new InvalidOperationException("Invalid Storage Url"); |
1567 |
Contract.EndContractBlock(); |
1568 |
|
1569 |
try |
1570 |
{ |
1571 |
|
1572 |
var objectUri = GetTargetUri(account).Combine(container).Combine(objectName); |
1573 |
var targetUri = new Uri(String.Format("{0}?format=json&hashmap", objectUri)); |
1574 |
|
1575 |
//Start downloading the object asynchronously |
1576 |
var json = await GetStringAsync(targetUri, "").ConfigureAwait(false); |
1577 |
var treeHash = TreeHash.Parse(json); |
1578 |
Log.InfoFormat("[GET HASH] END {0}", objectName); |
1579 |
return treeHash; |
1580 |
|
1581 |
} |
1582 |
catch (Exception exc) |
1583 |
{ |
1584 |
Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc); |
1585 |
throw; |
1586 |
} |
1587 |
|
1588 |
} |
1589 |
|
1590 |
|
1591 |
/// <summary> |
1592 |
/// |
1593 |
/// </summary> |
1594 |
/// <param name="account"></param> |
1595 |
/// <param name="container"></param> |
1596 |
/// <param name="objectName"></param> |
1597 |
/// <param name="fileName"></param> |
1598 |
/// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param> |
1599 |
/// <param name="contentType"> </param> |
1600 |
/// <remarks>>This method should have no timeout or a very long one</remarks> |
1601 |
public async Task PutObject(string account, Uri container, Uri objectName, string fileName, string hash = Signature.MERKLE_EMPTY, string contentType = "application/octet-stream") |
1602 |
{ |
1603 |
if (container == null) |
1604 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1605 |
if (container.IsAbsoluteUri) |
1606 |
throw new ArgumentException("The container must be relative","container"); |
1607 |
if (objectName == null) |
1608 |
throw new ArgumentNullException("objectName", "The objectName property can't be empty"); |
1609 |
if (objectName.IsAbsoluteUri) |
1610 |
throw new ArgumentException("The objectName must be relative","objectName"); |
1611 |
if (String.IsNullOrWhiteSpace(fileName)) |
1612 |
throw new ArgumentNullException("fileName", "The fileName property can't be empty"); |
1613 |
try |
1614 |
{ |
1615 |
|
1616 |
using (var client = new RestClient(_baseClient) { Timeout = 0 }) |
1617 |
{ |
1618 |
if (!String.IsNullOrWhiteSpace(account)) |
1619 |
client.BaseAddress = GetAccountUrl(account); |
1620 |
|
1621 |
var builder = client.GetAddressBuilder(container, objectName); |
1622 |
var uri = builder.Uri; |
1623 |
|
1624 |
string etag = hash ; |
1625 |
|
1626 |
client.Headers.Add("Content-Type", contentType); |
1627 |
if (contentType!=ObjectInfo.CONTENT_TYPE_DIRECTORY) |
1628 |
client.Headers.Add("ETag", etag); |
1629 |
|
1630 |
|
1631 |
Log.InfoFormat("[PUT] START {0}", objectName); |
1632 |
client.UploadProgressChanged += (sender, args) => |
1633 |
{ |
1634 |
using (ThreadContext.Stacks["PUT"].Push("Progress")) |
1635 |
{ |
1636 |
Log.InfoFormat("{0} {1}% {2} of {3}", fileName, |
1637 |
args.ProgressPercentage, |
1638 |
args.BytesSent, args.TotalBytesToSend); |
1639 |
} |
1640 |
}; |
1641 |
|
1642 |
client.UploadFileCompleted += (sender, args) => |
1643 |
{ |
1644 |
using (ThreadContext.Stacks["PUT"].Push("Progress")) |
1645 |
{ |
1646 |
Log.InfoFormat("Completed {0}", fileName); |
1647 |
} |
1648 |
}; |
1649 |
|
1650 |
if (contentType==ObjectInfo.CONTENT_TYPE_DIRECTORY) |
1651 |
await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false); |
1652 |
else |
1653 |
await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false); |
1654 |
} |
1655 |
|
1656 |
Log.InfoFormat("[PUT] END {0}", objectName); |
1657 |
} |
1658 |
catch (Exception exc) |
1659 |
{ |
1660 |
Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc); |
1661 |
throw; |
1662 |
} |
1663 |
|
1664 |
} |
1665 |
|
1666 |
public async Task MoveObject(string account, Uri sourceContainer, Uri oldObjectName, Uri targetContainer, Uri newObjectName) |
1667 |
{ |
1668 |
if (sourceContainer == null) |
1669 |
throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty"); |
1670 |
if (sourceContainer.IsAbsoluteUri) |
1671 |
throw new ArgumentException("The sourceContainer must be relative","sourceContainer"); |
1672 |
if (oldObjectName == null) |
1673 |
throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty"); |
1674 |
if (oldObjectName.IsAbsoluteUri) |
1675 |
throw new ArgumentException("The oldObjectName must be relative","oldObjectName"); |
1676 |
if (targetContainer == null) |
1677 |
throw new ArgumentNullException("targetContainer", "The targetContainer property can't be empty"); |
1678 |
if (targetContainer.IsAbsoluteUri) |
1679 |
throw new ArgumentException("The targetContainer must be relative","targetContainer"); |
1680 |
if (newObjectName == null) |
1681 |
throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty"); |
1682 |
if (newObjectName.IsAbsoluteUri) |
1683 |
throw new ArgumentException("The newObjectName must be relative","newObjectName"); |
1684 |
Contract.EndContractBlock(); |
1685 |
|
1686 |
var baseUri = GetTargetUri(account); |
1687 |
var targetUri = baseUri.Combine(targetContainer).Combine(newObjectName); |
1688 |
var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, oldObjectName),UriKind.Relative); |
1689 |
|
1690 |
var message = new HttpRequestMessage(HttpMethod.Put, targetUri); |
1691 |
message.Headers.Add("X-Move-From", sourceUri.ToString()); |
1692 |
using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false)) |
1693 |
{ |
1694 |
var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; |
1695 |
if (!expectedCodes.Contains(response.StatusCode)) |
1696 |
throw CreateWebException("MoveObject", response.StatusCode); |
1697 |
} |
1698 |
} |
1699 |
|
1700 |
public async Task DeleteObject(string account, Uri sourceContainer, Uri objectName, bool isDirectory) |
1701 |
{ |
1702 |
if (sourceContainer == null) |
1703 |
throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty"); |
1704 |
if (sourceContainer.IsAbsoluteUri) |
1705 |
throw new ArgumentException("The sourceContainer must be relative","sourceContainer"); |
1706 |
if (objectName == null) |
1707 |
throw new ArgumentNullException("objectName", "The objectName property can't be empty"); |
1708 |
if (objectName.IsAbsoluteUri) |
1709 |
throw new ArgumentException("The objectName must be relative","objectName"); |
1710 |
Contract.EndContractBlock(); |
1711 |
|
1712 |
|
1713 |
|
1714 |
var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, objectName),UriKind.Relative); |
1715 |
|
1716 |
|
1717 |
if (objectName.OriginalString.EndsWith(".ignore")) |
1718 |
using(var response = await _baseHttpClient.DeleteAsync(sourceUri)){} |
1719 |
else |
1720 |
{ |
1721 |
var relativeUri = new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName), |
1722 |
UriKind.Relative); |
1723 |
|
1724 |
/* |
1725 |
var relativeUri = isDirectory |
1726 |
? new Uri( |
1727 |
String.Format("{0}/{1}?delimiter=/", FolderConstants.TrashContainer, |
1728 |
objectName), UriKind.Relative) |
1729 |
: new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName), |
1730 |
UriKind.Relative); |
1731 |
|
1732 |
*/ |
1733 |
var targetUri = GetTargetUri(account).Combine(relativeUri); |
1734 |
|
1735 |
|
1736 |
var message = new HttpRequestMessage(HttpMethod.Put, targetUri); |
1737 |
message.Headers.Add("X-Move-From", sourceUri.ToString()); |
1738 |
|
1739 |
Log.InfoFormat("[TRASH] [{0}] to [{1}]", sourceUri, targetUri); |
1740 |
using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3)) |
1741 |
{ |
1742 |
var expectedCodes = new[] |
1743 |
{ |
1744 |
HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created, |
1745 |
HttpStatusCode.NotFound |
1746 |
}; |
1747 |
if (!expectedCodes.Contains(response.StatusCode)) |
1748 |
throw CreateWebException("DeleteObject", response.StatusCode); |
1749 |
} |
1750 |
} |
1751 |
/* |
1752 |
|
1753 |
|
1754 |
var targetUrl = FolderConstants.TrashContainer + "/" + objectName; |
1755 |
/* |
1756 |
if (isDirectory) |
1757 |
targetUrl = targetUrl + "?delimiter=/"; |
1758 |
#1# |
1759 |
|
1760 |
var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName); |
1761 |
|
1762 |
using (var client = new RestClient(_baseClient)) |
1763 |
{ |
1764 |
if (!String.IsNullOrWhiteSpace(account)) |
1765 |
client.BaseAddress = GetAccountUrl(account); |
1766 |
|
1767 |
client.Headers.Add("X-Move-From", sourceUrl); |
1768 |
client.AllowedStatusCodes.Add(HttpStatusCode.NotFound); |
1769 |
Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl); |
1770 |
client.PutWithRetry(new Uri(targetUrl,UriKind.Relative), 3); |
1771 |
|
1772 |
var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound}; |
1773 |
if (!expectedCodes.Contains(client.StatusCode)) |
1774 |
throw CreateWebException("DeleteObject", client.StatusCode); |
1775 |
} |
1776 |
*/ |
1777 |
} |
1778 |
|
1779 |
|
1780 |
private static WebException CreateWebException(string operation, HttpStatusCode statusCode) |
1781 |
{ |
1782 |
return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode)); |
1783 |
} |
1784 |
|
1785 |
|
1786 |
public async Task<bool> CanUpload(string account, ObjectInfo cloudFile) |
1787 |
{ |
1788 |
Contract.Requires(!String.IsNullOrWhiteSpace(account)); |
1789 |
Contract.Requires(cloudFile!=null); |
1790 |
|
1791 |
var parts = cloudFile.Name.ToString().Split('/'); |
1792 |
var folder = String.Join("/", parts,0,parts.Length-1); |
1793 |
|
1794 |
var fileName = String.Format("{0}/{1}.pithos.ignore", folder, Guid.NewGuid()); |
1795 |
var fileUri=fileName.ToEscapedUri(); |
1796 |
|
1797 |
try |
1798 |
{ |
1799 |
var relativeUri = cloudFile.Container.Combine(fileUri); |
1800 |
var targetUri = GetTargetUri(account).Combine(relativeUri); |
1801 |
var message = new HttpRequestMessage(HttpMethod.Put, targetUri); |
1802 |
message.Content.Headers.ContentType =new MediaTypeHeaderValue("application/octet-stream"); |
1803 |
var response=await _baseHttpClient.SendAsyncWithRetries(message, 3); |
1804 |
var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; |
1805 |
var result=(expectedCodes.Contains(response.StatusCode)); |
1806 |
await DeleteObject(account, cloudFile.Container, fileUri, cloudFile.IsDirectory); |
1807 |
return result; |
1808 |
} |
1809 |
catch |
1810 |
{ |
1811 |
return false; |
1812 |
} |
1813 |
|
1814 |
} |
1815 |
|
1816 |
~CloudFilesClient() |
1817 |
{ |
1818 |
Dispose(false); |
1819 |
} |
1820 |
|
1821 |
public void Dispose() |
1822 |
{ |
1823 |
Dispose(true); |
1824 |
GC.SuppressFinalize(this); |
1825 |
} |
1826 |
|
1827 |
protected virtual void Dispose(bool disposing) |
1828 |
{ |
1829 |
if (disposing) |
1830 |
{ |
1831 |
if (_httpClientHandler!=null) |
1832 |
_httpClientHandler.Dispose(); |
1833 |
if (_baseClient!=null) |
1834 |
_baseClient.Dispose(); |
1835 |
if(_baseHttpClient!=null) |
1836 |
_baseHttpClient.Dispose(); |
1837 |
if (_baseHttpClientNoTimeout!=null) |
1838 |
_baseHttpClientNoTimeout.Dispose(); |
1839 |
} |
1840 |
_httpClientHandler = null; |
1841 |
_baseClient = null; |
1842 |
_baseHttpClient = null; |
1843 |
_baseHttpClientNoTimeout = null; |
1844 |
} |
1845 |
|
1846 |
public async Task<string> ResolveName(Guid accountToken) |
1847 |
{ |
1848 |
string format = string.Format("{{\"uuids\":[\"{0}\"]}}", accountToken); |
1849 |
var content = new StringContent(format,Encoding.UTF8); |
1850 |
//content.Headers.ContentType=new MediaTypeHeaderValue("text/html; charset=utf-8"); |
1851 |
string catalogEntry; |
1852 |
//var catalogUrl = new Uri(_baseHttpClient.BaseAddress.Scheme + "://" +_baseHttpClient.BaseAddress.Host,UriKind.Absolute).Combine("user_catalogs"); |
1853 |
var catalogUrl = new Uri("https://accounts.okeanos.grnet.gr/account/v1.0/user_catalogs"); |
1854 |
using (var response = await _baseHttpClient.PostAsync(catalogUrl, content).ConfigureAwait(false)) |
1855 |
{ |
1856 |
catalogEntry=await response.Content.ReadAsStringAsync().ConfigureAwait(false); |
1857 |
} |
1858 |
|
1859 |
var entry = (JContainer)JsonConvert.DeserializeObject(catalogEntry); |
1860 |
string key = accountToken.ToString(); |
1861 |
return (string)entry["uuid_catalog"][key]; |
1862 |
|
1863 |
} |
1864 |
|
1865 |
public async Task<Guid> ResolveToken(string displayName) |
1866 |
{ |
1867 |
string format = string.Format("{{\"displaynames\":[\"{0}\"]}}", displayName); |
1868 |
var content = new StringContent(format,Encoding.UTF8); |
1869 |
//content.Headers.ContentType=new MediaTypeHeaderValue("text/html; charset=utf-8"); |
1870 |
string catalogEntry; |
1871 |
//var catalogUrl = new Uri(_baseHttpClient.BaseAddress.Scheme + "://" +_baseHttpClient.BaseAddress.Host,UriKind.Absolute).Combine("user_catalogs"); |
1872 |
var catalogUrl = new Uri("https://accounts.okeanos.grnet.gr/account/v1.0/user_catalogs"); |
1873 |
using (var response = await _baseHttpClient.PostAsync(catalogUrl, content).ConfigureAwait(false)) |
1874 |
{ |
1875 |
catalogEntry=await response.Content.ReadAsStringAsync().ConfigureAwait(false); |
1876 |
} |
1877 |
|
1878 |
var entry = (JContainer)JsonConvert.DeserializeObject(catalogEntry); |
1879 |
return new Guid((string)entry["displayname_catalog"][displayName]); |
1880 |
|
1881 |
} |
1882 |
} |
1883 |
|
1884 |
public class ShareAccountInfo |
1885 |
{ |
1886 |
public DateTime? last_modified { get; set; } |
1887 |
public string name { get; set; } |
1888 |
} |
1889 |
} |