root / trunk / Pithos.Network / CloudFilesClient.cs @ 039ce31d
History | View | Annotate | Download (62.6 kB)
1 |
#region |
---|---|
2 |
/* ----------------------------------------------------------------------- |
3 |
* <copyright file="CloudFilesClient.cs" company="GRNet"> |
4 |
* |
5 |
* Copyright 2011-2012 GRNET S.A. All rights reserved. |
6 |
* |
7 |
* Redistribution and use in source and binary forms, with or |
8 |
* without modification, are permitted provided that the following |
9 |
* conditions are met: |
10 |
* |
11 |
* 1. Redistributions of source code must retain the above |
12 |
* copyright notice, this list of conditions and the following |
13 |
* disclaimer. |
14 |
* |
15 |
* 2. Redistributions in binary form must reproduce the above |
16 |
* copyright notice, this list of conditions and the following |
17 |
* disclaimer in the documentation and/or other materials |
18 |
* provided with the distribution. |
19 |
* |
20 |
* |
21 |
* THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
22 |
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
23 |
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
24 |
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
25 |
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
26 |
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
27 |
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
28 |
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
29 |
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
30 |
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
31 |
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
32 |
* POSSIBILITY OF SUCH DAMAGE. |
33 |
* |
34 |
* The views and conclusions contained in the software and |
35 |
* documentation are those of the authors and should not be |
36 |
* interpreted as representing official policies, either expressed |
37 |
* or implied, of GRNET S.A. |
38 |
* </copyright> |
39 |
* ----------------------------------------------------------------------- |
40 |
*/ |
41 |
#endregion |
42 |
|
43 |
// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos |
44 |
// |
45 |
// The class provides methods to upload/download files, delete files, manage containers |
46 |
|
47 |
|
48 |
using System; |
49 |
using System.Collections.Generic; |
50 |
using System.Collections.Specialized; |
51 |
using System.ComponentModel.Composition; |
52 |
using System.Diagnostics; |
53 |
using System.Diagnostics.Contracts; |
54 |
using System.IO; |
55 |
using System.Linq; |
56 |
using System.Net; |
57 |
using System.Reflection; |
58 |
using System.Security.Cryptography; |
59 |
using System.Text; |
60 |
using System.Threading; |
61 |
using System.Threading.Tasks; |
62 |
using Newtonsoft.Json; |
63 |
using Pithos.Interfaces; |
64 |
using log4net; |
65 |
|
66 |
namespace Pithos.Network |
67 |
{ |
68 |
[Export(typeof(ICloudClient))] |
69 |
public class CloudFilesClient:ICloudClient |
70 |
{ |
71 |
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
72 |
|
73 |
//CloudFilesClient uses *_baseClient* internally to communicate with the server |
74 |
//RestClient provides a REST-friendly interface over the standard WebClient. |
75 |
private RestClient _baseClient; |
76 |
|
77 |
|
78 |
//During authentication the client provides a UserName |
79 |
public string UserName { get; set; } |
80 |
|
81 |
//and and ApiKey to the server |
82 |
public string ApiKey { get; set; } |
83 |
|
84 |
//And receives an authentication Token. This token must be provided in ALL other operations, |
85 |
//in the X-Auth-Token header |
86 |
private string _token; |
87 |
public string Token |
88 |
{ |
89 |
get { return _token; } |
90 |
set |
91 |
{ |
92 |
_token = value; |
93 |
_baseClient.Headers["X-Auth-Token"] = value; |
94 |
} |
95 |
} |
96 |
|
97 |
//The client also receives a StorageUrl after authentication. All subsequent operations must |
98 |
//use this url |
99 |
public Uri StorageUrl { get; set; } |
100 |
|
101 |
|
102 |
public Uri RootAddressUri { get; set; } |
103 |
|
104 |
/* private WebProxy _proxy; |
105 |
public WebProxy Proxy |
106 |
{ |
107 |
get { return _proxy; } |
108 |
set |
109 |
{ |
110 |
_proxy = value; |
111 |
if (_baseClient != null) |
112 |
_baseClient.Proxy = value; |
113 |
} |
114 |
} |
115 |
*/ |
116 |
|
117 |
/* private Uri _proxy; |
118 |
public Uri Proxy |
119 |
{ |
120 |
get { return _proxy; } |
121 |
set |
122 |
{ |
123 |
_proxy = value; |
124 |
if (_baseClient != null) |
125 |
_baseClient.Proxy = new WebProxy(value); |
126 |
} |
127 |
}*/ |
128 |
|
129 |
public double DownloadPercentLimit { get; set; } |
130 |
public double UploadPercentLimit { get; set; } |
131 |
|
132 |
public string AuthenticationUrl { get; set; } |
133 |
|
134 |
|
135 |
public string VersionPath |
136 |
{ |
137 |
get { return UsePithos ? "v1" : "v1.0"; } |
138 |
} |
139 |
|
140 |
public bool UsePithos { get; set; } |
141 |
|
142 |
|
143 |
|
144 |
public CloudFilesClient(string userName, string apiKey) |
145 |
{ |
146 |
UserName = userName; |
147 |
ApiKey = apiKey; |
148 |
} |
149 |
|
150 |
public CloudFilesClient(AccountInfo accountInfo) |
151 |
{ |
152 |
if (accountInfo==null) |
153 |
throw new ArgumentNullException("accountInfo"); |
154 |
Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); |
155 |
Contract.Ensures(StorageUrl != null); |
156 |
Contract.Ensures(_baseClient != null); |
157 |
Contract.Ensures(RootAddressUri != null); |
158 |
Contract.EndContractBlock(); |
159 |
|
160 |
_baseClient = new RestClient |
161 |
{ |
162 |
BaseAddress = accountInfo.StorageUri.ToString(), |
163 |
Timeout = 30000, |
164 |
Retries = 3, |
165 |
}; |
166 |
StorageUrl = accountInfo.StorageUri; |
167 |
Token = accountInfo.Token; |
168 |
UserName = accountInfo.UserName; |
169 |
|
170 |
//Get the root address (StorageUrl without the account) |
171 |
var storageUrl = StorageUrl.AbsoluteUri; |
172 |
var usernameIndex = storageUrl.LastIndexOf(UserName); |
173 |
var rootUrl = storageUrl.Substring(0, usernameIndex); |
174 |
RootAddressUri = new Uri(rootUrl); |
175 |
} |
176 |
|
177 |
|
178 |
public AccountInfo Authenticate() |
179 |
{ |
180 |
if (String.IsNullOrWhiteSpace(UserName)) |
181 |
throw new InvalidOperationException("UserName is empty"); |
182 |
if (String.IsNullOrWhiteSpace(ApiKey)) |
183 |
throw new InvalidOperationException("ApiKey is empty"); |
184 |
if (String.IsNullOrWhiteSpace(AuthenticationUrl)) |
185 |
throw new InvalidOperationException("AuthenticationUrl is empty"); |
186 |
Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); |
187 |
Contract.Ensures(StorageUrl != null); |
188 |
Contract.Ensures(_baseClient != null); |
189 |
Contract.Ensures(RootAddressUri != null); |
190 |
Contract.EndContractBlock(); |
191 |
|
192 |
|
193 |
Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName); |
194 |
|
195 |
var groups = new List<Group>(); |
196 |
|
197 |
using (var authClient = new RestClient{BaseAddress=AuthenticationUrl}) |
198 |
{ |
199 |
/* if (Proxy != null) |
200 |
authClient.Proxy = Proxy;*/ |
201 |
|
202 |
Contract.Assume(authClient.Headers!=null); |
203 |
|
204 |
authClient.Headers.Add("X-Auth-User", UserName); |
205 |
authClient.Headers.Add("X-Auth-Key", ApiKey); |
206 |
|
207 |
authClient.DownloadStringWithRetry(VersionPath, 3); |
208 |
|
209 |
authClient.AssertStatusOK("Authentication failed"); |
210 |
|
211 |
var storageUrl = authClient.GetHeaderValue("X-Storage-Url"); |
212 |
if (String.IsNullOrWhiteSpace(storageUrl)) |
213 |
throw new InvalidOperationException("Failed to obtain storage url"); |
214 |
|
215 |
_baseClient = new RestClient |
216 |
{ |
217 |
BaseAddress = storageUrl, |
218 |
Timeout = 10000, |
219 |
Retries = 3, |
220 |
//Proxy=Proxy |
221 |
}; |
222 |
|
223 |
StorageUrl = new Uri(storageUrl); |
224 |
|
225 |
//Get the root address (StorageUrl without the account) |
226 |
var usernameIndex=storageUrl.LastIndexOf(UserName); |
227 |
var rootUrl = storageUrl.Substring(0, usernameIndex); |
228 |
RootAddressUri = new Uri(rootUrl); |
229 |
|
230 |
var token = authClient.GetHeaderValue("X-Auth-Token"); |
231 |
if (String.IsNullOrWhiteSpace(token)) |
232 |
throw new InvalidOperationException("Failed to obtain token url"); |
233 |
Token = token; |
234 |
|
235 |
/* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable(); |
236 |
groups = (from key in keys |
237 |
where key.StartsWith("X-Account-Group-") |
238 |
let name = key.Substring(16) |
239 |
select new Group(name, authClient.ResponseHeaders[key])) |
240 |
.ToList(); |
241 |
|
242 |
*/ |
243 |
} |
244 |
|
245 |
Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName); |
246 |
Debug.Assert(_baseClient!=null); |
247 |
|
248 |
return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups}; |
249 |
|
250 |
} |
251 |
|
252 |
|
253 |
|
254 |
public IList<ContainerInfo> ListContainers(string account) |
255 |
{ |
256 |
using (var client = new RestClient(_baseClient)) |
257 |
{ |
258 |
if (!String.IsNullOrWhiteSpace(account)) |
259 |
client.BaseAddress = GetAccountUrl(account); |
260 |
|
261 |
client.Parameters.Clear(); |
262 |
client.Parameters.Add("format", "json"); |
263 |
var content = client.DownloadStringWithRetry("", 3); |
264 |
client.AssertStatusOK("List Containers failed"); |
265 |
|
266 |
if (client.StatusCode == HttpStatusCode.NoContent) |
267 |
return new List<ContainerInfo>(); |
268 |
var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content); |
269 |
|
270 |
foreach (var info in infos) |
271 |
{ |
272 |
info.Account = account; |
273 |
} |
274 |
return infos; |
275 |
} |
276 |
|
277 |
} |
278 |
|
279 |
private string GetAccountUrl(string account) |
280 |
{ |
281 |
return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri; |
282 |
} |
283 |
|
284 |
public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null) |
285 |
{ |
286 |
using (ThreadContext.Stacks["Share"].Push("List Accounts")) |
287 |
{ |
288 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
289 |
|
290 |
using (var client = new RestClient(_baseClient)) |
291 |
{ |
292 |
client.Parameters.Clear(); |
293 |
client.Parameters.Add("format", "json"); |
294 |
client.IfModifiedSince = since; |
295 |
|
296 |
//Extract the username from the base address |
297 |
client.BaseAddress = RootAddressUri.AbsoluteUri; |
298 |
|
299 |
var content = client.DownloadStringWithRetry(@"", 3); |
300 |
|
301 |
client.AssertStatusOK("ListSharingAccounts failed"); |
302 |
|
303 |
//If the result is empty, return an empty list, |
304 |
var infos = String.IsNullOrWhiteSpace(content) |
305 |
? new List<ShareAccountInfo>() |
306 |
//Otherwise deserialize the account list into a list of ShareAccountInfos |
307 |
: JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content); |
308 |
|
309 |
Log.DebugFormat("END"); |
310 |
return infos; |
311 |
} |
312 |
} |
313 |
} |
314 |
|
315 |
|
316 |
/// <summary> |
317 |
/// Request listing of all objects in a container modified since a specific time. |
318 |
/// If the *since* value is missing, return all objects |
319 |
/// </summary> |
320 |
/// <param name="knownContainers">Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new |
321 |
/// and should be polled anyway |
322 |
/// </param> |
323 |
/// <param name="since"></param> |
324 |
/// <returns></returns> |
325 |
public IList<ObjectInfo> ListSharedObjects(HashSet<string> knownContainers,DateTime? since = null ) |
326 |
{ |
327 |
|
328 |
using (ThreadContext.Stacks["Share"].Push("List Objects")) |
329 |
{ |
330 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
331 |
//'since' is not used here because we need to have ListObjects return a NoChange result |
332 |
//for all shared accounts,containers |
333 |
|
334 |
Func<ContainerInfo, string> GetKey = c => String.Format("{0}\\{1}", c.Account, c.Name); |
335 |
|
336 |
var accounts = ListSharingAccounts(); |
337 |
var containers = (from account in accounts |
338 |
let conts = ListContainers(account.name) |
339 |
from container in conts |
340 |
select container).ToList(); |
341 |
var items = from container in containers |
342 |
let actualSince=knownContainers.Contains(GetKey(container))?since:null |
343 |
select ListObjects(container.Account , container.Name, actualSince); |
344 |
var objects=items.SelectMany(r=> r).ToList(); |
345 |
|
346 |
//Store any new containers |
347 |
foreach (var container in containers) |
348 |
{ |
349 |
knownContainers.Add(GetKey(container)); |
350 |
} |
351 |
|
352 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
353 |
return objects; |
354 |
} |
355 |
} |
356 |
|
357 |
public void SetTags(ObjectInfo target,IDictionary<string,string> tags) |
358 |
{ |
359 |
if (String.IsNullOrWhiteSpace(Token)) |
360 |
throw new InvalidOperationException("The Token is not set"); |
361 |
if (StorageUrl == null) |
362 |
throw new InvalidOperationException("The StorageUrl is not set"); |
363 |
if (target == null) |
364 |
throw new ArgumentNullException("target"); |
365 |
Contract.EndContractBlock(); |
366 |
|
367 |
using (ThreadContext.Stacks["Share"].Push("Share Object")) |
368 |
{ |
369 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
370 |
|
371 |
using (var client = new RestClient(_baseClient)) |
372 |
{ |
373 |
|
374 |
client.BaseAddress = GetAccountUrl(target.Account); |
375 |
|
376 |
client.Parameters.Clear(); |
377 |
client.Parameters.Add("update", ""); |
378 |
|
379 |
foreach (var tag in tags) |
380 |
{ |
381 |
var headerTag = String.Format("X-Object-Meta-{0}", tag.Key); |
382 |
client.Headers.Add(headerTag, tag.Value); |
383 |
} |
384 |
|
385 |
client.DownloadStringWithRetry(target.Container, 3); |
386 |
|
387 |
|
388 |
client.AssertStatusOK("SetTags failed"); |
389 |
//If the status is NOT ACCEPTED we have a problem |
390 |
if (client.StatusCode != HttpStatusCode.Accepted) |
391 |
{ |
392 |
Log.Error("Failed to set tags"); |
393 |
throw new Exception("Failed to set tags"); |
394 |
} |
395 |
|
396 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
397 |
} |
398 |
} |
399 |
|
400 |
|
401 |
} |
402 |
|
403 |
public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write) |
404 |
{ |
405 |
if (String.IsNullOrWhiteSpace(Token)) |
406 |
throw new InvalidOperationException("The Token is not set"); |
407 |
if (StorageUrl==null) |
408 |
throw new InvalidOperationException("The StorageUrl is not set"); |
409 |
if (String.IsNullOrWhiteSpace(container)) |
410 |
throw new ArgumentNullException("container"); |
411 |
if (String.IsNullOrWhiteSpace(objectName)) |
412 |
throw new ArgumentNullException("objectName"); |
413 |
if (String.IsNullOrWhiteSpace(account)) |
414 |
throw new ArgumentNullException("account"); |
415 |
if (String.IsNullOrWhiteSpace(shareTo)) |
416 |
throw new ArgumentNullException("shareTo"); |
417 |
Contract.EndContractBlock(); |
418 |
|
419 |
using (ThreadContext.Stacks["Share"].Push("Share Object")) |
420 |
{ |
421 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
422 |
|
423 |
using (var client = new RestClient(_baseClient)) |
424 |
{ |
425 |
|
426 |
client.BaseAddress = GetAccountUrl(account); |
427 |
|
428 |
client.Parameters.Clear(); |
429 |
client.Parameters.Add("format", "json"); |
430 |
|
431 |
string permission = ""; |
432 |
if (write) |
433 |
permission = String.Format("write={0}", shareTo); |
434 |
else if (read) |
435 |
permission = String.Format("read={0}", shareTo); |
436 |
client.Headers.Add("X-Object-Sharing", permission); |
437 |
|
438 |
var content = client.DownloadStringWithRetry(container, 3); |
439 |
|
440 |
client.AssertStatusOK("ShareObject failed"); |
441 |
|
442 |
//If the result is empty, return an empty list, |
443 |
var infos = String.IsNullOrWhiteSpace(content) |
444 |
? new List<ObjectInfo>() |
445 |
//Otherwise deserialize the object list into a list of ObjectInfos |
446 |
: JsonConvert.DeserializeObject<IList<ObjectInfo>>(content); |
447 |
|
448 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
449 |
} |
450 |
} |
451 |
|
452 |
|
453 |
} |
454 |
|
455 |
public AccountInfo GetAccountPolicies(AccountInfo accountInfo) |
456 |
{ |
457 |
if (accountInfo==null) |
458 |
throw new ArgumentNullException("accountInfo"); |
459 |
Contract.EndContractBlock(); |
460 |
|
461 |
using (ThreadContext.Stacks["Account"].Push("GetPolicies")) |
462 |
{ |
463 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
464 |
|
465 |
using (var client = new RestClient(_baseClient)) |
466 |
{ |
467 |
if (!String.IsNullOrWhiteSpace(accountInfo.UserName)) |
468 |
client.BaseAddress = GetAccountUrl(accountInfo.UserName); |
469 |
|
470 |
client.Parameters.Clear(); |
471 |
client.Parameters.Add("format", "json"); |
472 |
client.Head(String.Empty, 3); |
473 |
|
474 |
var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"]; |
475 |
var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"]; |
476 |
|
477 |
long quota, bytes; |
478 |
if (long.TryParse(quotaValue, out quota)) |
479 |
accountInfo.Quota = quota; |
480 |
if (long.TryParse(bytesValue, out bytes)) |
481 |
accountInfo.BytesUsed = bytes; |
482 |
|
483 |
return accountInfo; |
484 |
|
485 |
} |
486 |
|
487 |
} |
488 |
} |
489 |
|
490 |
public void UpdateMetadata(ObjectInfo objectInfo) |
491 |
{ |
492 |
if (objectInfo == null) |
493 |
throw new ArgumentNullException("objectInfo"); |
494 |
Contract.EndContractBlock(); |
495 |
|
496 |
using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata")) |
497 |
{ |
498 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
499 |
|
500 |
|
501 |
using(var client=new RestClient(_baseClient)) |
502 |
{ |
503 |
|
504 |
client.BaseAddress = GetAccountUrl(objectInfo.Account); |
505 |
|
506 |
client.Parameters.Clear(); |
507 |
|
508 |
|
509 |
//Set Tags |
510 |
foreach (var tag in objectInfo.Tags) |
511 |
{ |
512 |
var headerTag = String.Format("X-Object-Meta-{0}", tag.Key); |
513 |
client.Headers.Add(headerTag, tag.Value); |
514 |
} |
515 |
|
516 |
//Set Permissions |
517 |
|
518 |
var permissions=objectInfo.GetPermissionString(); |
519 |
client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions); |
520 |
|
521 |
client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition); |
522 |
client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding); |
523 |
client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest); |
524 |
var isPublic = objectInfo.IsPublic.ToString().ToLower(); |
525 |
client.Headers.Add("X-Object-Public", isPublic); |
526 |
|
527 |
|
528 |
/*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name); |
529 |
uriBuilder.Query = "update="; |
530 |
var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/ |
531 |
var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name); |
532 |
client.PostWithRetry(address,"application/xml"); |
533 |
|
534 |
//client.UploadValues(uri,new NameValueCollection()); |
535 |
|
536 |
|
537 |
client.AssertStatusOK("UpdateMetadata failed"); |
538 |
//If the status is NOT ACCEPTED or OK we have a problem |
539 |
if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK)) |
540 |
{ |
541 |
Log.Error("Failed to update metadata"); |
542 |
throw new Exception("Failed to update metadata"); |
543 |
} |
544 |
|
545 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
546 |
} |
547 |
} |
548 |
|
549 |
} |
550 |
|
551 |
public void UpdateMetadata(ContainerInfo containerInfo) |
552 |
{ |
553 |
if (containerInfo == null) |
554 |
throw new ArgumentNullException("containerInfo"); |
555 |
Contract.EndContractBlock(); |
556 |
|
557 |
using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata")) |
558 |
{ |
559 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
560 |
|
561 |
|
562 |
using(var client=new RestClient(_baseClient)) |
563 |
{ |
564 |
|
565 |
client.BaseAddress = GetAccountUrl(containerInfo.Account); |
566 |
|
567 |
client.Parameters.Clear(); |
568 |
|
569 |
|
570 |
//Set Tags |
571 |
foreach (var tag in containerInfo.Tags) |
572 |
{ |
573 |
var headerTag = String.Format("X-Container-Meta-{0}", tag.Key); |
574 |
client.Headers.Add(headerTag, tag.Value); |
575 |
} |
576 |
|
577 |
|
578 |
//Set Policies |
579 |
foreach (var policy in containerInfo.Policies) |
580 |
{ |
581 |
var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key); |
582 |
client.Headers.Add(headerPolicy, policy.Value); |
583 |
} |
584 |
|
585 |
|
586 |
var uriBuilder = client.GetAddressBuilder(containerInfo.Name,""); |
587 |
var uri = uriBuilder.Uri; |
588 |
|
589 |
client.UploadValues(uri,new NameValueCollection()); |
590 |
|
591 |
|
592 |
client.AssertStatusOK("UpdateMetadata failed"); |
593 |
//If the status is NOT ACCEPTED or OK we have a problem |
594 |
if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK)) |
595 |
{ |
596 |
Log.Error("Failed to update metadata"); |
597 |
throw new Exception("Failed to update metadata"); |
598 |
} |
599 |
|
600 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
601 |
} |
602 |
} |
603 |
|
604 |
} |
605 |
|
606 |
|
607 |
public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null) |
608 |
{ |
609 |
if (String.IsNullOrWhiteSpace(container)) |
610 |
throw new ArgumentNullException("container"); |
611 |
Contract.EndContractBlock(); |
612 |
|
613 |
using (ThreadContext.Stacks["Objects"].Push("List")) |
614 |
{ |
615 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
616 |
|
617 |
using (var client = new RestClient(_baseClient)) |
618 |
{ |
619 |
if (!String.IsNullOrWhiteSpace(account)) |
620 |
client.BaseAddress = GetAccountUrl(account); |
621 |
|
622 |
client.Parameters.Clear(); |
623 |
client.Parameters.Add("format", "json"); |
624 |
client.IfModifiedSince = since; |
625 |
var content = client.DownloadStringWithRetry(container, 3); |
626 |
|
627 |
client.AssertStatusOK("ListObjects failed"); |
628 |
|
629 |
if (client.StatusCode==HttpStatusCode.NotModified) |
630 |
return new[]{new NoModificationInfo(account,container)}; |
631 |
//If the result is empty, return an empty list, |
632 |
var infos = String.IsNullOrWhiteSpace(content) |
633 |
? new List<ObjectInfo>() |
634 |
//Otherwise deserialize the object list into a list of ObjectInfos |
635 |
: JsonConvert.DeserializeObject<IList<ObjectInfo>>(content); |
636 |
|
637 |
foreach (var info in infos) |
638 |
{ |
639 |
info.Container = container; |
640 |
info.Account = account; |
641 |
info.StorageUri = this.StorageUrl; |
642 |
} |
643 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
644 |
return infos; |
645 |
} |
646 |
} |
647 |
} |
648 |
|
649 |
public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null) |
650 |
{ |
651 |
if (String.IsNullOrWhiteSpace(container)) |
652 |
throw new ArgumentNullException("container"); |
653 |
/* |
654 |
if (String.IsNullOrWhiteSpace(folder)) |
655 |
throw new ArgumentNullException("folder"); |
656 |
*/ |
657 |
Contract.EndContractBlock(); |
658 |
|
659 |
using (ThreadContext.Stacks["Objects"].Push("List")) |
660 |
{ |
661 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
662 |
|
663 |
using (var client = new RestClient(_baseClient)) |
664 |
{ |
665 |
if (!String.IsNullOrWhiteSpace(account)) |
666 |
client.BaseAddress = GetAccountUrl(account); |
667 |
|
668 |
client.Parameters.Clear(); |
669 |
client.Parameters.Add("format", "json"); |
670 |
client.Parameters.Add("path", folder); |
671 |
client.IfModifiedSince = since; |
672 |
var content = client.DownloadStringWithRetry(container, 3); |
673 |
client.AssertStatusOK("ListObjects failed"); |
674 |
|
675 |
if (client.StatusCode==HttpStatusCode.NotModified) |
676 |
return new[]{new NoModificationInfo(account,container,folder)}; |
677 |
|
678 |
var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content); |
679 |
foreach (var info in infos) |
680 |
{ |
681 |
info.Account = account; |
682 |
if (info.Container == null) |
683 |
info.Container = container; |
684 |
info.StorageUri = this.StorageUrl; |
685 |
} |
686 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
687 |
return infos; |
688 |
} |
689 |
} |
690 |
} |
691 |
|
692 |
|
693 |
public bool ContainerExists(string account, string container) |
694 |
{ |
695 |
if (String.IsNullOrWhiteSpace(container)) |
696 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
697 |
Contract.EndContractBlock(); |
698 |
|
699 |
using (ThreadContext.Stacks["Containters"].Push("Exists")) |
700 |
{ |
701 |
if (Log.IsDebugEnabled) Log.DebugFormat("START"); |
702 |
|
703 |
using (var client = new RestClient(_baseClient)) |
704 |
{ |
705 |
if (!String.IsNullOrWhiteSpace(account)) |
706 |
client.BaseAddress = GetAccountUrl(account); |
707 |
|
708 |
client.Parameters.Clear(); |
709 |
client.Head(container, 3); |
710 |
|
711 |
bool result; |
712 |
switch (client.StatusCode) |
713 |
{ |
714 |
case HttpStatusCode.OK: |
715 |
case HttpStatusCode.NoContent: |
716 |
result=true; |
717 |
break; |
718 |
case HttpStatusCode.NotFound: |
719 |
result=false; |
720 |
break; |
721 |
default: |
722 |
throw CreateWebException("ContainerExists", client.StatusCode); |
723 |
} |
724 |
if (Log.IsDebugEnabled) Log.DebugFormat("END"); |
725 |
|
726 |
return result; |
727 |
} |
728 |
|
729 |
} |
730 |
} |
731 |
|
732 |
public bool ObjectExists(string account, string container, string objectName) |
733 |
{ |
734 |
if (String.IsNullOrWhiteSpace(container)) |
735 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
736 |
if (String.IsNullOrWhiteSpace(objectName)) |
737 |
throw new ArgumentNullException("objectName", "The objectName property can't be empty"); |
738 |
Contract.EndContractBlock(); |
739 |
|
740 |
using (var client = new RestClient(_baseClient)) |
741 |
{ |
742 |
if (!String.IsNullOrWhiteSpace(account)) |
743 |
client.BaseAddress = GetAccountUrl(account); |
744 |
|
745 |
client.Parameters.Clear(); |
746 |
client.Head(container + "/" + objectName, 3); |
747 |
|
748 |
switch (client.StatusCode) |
749 |
{ |
750 |
case HttpStatusCode.OK: |
751 |
case HttpStatusCode.NoContent: |
752 |
return true; |
753 |
case HttpStatusCode.NotFound: |
754 |
return false; |
755 |
default: |
756 |
throw CreateWebException("ObjectExists", client.StatusCode); |
757 |
} |
758 |
} |
759 |
|
760 |
} |
761 |
|
762 |
public ObjectInfo GetObjectInfo(string account, string container, string objectName) |
763 |
{ |
764 |
if (String.IsNullOrWhiteSpace(container)) |
765 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
766 |
if (String.IsNullOrWhiteSpace(objectName)) |
767 |
throw new ArgumentNullException("objectName", "The objectName property can't be empty"); |
768 |
Contract.EndContractBlock(); |
769 |
|
770 |
using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo")) |
771 |
{ |
772 |
|
773 |
using (var client = new RestClient(_baseClient)) |
774 |
{ |
775 |
if (!String.IsNullOrWhiteSpace(account)) |
776 |
client.BaseAddress = GetAccountUrl(account); |
777 |
try |
778 |
{ |
779 |
client.Parameters.Clear(); |
780 |
|
781 |
client.Head(container + "/" + objectName, 3); |
782 |
|
783 |
if (client.TimedOut) |
784 |
return ObjectInfo.Empty; |
785 |
|
786 |
switch (client.StatusCode) |
787 |
{ |
788 |
case HttpStatusCode.OK: |
789 |
case HttpStatusCode.NoContent: |
790 |
var keys = client.ResponseHeaders.AllKeys.AsQueryable(); |
791 |
var tags = client.GetMeta("X-Object-Meta-"); |
792 |
var extensions = (from key in keys |
793 |
where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-") |
794 |
select new {Name = key, Value = client.ResponseHeaders[key]}) |
795 |
.ToDictionary(t => t.Name, t => t.Value); |
796 |
|
797 |
var permissions=client.GetHeaderValue("X-Object-Sharing", true); |
798 |
|
799 |
|
800 |
var info = new ObjectInfo |
801 |
{ |
802 |
Account = account, |
803 |
Container = container, |
804 |
Name = objectName, |
805 |
ETag = client.GetHeaderValue("ETag"), |
806 |
UUID=client.GetHeaderValue("X-Object-UUID"), |
807 |
X_Object_Hash = client.GetHeaderValue("X-Object-Hash"), |
808 |
Content_Type = client.GetHeaderValue("Content-Type"), |
809 |
Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)), |
810 |
Tags = tags, |
811 |
Last_Modified = client.LastModified, |
812 |
Extensions = extensions, |
813 |
ContentEncoding=client.GetHeaderValue("Content-Encoding",true), |
814 |
ContendDisposition = client.GetHeaderValue("Content-Disposition",true), |
815 |
Manifest=client.GetHeaderValue("X-Object-Manifest",true), |
816 |
PublicUrl=client.GetHeaderValue("X-Object-Public",true), |
817 |
StorageUri=this.StorageUrl, |
818 |
}; |
819 |
info.SetPermissions(permissions); |
820 |
return info; |
821 |
case HttpStatusCode.NotFound: |
822 |
return ObjectInfo.Empty; |
823 |
default: |
824 |
throw new WebException( |
825 |
String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", |
826 |
objectName, client.StatusCode)); |
827 |
} |
828 |
|
829 |
} |
830 |
catch (RetryException) |
831 |
{ |
832 |
Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName); |
833 |
return ObjectInfo.Empty; |
834 |
} |
835 |
catch (WebException e) |
836 |
{ |
837 |
Log.Error( |
838 |
String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", |
839 |
objectName, client.StatusCode), e); |
840 |
throw; |
841 |
} |
842 |
} |
843 |
} |
844 |
|
845 |
} |
846 |
|
847 |
public void CreateFolder(string account, string container, string folder) |
848 |
{ |
849 |
if (String.IsNullOrWhiteSpace(container)) |
850 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
851 |
if (String.IsNullOrWhiteSpace(folder)) |
852 |
throw new ArgumentNullException("folder", "The folder property can't be empty"); |
853 |
Contract.EndContractBlock(); |
854 |
|
855 |
var folderUrl=String.Format("{0}/{1}",container,folder); |
856 |
using (var client = new RestClient(_baseClient)) |
857 |
{ |
858 |
if (!String.IsNullOrWhiteSpace(account)) |
859 |
client.BaseAddress = GetAccountUrl(account); |
860 |
|
861 |
client.Parameters.Clear(); |
862 |
client.Headers.Add("Content-Type", @"application/directory"); |
863 |
client.Headers.Add("Content-Length", "0"); |
864 |
client.PutWithRetry(folderUrl, 3); |
865 |
|
866 |
if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted) |
867 |
throw CreateWebException("CreateFolder", client.StatusCode); |
868 |
} |
869 |
} |
870 |
|
871 |
|
872 |
|
873 |
public ContainerInfo GetContainerInfo(string account, string container) |
874 |
{ |
875 |
if (String.IsNullOrWhiteSpace(container)) |
876 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
877 |
Contract.EndContractBlock(); |
878 |
|
879 |
using (var client = new RestClient(_baseClient)) |
880 |
{ |
881 |
if (!String.IsNullOrWhiteSpace(account)) |
882 |
client.BaseAddress = GetAccountUrl(account); |
883 |
|
884 |
client.Head(container); |
885 |
switch (client.StatusCode) |
886 |
{ |
887 |
case HttpStatusCode.OK: |
888 |
case HttpStatusCode.NoContent: |
889 |
var tags = client.GetMeta("X-Container-Meta-"); |
890 |
var policies = client.GetMeta("X-Container-Policy-"); |
891 |
|
892 |
var containerInfo = new ContainerInfo |
893 |
{ |
894 |
Account=account, |
895 |
Name = container, |
896 |
StorageUrl=this.StorageUrl.ToString(), |
897 |
Count = |
898 |
long.Parse(client.GetHeaderValue("X-Container-Object-Count")), |
899 |
Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")), |
900 |
BlockHash = client.GetHeaderValue("X-Container-Block-Hash"), |
901 |
BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")), |
902 |
Last_Modified=client.LastModified, |
903 |
Tags=tags, |
904 |
Policies=policies |
905 |
}; |
906 |
|
907 |
|
908 |
return containerInfo; |
909 |
case HttpStatusCode.NotFound: |
910 |
return ContainerInfo.Empty; |
911 |
default: |
912 |
throw CreateWebException("GetContainerInfo", client.StatusCode); |
913 |
} |
914 |
} |
915 |
} |
916 |
|
917 |
public void CreateContainer(string account, string container) |
918 |
{ |
919 |
if (String.IsNullOrWhiteSpace(account)) |
920 |
throw new ArgumentNullException("account"); |
921 |
if (String.IsNullOrWhiteSpace(container)) |
922 |
throw new ArgumentNullException("container"); |
923 |
Contract.EndContractBlock(); |
924 |
|
925 |
using (var client = new RestClient(_baseClient)) |
926 |
{ |
927 |
if (!String.IsNullOrWhiteSpace(account)) |
928 |
client.BaseAddress = GetAccountUrl(account); |
929 |
|
930 |
client.PutWithRetry(container, 3); |
931 |
var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK}; |
932 |
if (!expectedCodes.Contains(client.StatusCode)) |
933 |
throw CreateWebException("CreateContainer", client.StatusCode); |
934 |
} |
935 |
} |
936 |
|
937 |
public void DeleteContainer(string account, string container) |
938 |
{ |
939 |
if (String.IsNullOrWhiteSpace(container)) |
940 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
941 |
Contract.EndContractBlock(); |
942 |
|
943 |
using (var client = new RestClient(_baseClient)) |
944 |
{ |
945 |
if (!String.IsNullOrWhiteSpace(account)) |
946 |
client.BaseAddress = GetAccountUrl(account); |
947 |
|
948 |
client.DeleteWithRetry(container, 3); |
949 |
var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent}; |
950 |
if (!expectedCodes.Contains(client.StatusCode)) |
951 |
throw CreateWebException("DeleteContainer", client.StatusCode); |
952 |
} |
953 |
|
954 |
} |
955 |
|
956 |
/// <summary> |
957 |
/// |
958 |
/// </summary> |
959 |
/// <param name="account"></param> |
960 |
/// <param name="container"></param> |
961 |
/// <param name="objectName"></param> |
962 |
/// <param name="fileName"></param> |
963 |
/// <returns></returns> |
964 |
/// <remarks>This method should have no timeout or a very long one</remarks> |
965 |
//Asynchronously download the object specified by *objectName* in a specific *container* to |
966 |
// a local file |
967 |
public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken) |
968 |
{ |
969 |
if (String.IsNullOrWhiteSpace(container)) |
970 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
971 |
if (String.IsNullOrWhiteSpace(objectName)) |
972 |
throw new ArgumentNullException("objectName", "The objectName property can't be empty"); |
973 |
Contract.EndContractBlock(); |
974 |
|
975 |
try |
976 |
{ |
977 |
//WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient |
978 |
//object to avoid concurrency errors. |
979 |
// |
980 |
//Download operations take a long time therefore they have no timeout. |
981 |
using(var client = new RestClient(_baseClient) { Timeout = 0 }) |
982 |
{ |
983 |
if (!String.IsNullOrWhiteSpace(account)) |
984 |
client.BaseAddress = GetAccountUrl(account); |
985 |
|
986 |
//The container and objectName are relative names. They are joined with the client's |
987 |
//BaseAddress to create the object's absolute address |
988 |
var builder = client.GetAddressBuilder(container, objectName); |
989 |
var uri = builder.Uri; |
990 |
|
991 |
//Download progress is reported to the Trace log |
992 |
Log.InfoFormat("[GET] START {0}", objectName); |
993 |
client.DownloadProgressChanged += (sender, args) => |
994 |
Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}", |
995 |
fileName, args.ProgressPercentage, |
996 |
args.BytesReceived, |
997 |
args.TotalBytesToReceive); |
998 |
|
999 |
|
1000 |
//Start downloading the object asynchronously |
1001 |
await client.DownloadFileTaskAsync(uri, fileName,cancellationToken); |
1002 |
|
1003 |
//Once the download completes |
1004 |
//Delete the local client object |
1005 |
} |
1006 |
//And report failure or completion |
1007 |
} |
1008 |
catch (Exception exc) |
1009 |
{ |
1010 |
Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc); |
1011 |
throw; |
1012 |
} |
1013 |
|
1014 |
Log.InfoFormat("[GET] END {0}", objectName); |
1015 |
|
1016 |
|
1017 |
} |
1018 |
|
1019 |
public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash) |
1020 |
{ |
1021 |
if (String.IsNullOrWhiteSpace(container)) |
1022 |
throw new ArgumentNullException("container"); |
1023 |
if (String.IsNullOrWhiteSpace(objectName)) |
1024 |
throw new ArgumentNullException("objectName"); |
1025 |
if (hash==null) |
1026 |
throw new ArgumentNullException("hash"); |
1027 |
if (String.IsNullOrWhiteSpace(Token)) |
1028 |
throw new InvalidOperationException("Invalid Token"); |
1029 |
if (StorageUrl == null) |
1030 |
throw new InvalidOperationException("Invalid Storage Url"); |
1031 |
Contract.EndContractBlock(); |
1032 |
|
1033 |
|
1034 |
//Don't use a timeout because putting the hashmap may be a long process |
1035 |
var client = new RestClient(_baseClient) { Timeout = 0 }; |
1036 |
if (!String.IsNullOrWhiteSpace(account)) |
1037 |
client.BaseAddress = GetAccountUrl(account); |
1038 |
|
1039 |
//The container and objectName are relative names. They are joined with the client's |
1040 |
//BaseAddress to create the object's absolute address |
1041 |
var builder = client.GetAddressBuilder(container, objectName); |
1042 |
builder.Query = "format=json&hashmap"; |
1043 |
var uri = builder.Uri; |
1044 |
|
1045 |
|
1046 |
//Send the tree hash as Json to the server |
1047 |
client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream"; |
1048 |
var jsonHash = hash.ToJson(); |
1049 |
|
1050 |
client.Headers.Add("ETag",hash.MD5); |
1051 |
var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash); |
1052 |
if (Log.IsDebugEnabled) |
1053 |
Log.DebugFormat("Hashes:\r\n{0}", jsonHash); |
1054 |
return uploadTask.ContinueWith(t => |
1055 |
{ |
1056 |
|
1057 |
var empty = (IList<string>)new List<string>(); |
1058 |
|
1059 |
|
1060 |
//The server will respond either with 201-created if all blocks were already on the server |
1061 |
if (client.StatusCode == HttpStatusCode.Created) |
1062 |
{ |
1063 |
//in which case we return an empty hash list |
1064 |
return empty; |
1065 |
} |
1066 |
//or with a 409-conflict and return the list of missing parts |
1067 |
//A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception |
1068 |
if (t.IsFaulted) |
1069 |
{ |
1070 |
var ex = t.Exception.InnerException; |
1071 |
var we = ex as WebException; |
1072 |
var response = we.Response as HttpWebResponse; |
1073 |
if (response!=null && response.StatusCode==HttpStatusCode.Conflict) |
1074 |
{ |
1075 |
//In case of 409 the missing parts will be in the response content |
1076 |
using (var stream = response.GetResponseStream()) |
1077 |
using(var reader=stream.GetLoggedReader(Log)) |
1078 |
{ |
1079 |
//We used to have to cleanup the content before returning it because it contains |
1080 |
//error content after the list of hashes |
1081 |
// |
1082 |
//As of 30/1/2012, the result is a proper Json array so we don't need to read the content |
1083 |
//line by line |
1084 |
|
1085 |
var serializer = new JsonSerializer(); |
1086 |
serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member); |
1087 |
var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>)); |
1088 |
return hashes; |
1089 |
} |
1090 |
} |
1091 |
//Any other status code is unexpected and the exception should be rethrown |
1092 |
Log.LogError(response); |
1093 |
throw ex; |
1094 |
|
1095 |
} |
1096 |
|
1097 |
//Any other status code is unexpected but there was no exception. We can probably continue processing |
1098 |
Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription); |
1099 |
|
1100 |
return empty; |
1101 |
}); |
1102 |
|
1103 |
} |
1104 |
|
1105 |
|
1106 |
public async Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken) |
1107 |
{ |
1108 |
if (String.IsNullOrWhiteSpace(Token)) |
1109 |
throw new InvalidOperationException("Invalid Token"); |
1110 |
if (StorageUrl == null) |
1111 |
throw new InvalidOperationException("Invalid Storage Url"); |
1112 |
if (String.IsNullOrWhiteSpace(container)) |
1113 |
throw new ArgumentNullException("container"); |
1114 |
if (relativeUrl == null) |
1115 |
throw new ArgumentNullException("relativeUrl"); |
1116 |
if (end.HasValue && end < 0) |
1117 |
throw new ArgumentOutOfRangeException("end"); |
1118 |
if (start < 0) |
1119 |
throw new ArgumentOutOfRangeException("start"); |
1120 |
Contract.EndContractBlock(); |
1121 |
|
1122 |
//Don't use a timeout because putting the hashmap may be a long process |
1123 |
using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end}) |
1124 |
{ |
1125 |
if (!String.IsNullOrWhiteSpace(account)) |
1126 |
client.BaseAddress = GetAccountUrl(account); |
1127 |
|
1128 |
var builder = client.GetAddressBuilder(container, relativeUrl.ToString()); |
1129 |
var uri = builder.Uri; |
1130 |
|
1131 |
client.DownloadProgressChanged += (sender, args) => |
1132 |
{ |
1133 |
Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}", |
1134 |
uri.Segments.Last(), args.ProgressPercentage, |
1135 |
args.BytesReceived, |
1136 |
args.TotalBytesToReceive); |
1137 |
DownloadProgressChanged(sender, args); |
1138 |
}; |
1139 |
|
1140 |
|
1141 |
var result = await client.DownloadDataTaskAsync(uri, cancellationToken); |
1142 |
return result; |
1143 |
} |
1144 |
} |
1145 |
|
1146 |
public event UploadProgressChangedEventHandler UploadProgressChanged; |
1147 |
public event DownloadProgressChangedEventHandler DownloadProgressChanged; |
1148 |
|
1149 |
public async Task PostBlock(string account, string container, byte[] block, int offset, int count,CancellationToken token) |
1150 |
{ |
1151 |
if (String.IsNullOrWhiteSpace(container)) |
1152 |
throw new ArgumentNullException("container"); |
1153 |
if (block == null) |
1154 |
throw new ArgumentNullException("block"); |
1155 |
if (offset < 0 || offset >= block.Length) |
1156 |
throw new ArgumentOutOfRangeException("offset"); |
1157 |
if (count < 0 || count > block.Length) |
1158 |
throw new ArgumentOutOfRangeException("count"); |
1159 |
if (String.IsNullOrWhiteSpace(Token)) |
1160 |
throw new InvalidOperationException("Invalid Token"); |
1161 |
if (StorageUrl == null) |
1162 |
throw new InvalidOperationException("Invalid Storage Url"); |
1163 |
Contract.EndContractBlock(); |
1164 |
|
1165 |
|
1166 |
try |
1167 |
{ |
1168 |
|
1169 |
//Don't use a timeout because putting the hashmap may be a long process |
1170 |
using (var client = new RestClient(_baseClient) { Timeout = 0 }) |
1171 |
{ |
1172 |
if (!String.IsNullOrWhiteSpace(account)) |
1173 |
client.BaseAddress = GetAccountUrl(account); |
1174 |
|
1175 |
token.Register(client.CancelAsync); |
1176 |
|
1177 |
var builder = client.GetAddressBuilder(container, ""); |
1178 |
//We are doing an update |
1179 |
builder.Query = "update"; |
1180 |
var uri = builder.Uri; |
1181 |
|
1182 |
client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream"; |
1183 |
|
1184 |
Log.InfoFormat("[BLOCK POST] START"); |
1185 |
|
1186 |
client.UploadProgressChanged += (sender, args) => |
1187 |
{ |
1188 |
Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}", |
1189 |
args.ProgressPercentage, args.BytesSent, |
1190 |
args.TotalBytesToSend); |
1191 |
UploadProgressChanged(sender, args); |
1192 |
}; |
1193 |
client.UploadFileCompleted += (sender, args) => |
1194 |
Log.InfoFormat("[BLOCK POST PROGRESS] Completed "); |
1195 |
|
1196 |
var buffer = new byte[count]; |
1197 |
Buffer.BlockCopy(block, offset, buffer, 0, count); |
1198 |
//Send the block |
1199 |
await client.UploadDataTaskAsync(uri, "POST", buffer); |
1200 |
Log.InfoFormat("[BLOCK POST] END"); |
1201 |
} |
1202 |
} |
1203 |
catch (TaskCanceledException ) |
1204 |
{ |
1205 |
Log.Info("Aborting block"); |
1206 |
throw; |
1207 |
} |
1208 |
catch (Exception exc) |
1209 |
{ |
1210 |
Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc); |
1211 |
throw; |
1212 |
} |
1213 |
} |
1214 |
|
1215 |
|
1216 |
public async Task<TreeHash> GetHashMap(string account, string container, string objectName) |
1217 |
{ |
1218 |
if (String.IsNullOrWhiteSpace(container)) |
1219 |
throw new ArgumentNullException("container"); |
1220 |
if (String.IsNullOrWhiteSpace(objectName)) |
1221 |
throw new ArgumentNullException("objectName"); |
1222 |
if (String.IsNullOrWhiteSpace(Token)) |
1223 |
throw new InvalidOperationException("Invalid Token"); |
1224 |
if (StorageUrl == null) |
1225 |
throw new InvalidOperationException("Invalid Storage Url"); |
1226 |
Contract.EndContractBlock(); |
1227 |
|
1228 |
try |
1229 |
{ |
1230 |
//WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient |
1231 |
//object to avoid concurrency errors. |
1232 |
// |
1233 |
//Download operations take a long time therefore they have no timeout. |
1234 |
//TODO: Do they really? this is a hashmap operation, not a download |
1235 |
|
1236 |
//Start downloading the object asynchronously |
1237 |
using (var client = new RestClient(_baseClient) { Timeout = 0 }) |
1238 |
{ |
1239 |
if (!String.IsNullOrWhiteSpace(account)) |
1240 |
client.BaseAddress = GetAccountUrl(account); |
1241 |
|
1242 |
//The container and objectName are relative names. They are joined with the client's |
1243 |
//BaseAddress to create the object's absolute address |
1244 |
var builder = client.GetAddressBuilder(container, objectName); |
1245 |
builder.Query = "format=json&hashmap"; |
1246 |
var uri = builder.Uri; |
1247 |
|
1248 |
|
1249 |
var json = await client.DownloadStringTaskAsync(uri); |
1250 |
var treeHash = TreeHash.Parse(json); |
1251 |
Log.InfoFormat("[GET HASH] END {0}", objectName); |
1252 |
return treeHash; |
1253 |
} |
1254 |
} |
1255 |
catch (Exception exc) |
1256 |
{ |
1257 |
Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc); |
1258 |
throw; |
1259 |
} |
1260 |
|
1261 |
} |
1262 |
|
1263 |
|
1264 |
/// <summary> |
1265 |
/// |
1266 |
/// </summary> |
1267 |
/// <param name="account"></param> |
1268 |
/// <param name="container"></param> |
1269 |
/// <param name="objectName"></param> |
1270 |
/// <param name="fileName"></param> |
1271 |
/// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param> |
1272 |
/// <remarks>>This method should have no timeout or a very long one</remarks> |
1273 |
public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream") |
1274 |
{ |
1275 |
if (String.IsNullOrWhiteSpace(container)) |
1276 |
throw new ArgumentNullException("container", "The container property can't be empty"); |
1277 |
if (String.IsNullOrWhiteSpace(objectName)) |
1278 |
throw new ArgumentNullException("objectName", "The objectName property can't be empty"); |
1279 |
if (String.IsNullOrWhiteSpace(fileName)) |
1280 |
throw new ArgumentNullException("fileName", "The fileName property can't be empty"); |
1281 |
/* |
1282 |
if (!File.Exists(fileName) && !Directory.Exists(fileName)) |
1283 |
throw new FileNotFoundException("The file or directory does not exist",fileName); |
1284 |
*/ |
1285 |
|
1286 |
try |
1287 |
{ |
1288 |
|
1289 |
using (var client = new RestClient(_baseClient) { Timeout = 0 }) |
1290 |
{ |
1291 |
if (!String.IsNullOrWhiteSpace(account)) |
1292 |
client.BaseAddress = GetAccountUrl(account); |
1293 |
|
1294 |
var builder = client.GetAddressBuilder(container, objectName); |
1295 |
var uri = builder.Uri; |
1296 |
|
1297 |
string etag = hash ?? CalculateHash(fileName); |
1298 |
|
1299 |
client.Headers.Add("Content-Type", contentType); |
1300 |
client.Headers.Add("ETag", etag); |
1301 |
|
1302 |
|
1303 |
Log.InfoFormat("[PUT] START {0}", objectName); |
1304 |
client.UploadProgressChanged += (sender, args) => |
1305 |
{ |
1306 |
using (ThreadContext.Stacks["PUT"].Push("Progress")) |
1307 |
{ |
1308 |
Log.InfoFormat("{0} {1}% {2} of {3}", fileName, |
1309 |
args.ProgressPercentage, |
1310 |
args.BytesSent, args.TotalBytesToSend); |
1311 |
} |
1312 |
}; |
1313 |
|
1314 |
client.UploadFileCompleted += (sender, args) => |
1315 |
{ |
1316 |
using (ThreadContext.Stacks["PUT"].Push("Progress")) |
1317 |
{ |
1318 |
Log.InfoFormat("Completed {0}", fileName); |
1319 |
} |
1320 |
}; |
1321 |
if (contentType=="application/directory") |
1322 |
await client.UploadDataTaskAsync(uri, "PUT", new byte[0]); |
1323 |
else |
1324 |
await client.UploadFileTaskAsync(uri, "PUT", fileName); |
1325 |
} |
1326 |
|
1327 |
Log.InfoFormat("[PUT] END {0}", objectName); |
1328 |
} |
1329 |
catch (Exception exc) |
1330 |
{ |
1331 |
Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc); |
1332 |
throw; |
1333 |
} |
1334 |
|
1335 |
} |
1336 |
|
1337 |
|
1338 |
private static string CalculateHash(string fileName) |
1339 |
{ |
1340 |
Contract.Requires(!String.IsNullOrWhiteSpace(fileName)); |
1341 |
Contract.EndContractBlock(); |
1342 |
|
1343 |
string hash; |
1344 |
using (var hasher = MD5.Create()) |
1345 |
using(var stream=File.OpenRead(fileName)) |
1346 |
{ |
1347 |
var hashBuilder=new StringBuilder(); |
1348 |
foreach (byte b in hasher.ComputeHash(stream)) |
1349 |
hashBuilder.Append(b.ToString("x2").ToLower()); |
1350 |
hash = hashBuilder.ToString(); |
1351 |
} |
1352 |
return hash; |
1353 |
} |
1354 |
|
1355 |
public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName) |
1356 |
{ |
1357 |
if (String.IsNullOrWhiteSpace(sourceContainer)) |
1358 |
throw new ArgumentNullException("sourceContainer", "The container property can't be empty"); |
1359 |
if (String.IsNullOrWhiteSpace(oldObjectName)) |
1360 |
throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty"); |
1361 |
if (String.IsNullOrWhiteSpace(targetContainer)) |
1362 |
throw new ArgumentNullException("targetContainer", "The container property can't be empty"); |
1363 |
if (String.IsNullOrWhiteSpace(newObjectName)) |
1364 |
throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty"); |
1365 |
Contract.EndContractBlock(); |
1366 |
|
1367 |
var targetUrl = targetContainer + "/" + newObjectName; |
1368 |
var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName); |
1369 |
|
1370 |
using (var client = new RestClient(_baseClient)) |
1371 |
{ |
1372 |
if (!String.IsNullOrWhiteSpace(account)) |
1373 |
client.BaseAddress = GetAccountUrl(account); |
1374 |
|
1375 |
client.Headers.Add("X-Move-From", sourceUrl); |
1376 |
client.PutWithRetry(targetUrl, 3); |
1377 |
|
1378 |
var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; |
1379 |
if (!expectedCodes.Contains(client.StatusCode)) |
1380 |
throw CreateWebException("MoveObject", client.StatusCode); |
1381 |
} |
1382 |
} |
1383 |
|
1384 |
public void DeleteObject(string account, string sourceContainer, string objectName) |
1385 |
{ |
1386 |
if (String.IsNullOrWhiteSpace(sourceContainer)) |
1387 |
throw new ArgumentNullException("sourceContainer", "The container property can't be empty"); |
1388 |
if (String.IsNullOrWhiteSpace(objectName)) |
1389 |
throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty"); |
1390 |
Contract.EndContractBlock(); |
1391 |
|
1392 |
var targetUrl = FolderConstants.TrashContainer + "/" + objectName; |
1393 |
var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName); |
1394 |
|
1395 |
using (var client = new RestClient(_baseClient)) |
1396 |
{ |
1397 |
if (!String.IsNullOrWhiteSpace(account)) |
1398 |
client.BaseAddress = GetAccountUrl(account); |
1399 |
|
1400 |
client.Headers.Add("X-Move-From", sourceUrl); |
1401 |
client.AllowedStatusCodes.Add(HttpStatusCode.NotFound); |
1402 |
Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl); |
1403 |
client.PutWithRetry(targetUrl, 3); |
1404 |
|
1405 |
var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound}; |
1406 |
if (!expectedCodes.Contains(client.StatusCode)) |
1407 |
throw CreateWebException("DeleteObject", client.StatusCode); |
1408 |
} |
1409 |
} |
1410 |
|
1411 |
|
1412 |
private static WebException CreateWebException(string operation, HttpStatusCode statusCode) |
1413 |
{ |
1414 |
return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode)); |
1415 |
} |
1416 |
|
1417 |
|
1418 |
/* |
1419 |
public IEnumerable<ObjectInfo> ListDirectories(ContainerInfo container) |
1420 |
{ |
1421 |
var directories=this.ListObjects(container.Account, container.Name, "/"); |
1422 |
} |
1423 |
*/ |
1424 |
|
1425 |
public bool CanUpload(string account, ObjectInfo cloudFile) |
1426 |
{ |
1427 |
Contract.Requires(!String.IsNullOrWhiteSpace(account)); |
1428 |
Contract.Requires(cloudFile!=null); |
1429 |
|
1430 |
using (var client = new RestClient(_baseClient)) |
1431 |
{ |
1432 |
if (!String.IsNullOrWhiteSpace(account)) |
1433 |
client.BaseAddress = GetAccountUrl(account); |
1434 |
|
1435 |
|
1436 |
var parts = cloudFile.Name.Split('/'); |
1437 |
var folder = String.Join("/", parts,0,parts.Length-1); |
1438 |
|
1439 |
var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid()); |
1440 |
|
1441 |
client.Parameters.Clear(); |
1442 |
try |
1443 |
{ |
1444 |
client.PutWithRetry(fileUrl, 3, @"application/octet-stream"); |
1445 |
|
1446 |
var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; |
1447 |
var result=(expectedCodes.Contains(client.StatusCode)); |
1448 |
DeleteObject(account, cloudFile.Container, fileUrl); |
1449 |
return result; |
1450 |
} |
1451 |
catch |
1452 |
{ |
1453 |
return false; |
1454 |
} |
1455 |
} |
1456 |
} |
1457 |
} |
1458 |
|
1459 |
public class ShareAccountInfo |
1460 |
{ |
1461 |
public DateTime? last_modified { get; set; } |
1462 |
public string name { get; set; } |
1463 |
} |
1464 |
} |