1 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
3 // The class provides methods to upload/download files, delete files, manage containers
7 using System.Collections.Generic;
8 using System.Collections.Specialized;
9 using System.ComponentModel.Composition;
10 using System.Diagnostics;
11 using System.Diagnostics.Contracts;
15 using System.Security.Cryptography;
17 using System.Threading.Tasks;
18 using Newtonsoft.Json;
19 using Pithos.Interfaces;
22 namespace Pithos.Network
24 [Export(typeof(ICloudClient))]
25 public class CloudFilesClient:ICloudClient
27 //CloudFilesClient uses *_baseClient* internally to communicate with the server
28 //RestClient provides a REST-friendly interface over the standard WebClient.
29 private RestClient _baseClient;
32 //During authentication the client provides a UserName
33 public string UserName { get; set; }
35 //and and ApiKey to the server
36 public string ApiKey { get; set; }
38 //And receives an authentication Token. This token must be provided in ALL other operations,
39 //in the X-Auth-Token header
40 private string _token;
43 get { return _token; }
47 _baseClient.Headers["X-Auth-Token"] = value;
51 //The client also receives a StorageUrl after authentication. All subsequent operations must
53 public Uri StorageUrl { get; set; }
56 protected Uri RootAddressUri { get; set; }
61 get { return _proxy; }
65 if (_baseClient != null)
66 _baseClient.Proxy = new WebProxy(value);
70 public double DownloadPercentLimit { get; set; }
71 public double UploadPercentLimit { get; set; }
73 public string AuthenticationUrl { get; set; }
76 public string VersionPath
78 get { return UsePithos ? "v1" : "v1.0"; }
81 public bool UsePithos { get; set; }
84 private static readonly ILog Log = LogManager.GetLogger("CloudFilesClient");
86 public CloudFilesClient(string userName, string apiKey)
92 public CloudFilesClient(AccountInfo accountInfo)
94 if (accountInfo==null)
95 throw new ArgumentNullException("accountInfo");
96 Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
97 Contract.Ensures(StorageUrl != null);
98 Contract.Ensures(_baseClient != null);
99 Contract.Ensures(RootAddressUri != null);
100 Contract.EndContractBlock();
102 _baseClient = new RestClient
104 BaseAddress = accountInfo.StorageUri.ToString(),
108 StorageUrl = accountInfo.StorageUri;
109 Token = accountInfo.Token;
110 UserName = accountInfo.UserName;
112 //Get the root address (StorageUrl without the account)
113 var storageUrl = StorageUrl.AbsoluteUri;
114 var usernameIndex = storageUrl.LastIndexOf(UserName);
115 var rootUrl = storageUrl.Substring(0, usernameIndex);
116 RootAddressUri = new Uri(rootUrl);
120 public AccountInfo Authenticate()
122 if (String.IsNullOrWhiteSpace(UserName))
123 throw new InvalidOperationException("UserName is empty");
124 if (String.IsNullOrWhiteSpace(ApiKey))
125 throw new InvalidOperationException("ApiKey is empty");
126 if (String.IsNullOrWhiteSpace(AuthenticationUrl))
127 throw new InvalidOperationException("AuthenticationUrl is empty");
128 Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
129 Contract.Ensures(StorageUrl != null);
130 Contract.Ensures(_baseClient != null);
131 Contract.Ensures(RootAddressUri != null);
132 Contract.EndContractBlock();
135 Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
137 var groups = new List<Group>();
139 using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
142 authClient.Proxy = new WebProxy(Proxy);
144 Contract.Assume(authClient.Headers!=null);
146 authClient.Headers.Add("X-Auth-User", UserName);
147 authClient.Headers.Add("X-Auth-Key", ApiKey);
149 authClient.DownloadStringWithRetry(VersionPath, 3);
151 authClient.AssertStatusOK("Authentication failed");
153 var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
154 if (String.IsNullOrWhiteSpace(storageUrl))
155 throw new InvalidOperationException("Failed to obtain storage url");
157 _baseClient = new RestClient
159 BaseAddress = storageUrl,
164 StorageUrl = new Uri(storageUrl);
166 //Get the root address (StorageUrl without the account)
167 var usernameIndex=storageUrl.LastIndexOf(UserName);
168 var rootUrl = storageUrl.Substring(0, usernameIndex);
169 RootAddressUri = new Uri(rootUrl);
171 var token = authClient.GetHeaderValue("X-Auth-Token");
172 if (String.IsNullOrWhiteSpace(token))
173 throw new InvalidOperationException("Failed to obtain token url");
176 /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
177 groups = (from key in keys
178 where key.StartsWith("X-Account-Group-")
179 let name = key.Substring(16)
180 select new Group(name, authClient.ResponseHeaders[key]))
186 Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
189 return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};
195 public IList<ContainerInfo> ListContainers(string account)
197 using (var client = new RestClient(_baseClient))
199 if (!String.IsNullOrWhiteSpace(account))
200 client.BaseAddress = GetAccountUrl(account);
202 client.Parameters.Clear();
203 client.Parameters.Add("format", "json");
204 var content = client.DownloadStringWithRetry("", 3);
205 client.AssertStatusOK("List Containers failed");
207 if (client.StatusCode == HttpStatusCode.NoContent)
208 return new List<ContainerInfo>();
209 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
211 foreach (var info in infos)
213 info.Account = account;
220 private string GetAccountUrl(string account)
222 return new Uri(this.RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
225 public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
227 using (log4net.ThreadContext.Stacks["Share"].Push("List Accounts"))
229 if (Log.IsDebugEnabled) Log.DebugFormat("START");
231 using (var client = new RestClient(_baseClient))
233 client.Parameters.Clear();
234 client.Parameters.Add("format", "json");
235 client.IfModifiedSince = since;
237 //Extract the username from the base address
238 client.BaseAddress = RootAddressUri.AbsoluteUri;
240 var content = client.DownloadStringWithRetry(@"", 3);
242 client.AssertStatusOK("ListSharingAccounts failed");
244 //If the result is empty, return an empty list,
245 var infos = String.IsNullOrWhiteSpace(content)
246 ? new List<ShareAccountInfo>()
247 //Otherwise deserialize the account list into a list of ShareAccountInfos
248 : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
250 Log.DebugFormat("END");
256 //Request listing of all objects in a container modified since a specific time.
257 //If the *since* value is missing, return all objects
258 public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
261 using (log4net.ThreadContext.Stacks["Share"].Push("List Objects"))
263 if (Log.IsDebugEnabled) Log.DebugFormat("START");
265 var objects = new List<ObjectInfo>();
266 var accounts = ListSharingAccounts(since);
267 foreach (var account in accounts)
269 var containers = ListContainers(account.name);
270 foreach (var container in containers)
272 var containerObjects = ListObjects(account.name, container.Name, null);
273 objects.AddRange(containerObjects);
276 if (Log.IsDebugEnabled) Log.DebugFormat("END");
281 public void SetTags(ObjectInfo target,IDictionary<string,string> tags)
283 if (String.IsNullOrWhiteSpace(Token))
284 throw new InvalidOperationException("The Token is not set");
285 if (StorageUrl == null)
286 throw new InvalidOperationException("The StorageUrl is not set");
288 throw new ArgumentNullException("target");
289 Contract.EndContractBlock();
291 using (log4net.ThreadContext.Stacks["Share"].Push("Share Object"))
293 if (Log.IsDebugEnabled) Log.DebugFormat("START");
295 using (var client = new RestClient(_baseClient))
298 client.BaseAddress = GetAccountUrl(target.Account);
300 client.Parameters.Clear();
301 client.Parameters.Add("update", "");
303 foreach (var tag in tags)
305 var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
306 client.Headers.Add(headerTag, tag.Value);
309 var content = client.DownloadStringWithRetry(target.Container, 3);
312 client.AssertStatusOK("SetTags failed");
313 //If the status is NOT ACCEPTED we have a problem
314 if (client.StatusCode != HttpStatusCode.Accepted)
316 Log.Error("Failed to set tags");
317 throw new Exception("Failed to set tags");
320 if (Log.IsDebugEnabled) Log.DebugFormat("END");
327 public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
329 if (String.IsNullOrWhiteSpace(Token))
330 throw new InvalidOperationException("The Token is not set");
331 if (StorageUrl==null)
332 throw new InvalidOperationException("The StorageUrl is not set");
333 if (String.IsNullOrWhiteSpace(container))
334 throw new ArgumentNullException("container");
335 if (String.IsNullOrWhiteSpace(objectName))
336 throw new ArgumentNullException("objectName");
337 if (String.IsNullOrWhiteSpace(account))
338 throw new ArgumentNullException("account");
339 if (String.IsNullOrWhiteSpace(shareTo))
340 throw new ArgumentNullException("shareTo");
341 Contract.EndContractBlock();
343 using (log4net.ThreadContext.Stacks["Share"].Push("Share Object"))
345 if (Log.IsDebugEnabled) Log.DebugFormat("START");
347 using (var client = new RestClient(_baseClient))
350 client.BaseAddress = GetAccountUrl(account);
352 client.Parameters.Clear();
353 client.Parameters.Add("format", "json");
355 string permission = "";
357 permission = String.Format("write={0}", shareTo);
359 permission = String.Format("read={0}", shareTo);
360 client.Headers.Add("X-Object-Sharing", permission);
362 var content = client.DownloadStringWithRetry(container, 3);
364 client.AssertStatusOK("ShareObject failed");
366 //If the result is empty, return an empty list,
367 var infos = String.IsNullOrWhiteSpace(content)
368 ? new List<ObjectInfo>()
369 //Otherwise deserialize the object list into a list of ObjectInfos
370 : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
372 if (Log.IsDebugEnabled) Log.DebugFormat("END");
379 public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
381 if (accountInfo==null)
382 throw new ArgumentNullException("accountInfo");
383 Contract.EndContractBlock();
385 using (log4net.ThreadContext.Stacks["Account"].Push("GetPolicies"))
387 if (Log.IsDebugEnabled) Log.DebugFormat("START");
389 using (var client = new RestClient(_baseClient))
391 if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
392 client.BaseAddress = GetAccountUrl(accountInfo.UserName);
394 client.Parameters.Clear();
395 client.Parameters.Add("format", "json");
396 client.Head(String.Empty, 3);
398 var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
399 var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
402 if (long.TryParse(quotaValue, out quota))
403 accountInfo.Quota = quota;
404 if (long.TryParse(bytesValue, out bytes))
405 accountInfo.BytesUsed = bytes;
414 public void UpdateMetadata(ObjectInfo objectInfo)
416 if (objectInfo == null)
417 throw new ArgumentNullException("objectInfo");
418 Contract.EndContractBlock();
420 using (log4net.ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
422 if (Log.IsDebugEnabled) Log.DebugFormat("START");
425 using(var client=new RestClient(_baseClient))
428 client.BaseAddress = GetAccountUrl(objectInfo.Account);
430 client.Parameters.Clear();
434 foreach (var tag in objectInfo.Tags)
436 var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
437 client.Headers.Add(headerTag, tag.Value);
442 var permissions=objectInfo.GetPermissionString();
443 client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);
445 client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
446 client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
447 client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
448 var isPublic = objectInfo.IsPublic.ToString().ToLower();
449 client.Headers.Add("X-Object-Public", isPublic);
452 var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);
453 var uri = uriBuilder.Uri;
455 var content = client.UploadValues(uri,new NameValueCollection());
458 client.AssertStatusOK("UpdateMetadata failed");
459 //If the status is NOT ACCEPTED or OK we have a problem
460 if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
462 Log.Error("Failed to update metadata");
463 throw new Exception("Failed to update metadata");
466 if (Log.IsDebugEnabled) Log.DebugFormat("END");
472 public void UpdateMetadata(ContainerInfo containerInfo)
474 if (containerInfo == null)
475 throw new ArgumentNullException("containerInfo");
476 Contract.EndContractBlock();
478 using (log4net.ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
480 if (Log.IsDebugEnabled) Log.DebugFormat("START");
483 using(var client=new RestClient(_baseClient))
486 client.BaseAddress = GetAccountUrl(containerInfo.Account);
488 client.Parameters.Clear();
492 foreach (var tag in containerInfo.Tags)
494 var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
495 client.Headers.Add(headerTag, tag.Value);
500 foreach (var policy in containerInfo.Policies)
502 var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
503 client.Headers.Add(headerPolicy, policy.Value);
507 var uriBuilder = client.GetAddressBuilder(containerInfo.Name,"");
508 var uri = uriBuilder.Uri;
510 var content = client.UploadValues(uri,new NameValueCollection());
513 client.AssertStatusOK("UpdateMetadata failed");
514 //If the status is NOT ACCEPTED or OK we have a problem
515 if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
517 Log.Error("Failed to update metadata");
518 throw new Exception("Failed to update metadata");
521 if (Log.IsDebugEnabled) Log.DebugFormat("END");
528 public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
530 if (String.IsNullOrWhiteSpace(container))
531 throw new ArgumentNullException("container");
532 Contract.EndContractBlock();
534 using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
536 if (Log.IsDebugEnabled) Log.DebugFormat("START");
538 using (var client = new RestClient(_baseClient))
540 if (!String.IsNullOrWhiteSpace(account))
541 client.BaseAddress = GetAccountUrl(account);
543 client.Parameters.Clear();
544 client.Parameters.Add("format", "json");
545 client.IfModifiedSince = since;
546 var content = client.DownloadStringWithRetry(container, 3);
548 client.AssertStatusOK("ListObjects failed");
550 //If the result is empty, return an empty list,
551 var infos = String.IsNullOrWhiteSpace(content)
552 ? new List<ObjectInfo>()
553 //Otherwise deserialize the object list into a list of ObjectInfos
554 : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
556 foreach (var info in infos)
558 info.Container = container;
559 info.Account = account;
561 if (Log.IsDebugEnabled) Log.DebugFormat("START");
569 public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
571 if (String.IsNullOrWhiteSpace(container))
572 throw new ArgumentNullException("container");
573 if (String.IsNullOrWhiteSpace(folder))
574 throw new ArgumentNullException("folder");
575 Contract.EndContractBlock();
577 using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
579 if (Log.IsDebugEnabled) Log.DebugFormat("START");
581 using (var client = new RestClient(_baseClient))
583 if (!String.IsNullOrWhiteSpace(account))
584 client.BaseAddress = GetAccountUrl(account);
586 client.Parameters.Clear();
587 client.Parameters.Add("format", "json");
588 client.Parameters.Add("path", folder);
589 client.IfModifiedSince = since;
590 var content = client.DownloadStringWithRetry(container, 3);
591 client.AssertStatusOK("ListObjects failed");
593 var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
595 if (Log.IsDebugEnabled) Log.DebugFormat("END");
602 public bool ContainerExists(string account, string container)
604 if (String.IsNullOrWhiteSpace(container))
605 throw new ArgumentNullException("container", "The container property can't be empty");
606 Contract.EndContractBlock();
608 using (log4net.ThreadContext.Stacks["Containters"].Push("Exists"))
610 if (Log.IsDebugEnabled) Log.DebugFormat("START");
612 using (var client = new RestClient(_baseClient))
614 if (!String.IsNullOrWhiteSpace(account))
615 client.BaseAddress = GetAccountUrl(account);
617 client.Parameters.Clear();
618 client.Head(container, 3);
621 switch (client.StatusCode)
623 case HttpStatusCode.OK:
624 case HttpStatusCode.NoContent:
627 case HttpStatusCode.NotFound:
631 throw CreateWebException("ContainerExists", client.StatusCode);
633 if (Log.IsDebugEnabled) Log.DebugFormat("END");
641 public bool ObjectExists(string account, string container, string objectName)
643 if (String.IsNullOrWhiteSpace(container))
644 throw new ArgumentNullException("container", "The container property can't be empty");
645 if (String.IsNullOrWhiteSpace(objectName))
646 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
647 Contract.EndContractBlock();
649 using (var client = new RestClient(_baseClient))
651 if (!String.IsNullOrWhiteSpace(account))
652 client.BaseAddress = GetAccountUrl(account);
654 client.Parameters.Clear();
655 client.Head(container + "/" + objectName, 3);
657 switch (client.StatusCode)
659 case HttpStatusCode.OK:
660 case HttpStatusCode.NoContent:
662 case HttpStatusCode.NotFound:
665 throw CreateWebException("ObjectExists", client.StatusCode);
671 public ObjectInfo GetObjectInfo(string account, string container, string objectName)
673 if (String.IsNullOrWhiteSpace(container))
674 throw new ArgumentNullException("container", "The container property can't be empty");
675 if (String.IsNullOrWhiteSpace(objectName))
676 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
677 Contract.EndContractBlock();
679 using (log4net.ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
682 using (var client = new RestClient(_baseClient))
684 if (!String.IsNullOrWhiteSpace(account))
685 client.BaseAddress = GetAccountUrl(account);
688 client.Parameters.Clear();
690 client.Head(container + "/" + objectName, 3);
693 return ObjectInfo.Empty;
695 switch (client.StatusCode)
697 case HttpStatusCode.OK:
698 case HttpStatusCode.NoContent:
699 var keys = client.ResponseHeaders.AllKeys.AsQueryable();
700 var tags = client.GetMeta("X-Object-Meta-");
701 var extensions = (from key in keys
702 where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
703 select new {Name = key, Value = client.ResponseHeaders[key]})
704 .ToDictionary(t => t.Name, t => t.Value);
706 var permissions=client.GetHeaderValue("X-Object-Sharing", true);
709 var info = new ObjectInfo
712 Container = container,
714 Hash = client.GetHeaderValue("ETag"),
715 Content_Type = client.GetHeaderValue("Content-Type"),
716 Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),
718 Last_Modified = client.LastModified,
719 Extensions = extensions,
720 ContentEncoding=client.GetHeaderValue("Content-Encoding",true),
721 ContendDisposition = client.GetHeaderValue("Content-Disposition",true),
722 Manifest=client.GetHeaderValue("X-Object-Manifest",true),
723 PublicUrl=client.GetHeaderValue("X-Object-Public",true),
725 info.SetPermissions(permissions);
727 case HttpStatusCode.NotFound:
728 return ObjectInfo.Empty;
730 throw new WebException(
731 String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
732 objectName, client.StatusCode));
736 catch (RetryException)
738 Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.");
739 return ObjectInfo.Empty;
741 catch (WebException e)
744 String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
745 objectName, client.StatusCode), e);
753 public void CreateFolder(string account, string container, string folder)
755 if (String.IsNullOrWhiteSpace(container))
756 throw new ArgumentNullException("container", "The container property can't be empty");
757 if (String.IsNullOrWhiteSpace(folder))
758 throw new ArgumentNullException("folder", "The folder property can't be empty");
759 Contract.EndContractBlock();
761 var folderUrl=String.Format("{0}/{1}",container,folder);
762 using (var client = new RestClient(_baseClient))
764 if (!String.IsNullOrWhiteSpace(account))
765 client.BaseAddress = GetAccountUrl(account);
767 client.Parameters.Clear();
768 client.Headers.Add("Content-Type", @"application/directory");
769 client.Headers.Add("Content-Length", "0");
770 client.PutWithRetry(folderUrl, 3);
772 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
773 throw CreateWebException("CreateFolder", client.StatusCode);
779 public ContainerInfo GetContainerInfo(string account, string container)
781 if (String.IsNullOrWhiteSpace(container))
782 throw new ArgumentNullException("container", "The container property can't be empty");
783 Contract.EndContractBlock();
785 using (var client = new RestClient(_baseClient))
787 if (!String.IsNullOrWhiteSpace(account))
788 client.BaseAddress = GetAccountUrl(account);
790 client.Head(container);
791 switch (client.StatusCode)
793 case HttpStatusCode.OK:
794 case HttpStatusCode.NoContent:
795 var tags = client.GetMeta("X-Container-Meta-");
796 var policies = client.GetMeta("X-Container-Policy-");
798 var containerInfo = new ContainerInfo
803 long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
804 Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
805 BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
806 BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),
807 Last_Modified=client.LastModified,
813 return containerInfo;
814 case HttpStatusCode.NotFound:
815 return ContainerInfo.Empty;
817 throw CreateWebException("GetContainerInfo", client.StatusCode);
822 public void CreateContainer(string account, string container)
824 if (String.IsNullOrWhiteSpace(account))
825 throw new ArgumentNullException("account");
826 if (String.IsNullOrWhiteSpace(container))
827 throw new ArgumentNullException("container");
828 Contract.EndContractBlock();
830 using (var client = new RestClient(_baseClient))
832 if (!String.IsNullOrWhiteSpace(account))
833 client.BaseAddress = GetAccountUrl(account);
835 client.PutWithRetry(container, 3);
836 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
837 if (!expectedCodes.Contains(client.StatusCode))
838 throw CreateWebException("CreateContainer", client.StatusCode);
842 public void DeleteContainer(string account, string container)
844 if (String.IsNullOrWhiteSpace(container))
845 throw new ArgumentNullException("container", "The container property can't be empty");
846 Contract.EndContractBlock();
848 using (var client = new RestClient(_baseClient))
850 if (!String.IsNullOrWhiteSpace(account))
851 client.BaseAddress = GetAccountUrl(account);
853 client.DeleteWithRetry(container, 3);
854 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
855 if (!expectedCodes.Contains(client.StatusCode))
856 throw CreateWebException("DeleteContainer", client.StatusCode);
864 /// <param name="account"></param>
865 /// <param name="container"></param>
866 /// <param name="objectName"></param>
867 /// <param name="fileName"></param>
868 /// <returns></returns>
869 /// <remarks>This method should have no timeout or a very long one</remarks>
870 //Asynchronously download the object specified by *objectName* in a specific *container* to
872 public Task GetObject(string account, string container, string objectName, string fileName)
874 if (String.IsNullOrWhiteSpace(container))
875 throw new ArgumentNullException("container", "The container property can't be empty");
876 if (String.IsNullOrWhiteSpace(objectName))
877 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
878 Contract.EndContractBlock();
882 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
883 //object to avoid concurrency errors.
885 //Download operations take a long time therefore they have no timeout.
886 var client = new RestClient(_baseClient) { Timeout = 0 };
887 if (!String.IsNullOrWhiteSpace(account))
888 client.BaseAddress = GetAccountUrl(account);
890 //The container and objectName are relative names. They are joined with the client's
891 //BaseAddress to create the object's absolute address
892 var builder = client.GetAddressBuilder(container, objectName);
893 var uri = builder.Uri;
895 //Download progress is reported to the Trace log
896 Log.InfoFormat("[GET] START {0}", objectName);
897 client.DownloadProgressChanged += (sender, args) =>
898 Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
899 fileName, args.ProgressPercentage,
901 args.TotalBytesToReceive);
904 //Start downloading the object asynchronously
905 var downloadTask = client.DownloadFileTask(uri, fileName);
907 //Once the download completes
908 return downloadTask.ContinueWith(download =>
910 //Delete the local client object
912 //And report failure or completion
913 if (download.IsFaulted)
915 Log.ErrorFormat("[GET] FAIL for {0} with \r{1}", objectName,
920 Log.InfoFormat("[GET] END {0}", objectName);
924 catch (Exception exc)
926 Log.ErrorFormat("[GET] END {0} with {1}", objectName, exc);
934 public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
936 if (String.IsNullOrWhiteSpace(container))
937 throw new ArgumentNullException("container");
938 if (String.IsNullOrWhiteSpace(objectName))
939 throw new ArgumentNullException("objectName");
941 throw new ArgumentNullException("hash");
942 if (String.IsNullOrWhiteSpace(Token))
943 throw new InvalidOperationException("Invalid Token");
944 if (StorageUrl == null)
945 throw new InvalidOperationException("Invalid Storage Url");
946 Contract.EndContractBlock();
949 //Don't use a timeout because putting the hashmap may be a long process
950 var client = new RestClient(_baseClient) { Timeout = 0 };
951 if (!String.IsNullOrWhiteSpace(account))
952 client.BaseAddress = GetAccountUrl(account);
954 //The container and objectName are relative names. They are joined with the client's
955 //BaseAddress to create the object's absolute address
956 var builder = client.GetAddressBuilder(container, objectName);
957 builder.Query = "format=json&hashmap";
958 var uri = builder.Uri;
961 //Send the tree hash as Json to the server
962 client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
963 var jsonHash = hash.ToJson();
964 var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);
966 return uploadTask.ContinueWith(t =>
969 var empty = (IList<string>)new List<string>();
972 //The server will respond either with 201-created if all blocks were already on the server
973 if (client.StatusCode == HttpStatusCode.Created)
975 //in which case we return an empty hash list
978 //or with a 409-conflict and return the list of missing parts
979 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
982 var ex = t.Exception.InnerException;
983 var we = ex as WebException;
984 var response = we.Response as HttpWebResponse;
985 if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
987 //In case of 409 the missing parts will be in the response content
988 using (var stream = response.GetResponseStream())
989 using(var reader=new StreamReader(stream))
991 Debug.Assert(stream.Position == 0);
992 //We need to cleanup the content before returning it because it contains
993 //error content after the list of hashes
994 var hashes = new List<string>();
996 //All lines up to the first empty line are hashes
997 while(!String.IsNullOrWhiteSpace(line=reader.ReadLine()))
1006 //Any other status code is unexpected and the exception should be rethrown
1010 //Any other status code is unexpected but there was no exception. We can probably continue processing
1013 Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
1020 public Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end)
1022 if (String.IsNullOrWhiteSpace(Token))
1023 throw new InvalidOperationException("Invalid Token");
1024 if (StorageUrl == null)
1025 throw new InvalidOperationException("Invalid Storage Url");
1026 if (String.IsNullOrWhiteSpace(container))
1027 throw new ArgumentNullException("container");
1028 if (relativeUrl== null)
1029 throw new ArgumentNullException("relativeUrl");
1030 if (end.HasValue && end<0)
1031 throw new ArgumentOutOfRangeException("end");
1033 throw new ArgumentOutOfRangeException("start");
1034 Contract.EndContractBlock();
1037 //Don't use a timeout because putting the hashmap may be a long process
1038 var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
1039 if (!String.IsNullOrWhiteSpace(account))
1040 client.BaseAddress = GetAccountUrl(account);
1042 var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
1043 var uri = builder.Uri;
1045 return client.DownloadDataTask(uri)
1054 public Task PostBlock(string account, string container, byte[] block, int offset, int count)
1056 if (String.IsNullOrWhiteSpace(container))
1057 throw new ArgumentNullException("container");
1059 throw new ArgumentNullException("block");
1060 if (offset < 0 || offset >= block.Length)
1061 throw new ArgumentOutOfRangeException("offset");
1062 if (count < 0 || count > block.Length)
1063 throw new ArgumentOutOfRangeException("count");
1064 if (String.IsNullOrWhiteSpace(Token))
1065 throw new InvalidOperationException("Invalid Token");
1066 if (StorageUrl == null)
1067 throw new InvalidOperationException("Invalid Storage Url");
1068 Contract.EndContractBlock();
1071 //Don't use a timeout because putting the hashmap may be a long process
1072 var client = new RestClient(_baseClient) { Timeout = 0 };
1073 if (!String.IsNullOrWhiteSpace(account))
1074 client.BaseAddress = GetAccountUrl(account);
1076 var builder = client.GetAddressBuilder(container, "");
1077 //We are doing an update
1078 builder.Query = "update";
1079 var uri = builder.Uri;
1081 client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
1083 Log.InfoFormat("[BLOCK POST] START");
1085 client.UploadProgressChanged += (sender, args) =>
1086 Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
1087 args.ProgressPercentage, args.BytesSent,
1088 args.TotalBytesToSend);
1089 client.UploadFileCompleted += (sender, args) =>
1090 Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
1094 var uploadTask = client.UploadDataTask(uri, "POST", block)
1095 .ContinueWith(upload =>
1099 if (upload.IsFaulted)
1101 var exception = upload.Exception.InnerException;
1102 Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exception);
1106 Log.InfoFormat("[BLOCK POST] END");
1112 public Task<TreeHash> GetHashMap(string account, string container, string objectName)
1114 if (String.IsNullOrWhiteSpace(container))
1115 throw new ArgumentNullException("container");
1116 if (String.IsNullOrWhiteSpace(objectName))
1117 throw new ArgumentNullException("objectName");
1118 if (String.IsNullOrWhiteSpace(Token))
1119 throw new InvalidOperationException("Invalid Token");
1120 if (StorageUrl == null)
1121 throw new InvalidOperationException("Invalid Storage Url");
1122 Contract.EndContractBlock();
1126 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
1127 //object to avoid concurrency errors.
1129 //Download operations take a long time therefore they have no timeout.
1130 //TODO: Do we really? this is a hashmap operation, not a download
1131 var client = new RestClient(_baseClient) { Timeout = 0 };
1132 if (!String.IsNullOrWhiteSpace(account))
1133 client.BaseAddress = GetAccountUrl(account);
1136 //The container and objectName are relative names. They are joined with the client's
1137 //BaseAddress to create the object's absolute address
1138 var builder = client.GetAddressBuilder(container, objectName);
1139 builder.Query = "format=json&hashmap";
1140 var uri = builder.Uri;
1142 //Start downloading the object asynchronously
1143 var downloadTask = client.DownloadStringTask(uri);
1145 //Once the download completes
1146 return downloadTask.ContinueWith(download =>
1148 //Delete the local client object
1150 //And report failure or completion
1151 if (download.IsFaulted)
1153 Log.ErrorFormat("[GET HASH] FAIL for {0} with \r{1}", objectName,
1154 download.Exception);
1155 throw download.Exception;
1158 //The server will return an empty string if the file is empty
1159 var json = download.Result;
1160 var treeHash = TreeHash.Parse(json);
1161 Log.InfoFormat("[GET HASH] END {0}", objectName);
1165 catch (Exception exc)
1167 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
1179 /// <param name="account"></param>
1180 /// <param name="container"></param>
1181 /// <param name="objectName"></param>
1182 /// <param name="fileName"></param>
1183 /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
1184 /// <remarks>>This method should have no timeout or a very long one</remarks>
1185 public Task PutObject(string account, string container, string objectName, string fileName, string hash = null)
1187 if (String.IsNullOrWhiteSpace(container))
1188 throw new ArgumentNullException("container", "The container property can't be empty");
1189 if (String.IsNullOrWhiteSpace(objectName))
1190 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1191 if (String.IsNullOrWhiteSpace(fileName))
1192 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
1193 if (!File.Exists(fileName))
1194 throw new FileNotFoundException("The file does not exist",fileName);
1195 Contract.EndContractBlock();
1200 var client = new RestClient(_baseClient){Timeout=0};
1201 if (!String.IsNullOrWhiteSpace(account))
1202 client.BaseAddress = GetAccountUrl(account);
1204 var builder = client.GetAddressBuilder(container, objectName);
1205 var uri = builder.Uri;
1207 string etag = hash ?? CalculateHash(fileName);
1209 client.Headers.Add("Content-Type", "application/octet-stream");
1210 client.Headers.Add("ETag", etag);
1213 Log.InfoFormat("[PUT] START {0}", objectName);
1214 client.UploadProgressChanged += (sender, args) =>
1216 using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
1218 Log.InfoFormat("{0} {1}% {2} of {3}", fileName, args.ProgressPercentage,
1219 args.BytesSent, args.TotalBytesToSend);
1223 client.UploadFileCompleted += (sender, args) =>
1225 using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
1227 Log.InfoFormat("Completed {0}", fileName);
1230 return client.UploadFileTask(uri, "PUT", fileName)
1231 .ContinueWith(upload=>
1235 if (upload.IsFaulted)
1237 var exc = upload.Exception.InnerException;
1238 Log.ErrorFormat("[PUT] FAIL for {0} with \r{1}",objectName,exc);
1242 Log.InfoFormat("[PUT] END {0}", objectName);
1245 catch (Exception exc)
1247 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
1254 private static string CalculateHash(string fileName)
1256 Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
1257 Contract.EndContractBlock();
1260 using (var hasher = MD5.Create())
1261 using(var stream=File.OpenRead(fileName))
1263 var hashBuilder=new StringBuilder();
1264 foreach (byte b in hasher.ComputeHash(stream))
1265 hashBuilder.Append(b.ToString("x2").ToLower());
1266 hash = hashBuilder.ToString();
1271 public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
1273 if (String.IsNullOrWhiteSpace(sourceContainer))
1274 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1275 if (String.IsNullOrWhiteSpace(oldObjectName))
1276 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
1277 if (String.IsNullOrWhiteSpace(targetContainer))
1278 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
1279 if (String.IsNullOrWhiteSpace(newObjectName))
1280 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
1281 Contract.EndContractBlock();
1283 var targetUrl = targetContainer + "/" + newObjectName;
1284 var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
1286 using (var client = new RestClient(_baseClient))
1288 if (!String.IsNullOrWhiteSpace(account))
1289 client.BaseAddress = GetAccountUrl(account);
1291 client.Headers.Add("X-Move-From", sourceUrl);
1292 client.PutWithRetry(targetUrl, 3);
1294 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1295 if (!expectedCodes.Contains(client.StatusCode))
1296 throw CreateWebException("MoveObject", client.StatusCode);
1300 public void DeleteObject(string account, string sourceContainer, string objectName)
1302 if (String.IsNullOrWhiteSpace(sourceContainer))
1303 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1304 if (String.IsNullOrWhiteSpace(objectName))
1305 throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
1306 Contract.EndContractBlock();
1308 var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
1309 var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
1311 using (var client = new RestClient(_baseClient))
1313 if (!String.IsNullOrWhiteSpace(account))
1314 client.BaseAddress = GetAccountUrl(account);
1316 client.Headers.Add("X-Move-From", sourceUrl);
1317 client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
1318 client.PutWithRetry(targetUrl, 3);
1320 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
1321 if (!expectedCodes.Contains(client.StatusCode))
1322 throw CreateWebException("DeleteObject", client.StatusCode);
1327 private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
1329 return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
1335 public class ShareAccountInfo
1337 public DateTime? last_modified { get; set; }
1338 public string name { get; set; }