1 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
3 // The class provides methods to upload/download files, delete files, manage containers
7 using System.Collections.Generic;
8 using System.ComponentModel.Composition;
9 using System.Diagnostics;
10 using System.Diagnostics.Contracts;
11 using System.Globalization;
15 using System.Security.Cryptography;
17 using System.Threading.Algorithms;
18 using System.Threading.Tasks;
19 using Newtonsoft.Json;
20 using Pithos.Interfaces;
22 using WebHeaderCollection = System.Net.WebHeaderCollection;
24 namespace Pithos.Network
26 [Export(typeof(ICloudClient))]
27 public class CloudFilesClient:ICloudClient
29 //CloudFilesClient uses *_baseClient* internally to communicate with the server
30 //RestClient provides a REST-friendly interface over the standard WebClient.
31 private RestClient _baseClient;
34 //During authentication the client provides a UserName
35 public string UserName { get; set; }
37 //and and ApiKey to the server
38 public string ApiKey { get; set; }
40 //And receives an authentication Token. This token must be provided in ALL other operations,
41 //in the X-Auth-Token header
42 private string _token;
45 get { return _token; }
49 _baseClient.Headers["X-Auth-Token"] = value;
53 //The client also receives a StorageUrl after authentication. All subsequent operations must
55 public Uri StorageUrl { get; set; }
58 protected Uri RootAddressUri { get; set; }
63 get { return _proxy; }
67 if (_baseClient != null)
68 _baseClient.Proxy = new WebProxy(value);
72 public double DownloadPercentLimit { get; set; }
73 public double UploadPercentLimit { get; set; }
75 public string AuthenticationUrl { get; set; }
78 public string VersionPath
80 get { return UsePithos ? "v1" : "v1.0"; }
83 public bool UsePithos { get; set; }
86 private static readonly ILog Log = LogManager.GetLogger("CloudFilesClient");
88 public CloudFilesClient(string userName, string apiKey)
94 public CloudFilesClient(AccountInfo accountInfo)
96 if (accountInfo==null)
97 throw new ArgumentNullException("accountInfo");
98 Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
99 Contract.Ensures(StorageUrl != null);
100 Contract.Ensures(_baseClient != null);
101 Contract.Ensures(RootAddressUri != null);
102 Contract.EndContractBlock();
104 _baseClient = new RestClient
106 BaseAddress = accountInfo.StorageUri.ToString(),
110 StorageUrl = accountInfo.StorageUri;
111 Token = accountInfo.Token;
112 UserName = accountInfo.UserName;
114 //Get the root address (StorageUrl without the account)
115 var storageUrl = StorageUrl.AbsoluteUri;
116 var usernameIndex = storageUrl.LastIndexOf(UserName);
117 var rootUrl = storageUrl.Substring(0, usernameIndex);
118 RootAddressUri = new Uri(rootUrl);
122 public AccountInfo Authenticate()
124 if (String.IsNullOrWhiteSpace(UserName))
125 throw new InvalidOperationException("UserName is empty");
126 if (String.IsNullOrWhiteSpace(ApiKey))
127 throw new InvalidOperationException("ApiKey is empty");
128 if (String.IsNullOrWhiteSpace(AuthenticationUrl))
129 throw new InvalidOperationException("AuthenticationUrl is empty");
130 Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
131 Contract.Ensures(StorageUrl != null);
132 Contract.Ensures(_baseClient != null);
133 Contract.Ensures(RootAddressUri != null);
134 Contract.EndContractBlock();
137 Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
139 using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
142 authClient.Proxy = new WebProxy(Proxy);
144 Contract.Assume(authClient.Headers!=null);
146 authClient.Headers.Add("X-Auth-User", UserName);
147 authClient.Headers.Add("X-Auth-Key", ApiKey);
149 authClient.DownloadStringWithRetry(VersionPath, 3);
151 authClient.AssertStatusOK("Authentication failed");
153 var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
154 if (String.IsNullOrWhiteSpace(storageUrl))
155 throw new InvalidOperationException("Failed to obtain storage url");
157 _baseClient = new RestClient
159 BaseAddress = storageUrl,
164 StorageUrl = new Uri(storageUrl);
166 //Get the root address (StorageUrl without the account)
167 var usernameIndex=storageUrl.LastIndexOf(UserName);
168 var rootUrl = storageUrl.Substring(0, usernameIndex);
169 RootAddressUri = new Uri(rootUrl);
171 var token = authClient.GetHeaderValue("X-Auth-Token");
172 if (String.IsNullOrWhiteSpace(token))
173 throw new InvalidOperationException("Failed to obtain token url");
178 Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
180 return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName};
186 public IList<ContainerInfo> ListContainers(string account)
189 using (var client = new RestClient(_baseClient))
191 if (!String.IsNullOrWhiteSpace(account))
192 client.BaseAddress = GetAccountUrl(account);
194 client.Parameters.Clear();
195 client.Parameters.Add("format", "json");
196 var content = client.DownloadStringWithRetry("", 3);
197 client.AssertStatusOK("List Containers failed");
199 if (client.StatusCode == HttpStatusCode.NoContent)
200 return new List<ContainerInfo>();
201 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
207 private string GetAccountUrl(string account)
209 return new Uri(this.RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
212 public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
214 using (log4net.ThreadContext.Stacks["Share"].Push("List Accounts"))
216 if (Log.IsDebugEnabled) Log.DebugFormat("START");
218 using (var client = new RestClient(_baseClient))
220 client.Parameters.Clear();
221 client.Parameters.Add("format", "json");
222 client.IfModifiedSince = since;
224 //Extract the username from the base address
225 client.BaseAddress = RootAddressUri.AbsoluteUri;
227 var content = client.DownloadStringWithRetry(@"", 3);
229 client.AssertStatusOK("ListSharingAccounts failed");
231 //If the result is empty, return an empty list,
232 var infos = String.IsNullOrWhiteSpace(content)
233 ? new List<ShareAccountInfo>()
234 //Otherwise deserialize the account list into a list of ShareAccountInfos
235 : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
237 Log.DebugFormat("END");
243 //Request listing of all objects in a container modified since a specific time.
244 //If the *since* value is missing, return all objects
245 public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
248 using (log4net.ThreadContext.Stacks["Share"].Push("List Objects"))
250 if (Log.IsDebugEnabled) Log.DebugFormat("START");
252 var objects = new List<ObjectInfo>();
253 var accounts = ListSharingAccounts(since);
254 foreach (var account in accounts)
256 //Skip the account if it hasn't been modified
257 if (account.last_modified < since)
260 var containers = ListContainers(account.name);
261 foreach (var container in containers)
263 var containerObjects = ListObjects(account.name, container.Name, since);
264 objects.AddRange(containerObjects);
267 if (Log.IsDebugEnabled) Log.DebugFormat("END");
272 public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
274 if (String.IsNullOrWhiteSpace(Token))
275 throw new InvalidOperationException("The Token is not set");
276 if (StorageUrl==null)
277 throw new InvalidOperationException("The StorageUrl is not set");
278 if (String.IsNullOrWhiteSpace(container))
279 throw new ArgumentNullException("container");
280 if (String.IsNullOrWhiteSpace(objectName))
281 throw new ArgumentNullException("objectName");
282 if (String.IsNullOrWhiteSpace(account))
283 throw new ArgumentNullException("account");
284 if (String.IsNullOrWhiteSpace(shareTo))
285 throw new ArgumentNullException("shareTo");
286 Contract.EndContractBlock();
288 using (log4net.ThreadContext.Stacks["Share"].Push("Share Object"))
290 if (Log.IsDebugEnabled) Log.DebugFormat("START");
292 using (var client = new RestClient(_baseClient))
295 client.BaseAddress = GetAccountUrl(account);
297 client.Parameters.Clear();
298 client.Parameters.Add("format", "json");
300 string permission = "";
302 permission = String.Format("write={0}", shareTo);
304 permission = String.Format("read={0}", shareTo);
305 client.Headers.Add("X-Object-Sharing", permission);
307 var content = client.DownloadStringWithRetry(container, 3);
309 client.AssertStatusOK("ShareObject failed");
311 //If the result is empty, return an empty list,
312 var infos = String.IsNullOrWhiteSpace(content)
313 ? new List<ObjectInfo>()
314 //Otherwise deserialize the object list into a list of ObjectInfos
315 : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
317 if (Log.IsDebugEnabled) Log.DebugFormat("END");
325 public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
327 if (String.IsNullOrWhiteSpace(container))
328 throw new ArgumentNullException("container");
329 Contract.EndContractBlock();
331 using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
333 if (Log.IsDebugEnabled) Log.DebugFormat("START");
335 using (var client = new RestClient(_baseClient))
337 if (!String.IsNullOrWhiteSpace(account))
338 client.BaseAddress = GetAccountUrl(account);
340 client.Parameters.Clear();
341 client.Parameters.Add("format", "json");
342 client.IfModifiedSince = since;
343 var content = client.DownloadStringWithRetry(container, 3);
345 client.AssertStatusOK("ListObjects failed");
347 //If the result is empty, return an empty list,
348 var infos = String.IsNullOrWhiteSpace(content)
349 ? new List<ObjectInfo>()
350 //Otherwise deserialize the object list into a list of ObjectInfos
351 : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
353 foreach (var info in infos)
355 info.Container = container;
356 info.Account = account;
358 if (Log.IsDebugEnabled) Log.DebugFormat("START");
366 public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
368 if (String.IsNullOrWhiteSpace(container))
369 throw new ArgumentNullException("container");
370 if (String.IsNullOrWhiteSpace(folder))
371 throw new ArgumentNullException("folder");
372 Contract.EndContractBlock();
374 using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
376 if (Log.IsDebugEnabled) Log.DebugFormat("START");
378 using (var client = new RestClient(_baseClient))
380 if (!String.IsNullOrWhiteSpace(account))
381 client.BaseAddress = GetAccountUrl(account);
383 client.Parameters.Clear();
384 client.Parameters.Add("format", "json");
385 client.Parameters.Add("path", folder);
386 client.IfModifiedSince = since;
387 var content = client.DownloadStringWithRetry(container, 3);
388 client.AssertStatusOK("ListObjects failed");
390 var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
392 if (Log.IsDebugEnabled) Log.DebugFormat("END");
399 public bool ContainerExists(string account, string container)
401 if (String.IsNullOrWhiteSpace(container))
402 throw new ArgumentNullException("container", "The container property can't be empty");
403 Contract.EndContractBlock();
405 using (log4net.ThreadContext.Stacks["Containters"].Push("Exists"))
407 if (Log.IsDebugEnabled) Log.DebugFormat("START");
409 using (var client = new RestClient(_baseClient))
411 if (!String.IsNullOrWhiteSpace(account))
412 client.BaseAddress = GetAccountUrl(account);
414 client.Parameters.Clear();
415 client.Head(container, 3);
418 switch (client.StatusCode)
420 case HttpStatusCode.OK:
421 case HttpStatusCode.NoContent:
424 case HttpStatusCode.NotFound:
428 throw CreateWebException("ContainerExists", client.StatusCode);
430 if (Log.IsDebugEnabled) Log.DebugFormat("END");
438 public bool ObjectExists(string account, string container, string objectName)
440 if (String.IsNullOrWhiteSpace(container))
441 throw new ArgumentNullException("container", "The container property can't be empty");
442 if (String.IsNullOrWhiteSpace(objectName))
443 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
444 Contract.EndContractBlock();
446 using (var client = new RestClient(_baseClient))
448 if (!String.IsNullOrWhiteSpace(account))
449 client.BaseAddress = GetAccountUrl(account);
451 client.Parameters.Clear();
452 client.Head(container + "/" + objectName, 3);
454 switch (client.StatusCode)
456 case HttpStatusCode.OK:
457 case HttpStatusCode.NoContent:
459 case HttpStatusCode.NotFound:
462 throw CreateWebException("ObjectExists", client.StatusCode);
468 public ObjectInfo GetObjectInfo(string account, string container, string objectName)
470 if (String.IsNullOrWhiteSpace(container))
471 throw new ArgumentNullException("container", "The container property can't be empty");
472 if (String.IsNullOrWhiteSpace(objectName))
473 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
474 Contract.EndContractBlock();
476 using (log4net.ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
479 using (var client = new RestClient(_baseClient))
481 if (!String.IsNullOrWhiteSpace(account))
482 client.BaseAddress = GetAccountUrl(account);
485 client.Parameters.Clear();
487 client.Head(container + "/" + objectName, 3);
490 return ObjectInfo.Empty;
492 switch (client.StatusCode)
494 case HttpStatusCode.OK:
495 case HttpStatusCode.NoContent:
496 var keys = client.ResponseHeaders.AllKeys.AsQueryable();
497 var tags = (from key in keys
498 where key.StartsWith("X-Object-Meta-")
499 let name = key.Substring(14)
500 select new {Name = name, Value = client.ResponseHeaders[name]})
501 .ToDictionary(t => t.Name, t => t.Value);
502 var extensions = (from key in keys
503 where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
504 select new {Name = key, Value = client.ResponseHeaders[key]})
505 .ToDictionary(t => t.Name, t => t.Value);
506 var info = new ObjectInfo
509 Hash = client.GetHeaderValue("ETag"),
510 Content_Type = client.GetHeaderValue("Content-Type"),
512 Last_Modified = client.LastModified,
513 Extensions = extensions
516 case HttpStatusCode.NotFound:
517 return ObjectInfo.Empty;
519 throw new WebException(
520 String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
521 objectName, client.StatusCode));
525 catch (RetryException)
527 Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.");
528 return ObjectInfo.Empty;
530 catch (WebException e)
533 String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
534 objectName, client.StatusCode), e);
542 public void CreateFolder(string account, string container, string folder)
544 if (String.IsNullOrWhiteSpace(container))
545 throw new ArgumentNullException("container", "The container property can't be empty");
546 if (String.IsNullOrWhiteSpace(folder))
547 throw new ArgumentNullException("folder", "The folder property can't be empty");
548 Contract.EndContractBlock();
550 var folderUrl=String.Format("{0}/{1}",container,folder);
551 using (var client = new RestClient(_baseClient))
553 if (!String.IsNullOrWhiteSpace(account))
554 client.BaseAddress = GetAccountUrl(account);
556 client.Parameters.Clear();
557 client.Headers.Add("Content-Type", @"application/directory");
558 client.Headers.Add("Content-Length", "0");
559 client.PutWithRetry(folderUrl, 3);
561 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
562 throw CreateWebException("CreateFolder", client.StatusCode);
566 public ContainerInfo GetContainerInfo(string account, string container)
568 if (String.IsNullOrWhiteSpace(container))
569 throw new ArgumentNullException("container", "The container property can't be empty");
570 Contract.EndContractBlock();
572 using (var client = new RestClient(_baseClient))
574 if (!String.IsNullOrWhiteSpace(account))
575 client.BaseAddress = GetAccountUrl(account);
577 client.Head(container);
578 switch (client.StatusCode)
580 case HttpStatusCode.OK:
581 case HttpStatusCode.NoContent:
582 var containerInfo = new ContainerInfo
586 long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
587 Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
588 BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
589 BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size"))
591 return containerInfo;
592 case HttpStatusCode.NotFound:
593 return ContainerInfo.Empty;
595 throw CreateWebException("GetContainerInfo", client.StatusCode);
600 public void CreateContainer(string account, string container)
602 if (String.IsNullOrWhiteSpace(container))
603 throw new ArgumentNullException("container", "The container property can't be empty");
604 Contract.EndContractBlock();
606 using (var client = new RestClient(_baseClient))
608 if (!String.IsNullOrWhiteSpace(account))
609 client.BaseAddress = GetAccountUrl(account);
611 client.PutWithRetry(container, 3);
612 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
613 if (!expectedCodes.Contains(client.StatusCode))
614 throw CreateWebException("CreateContainer", client.StatusCode);
618 public void DeleteContainer(string account, string container)
620 if (String.IsNullOrWhiteSpace(container))
621 throw new ArgumentNullException("container", "The container property can't be empty");
622 Contract.EndContractBlock();
624 using (var client = new RestClient(_baseClient))
626 if (!String.IsNullOrWhiteSpace(account))
627 client.BaseAddress = GetAccountUrl(account);
629 client.DeleteWithRetry(container, 3);
630 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
631 if (!expectedCodes.Contains(client.StatusCode))
632 throw CreateWebException("DeleteContainer", client.StatusCode);
640 /// <param name="account"></param>
641 /// <param name="container"></param>
642 /// <param name="objectName"></param>
643 /// <param name="fileName"></param>
644 /// <returns></returns>
645 /// <remarks>This method should have no timeout or a very long one</remarks>
646 //Asynchronously download the object specified by *objectName* in a specific *container* to
648 public Task GetObject(string account, string container, string objectName, string fileName)
650 if (String.IsNullOrWhiteSpace(container))
651 throw new ArgumentNullException("container", "The container property can't be empty");
652 if (String.IsNullOrWhiteSpace(objectName))
653 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
654 Contract.EndContractBlock();
658 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
659 //object to avoid concurrency errors.
661 //Download operations take a long time therefore they have no timeout.
662 var client = new RestClient(_baseClient) { Timeout = 0 };
663 if (!String.IsNullOrWhiteSpace(account))
664 client.BaseAddress = GetAccountUrl(account);
666 //The container and objectName are relative names. They are joined with the client's
667 //BaseAddress to create the object's absolute address
668 var builder = client.GetAddressBuilder(container, objectName);
669 var uri = builder.Uri;
671 //Download progress is reported to the Trace log
672 Log.InfoFormat("[GET] START {0}", objectName);
673 client.DownloadProgressChanged += (sender, args) =>
674 Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
675 fileName, args.ProgressPercentage,
677 args.TotalBytesToReceive);
680 //Start downloading the object asynchronously
681 var downloadTask = client.DownloadFileTask(uri, fileName);
683 //Once the download completes
684 return downloadTask.ContinueWith(download =>
686 //Delete the local client object
688 //And report failure or completion
689 if (download.IsFaulted)
691 Log.ErrorFormat("[GET] FAIL for {0} with \r{1}", objectName,
696 Log.InfoFormat("[GET] END {0}", objectName);
700 catch (Exception exc)
702 Log.ErrorFormat("[GET] END {0} with {1}", objectName, exc);
710 public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
712 if (String.IsNullOrWhiteSpace(container))
713 throw new ArgumentNullException("container");
714 if (String.IsNullOrWhiteSpace(objectName))
715 throw new ArgumentNullException("objectName");
717 throw new ArgumentNullException("hash");
718 if (String.IsNullOrWhiteSpace(Token))
719 throw new InvalidOperationException("Invalid Token");
720 if (StorageUrl == null)
721 throw new InvalidOperationException("Invalid Storage Url");
722 Contract.EndContractBlock();
725 //Don't use a timeout because putting the hashmap may be a long process
726 var client = new RestClient(_baseClient) { Timeout = 0 };
727 if (!String.IsNullOrWhiteSpace(account))
728 client.BaseAddress = GetAccountUrl(account);
730 //The container and objectName are relative names. They are joined with the client's
731 //BaseAddress to create the object's absolute address
732 var builder = client.GetAddressBuilder(container, objectName);
733 builder.Query = "format=json&hashmap";
734 var uri = builder.Uri;
737 //Send the tree hash as Json to the server
738 client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
739 var uploadTask=client.UploadStringTask(uri, "PUT", hash.ToJson());
742 return uploadTask.ContinueWith(t =>
745 var empty = (IList<string>)new List<string>();
748 //The server will respond either with 201-created if all blocks were already on the server
749 if (client.StatusCode == HttpStatusCode.Created)
751 //in which case we return an empty hash list
754 //or with a 409-conflict and return the list of missing parts
755 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
758 var ex = t.Exception.InnerException;
759 var we = ex as WebException;
760 var response = we.Response as HttpWebResponse;
761 if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
763 //In case of 409 the missing parts will be in the response content
764 using (var stream = response.GetResponseStream())
765 using(var reader=new StreamReader(stream))
767 //We need to cleanup the content before returning it because it contains
768 //error content after the list of hashes
769 var hashes = new List<string>();
771 //All lines up to the first empty line are hashes
772 while(!String.IsNullOrWhiteSpace(line=reader.ReadLine()))
781 //Any other status code is unexpected and the exception should be rethrown
785 //Any other status code is unexpected but there was no exception. We can probably continue processing
788 Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
795 public Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end)
797 if (String.IsNullOrWhiteSpace(Token))
798 throw new InvalidOperationException("Invalid Token");
799 if (StorageUrl == null)
800 throw new InvalidOperationException("Invalid Storage Url");
801 if (String.IsNullOrWhiteSpace(container))
802 throw new ArgumentNullException("container");
803 if (relativeUrl== null)
804 throw new ArgumentNullException("relativeUrl");
805 if (end.HasValue && end<0)
806 throw new ArgumentOutOfRangeException("end");
808 throw new ArgumentOutOfRangeException("start");
809 Contract.EndContractBlock();
812 //Don't use a timeout because putting the hashmap may be a long process
813 var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
814 if (!String.IsNullOrWhiteSpace(account))
815 client.BaseAddress = GetAccountUrl(account);
817 var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
818 var uri = builder.Uri;
820 return client.DownloadDataTask(uri)
829 public Task PostBlock(string account, string container, byte[] block, int offset, int count)
831 if (String.IsNullOrWhiteSpace(container))
832 throw new ArgumentNullException("container");
834 throw new ArgumentNullException("block");
835 if (offset < 0 || offset >= block.Length)
836 throw new ArgumentOutOfRangeException("offset");
837 if (count < 0 || count > block.Length)
838 throw new ArgumentOutOfRangeException("count");
839 if (String.IsNullOrWhiteSpace(Token))
840 throw new InvalidOperationException("Invalid Token");
841 if (StorageUrl == null)
842 throw new InvalidOperationException("Invalid Storage Url");
843 Contract.EndContractBlock();
846 //Don't use a timeout because putting the hashmap may be a long process
847 var client = new RestClient(_baseClient) { Timeout = 0 };
848 if (!String.IsNullOrWhiteSpace(account))
849 client.BaseAddress = GetAccountUrl(account);
851 var builder = client.GetAddressBuilder(container, "");
852 //We are doing an update
853 builder.Query = "update";
854 var uri = builder.Uri;
856 client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
858 Log.InfoFormat("[BLOCK POST] START");
860 client.UploadProgressChanged += (sender, args) =>
861 Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
862 args.ProgressPercentage, args.BytesSent,
863 args.TotalBytesToSend);
864 client.UploadFileCompleted += (sender, args) =>
865 Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
869 var uploadTask = client.UploadDataTask(uri, "POST", block)
870 .ContinueWith(upload =>
874 if (upload.IsFaulted)
876 var exception = upload.Exception.InnerException;
877 Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exception);
881 Log.InfoFormat("[BLOCK POST] END");
887 public Task<TreeHash> GetHashMap(string account, string container, string objectName)
889 if (String.IsNullOrWhiteSpace(container))
890 throw new ArgumentNullException("container");
891 if (String.IsNullOrWhiteSpace(objectName))
892 throw new ArgumentNullException("objectName");
893 if (String.IsNullOrWhiteSpace(Token))
894 throw new InvalidOperationException("Invalid Token");
895 if (StorageUrl == null)
896 throw new InvalidOperationException("Invalid Storage Url");
897 Contract.EndContractBlock();
901 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
902 //object to avoid concurrency errors.
904 //Download operations take a long time therefore they have no timeout.
905 //TODO: Do we really? this is a hashmap operation, not a download
906 var client = new RestClient(_baseClient) { Timeout = 0 };
907 if (!String.IsNullOrWhiteSpace(account))
908 client.BaseAddress = GetAccountUrl(account);
911 //The container and objectName are relative names. They are joined with the client's
912 //BaseAddress to create the object's absolute address
913 var builder = client.GetAddressBuilder(container, objectName);
914 builder.Query = "format=json&hashmap";
915 var uri = builder.Uri;
917 //Start downloading the object asynchronously
918 var downloadTask = client.DownloadStringTask(uri);
920 //Once the download completes
921 return downloadTask.ContinueWith(download =>
923 //Delete the local client object
925 //And report failure or completion
926 if (download.IsFaulted)
928 Log.ErrorFormat("[GET HASH] FAIL for {0} with \r{1}", objectName,
930 throw download.Exception;
933 //The server will return an empty string if the file is empty
934 var json = download.Result;
935 var treeHash = TreeHash.Parse(json);
936 Log.InfoFormat("[GET HASH] END {0}", objectName);
940 catch (Exception exc)
942 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
954 /// <param name="account"></param>
955 /// <param name="container"></param>
956 /// <param name="objectName"></param>
957 /// <param name="fileName"></param>
958 /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
959 /// <remarks>>This method should have no timeout or a very long one</remarks>
960 public Task PutObject(string account, string container, string objectName, string fileName, string hash = null)
962 if (String.IsNullOrWhiteSpace(container))
963 throw new ArgumentNullException("container", "The container property can't be empty");
964 if (String.IsNullOrWhiteSpace(objectName))
965 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
966 if (String.IsNullOrWhiteSpace(fileName))
967 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
968 if (!File.Exists(fileName))
969 throw new FileNotFoundException("The file does not exist",fileName);
970 Contract.EndContractBlock();
975 var client = new RestClient(_baseClient){Timeout=0};
976 if (!String.IsNullOrWhiteSpace(account))
977 client.BaseAddress = GetAccountUrl(account);
979 var builder = client.GetAddressBuilder(container, objectName);
980 var uri = builder.Uri;
982 string etag = hash ?? CalculateHash(fileName);
984 client.Headers.Add("Content-Type", "application/octet-stream");
985 client.Headers.Add("ETag", etag);
988 Log.InfoFormat("[PUT] START {0}", objectName);
989 client.UploadProgressChanged += (sender, args) =>
991 using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
993 Log.InfoFormat("{0} {1}% {2} of {3}", fileName, args.ProgressPercentage,
994 args.BytesSent, args.TotalBytesToSend);
998 client.UploadFileCompleted += (sender, args) =>
1000 using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
1002 Log.InfoFormat("Completed {0}", fileName);
1005 return client.UploadFileTask(uri, "PUT", fileName)
1006 .ContinueWith(upload=>
1010 if (upload.IsFaulted)
1012 var exc = upload.Exception.InnerException;
1013 Log.ErrorFormat("[PUT] FAIL for {0} with \r{1}",objectName,exc);
1017 Log.InfoFormat("[PUT] END {0}", objectName);
1020 catch (Exception exc)
1022 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
1029 private static string CalculateHash(string fileName)
1032 using (var hasher = MD5.Create())
1033 using(var stream=File.OpenRead(fileName))
1035 var hashBuilder=new StringBuilder();
1036 foreach (byte b in hasher.ComputeHash(stream))
1037 hashBuilder.Append(b.ToString("x2").ToLower());
1038 hash = hashBuilder.ToString();
1043 /* public void DeleteObject(string container, string objectName,string account)
1045 if (String.IsNullOrWhiteSpace(container))
1046 throw new ArgumentNullException("container", "The container property can't be empty");
1047 if (String.IsNullOrWhiteSpace(objectName))
1048 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1049 Contract.EndContractBlock();
1051 using (var client = new RestClient(_baseClient))
1053 if (!String.IsNullOrWhiteSpace(account))
1054 client.BaseAddress = GetAccountUrl(account);
1056 client.DeleteWithRetry(container + "/" + objectName, 3);
1058 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
1059 if (!expectedCodes.Contains(client.StatusCode))
1060 throw CreateWebException("DeleteObject", client.StatusCode);
1065 public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
1067 if (String.IsNullOrWhiteSpace(sourceContainer))
1068 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1069 if (String.IsNullOrWhiteSpace(oldObjectName))
1070 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
1071 if (String.IsNullOrWhiteSpace(targetContainer))
1072 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
1073 if (String.IsNullOrWhiteSpace(newObjectName))
1074 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
1075 Contract.EndContractBlock();
1077 var targetUrl = targetContainer + "/" + newObjectName;
1078 var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
1080 using (var client = new RestClient(_baseClient))
1082 if (!String.IsNullOrWhiteSpace(account))
1083 client.BaseAddress = GetAccountUrl(account);
1085 client.Headers.Add("X-Move-From", sourceUrl);
1086 client.PutWithRetry(targetUrl, 3);
1088 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1089 if (!expectedCodes.Contains(client.StatusCode))
1090 throw CreateWebException("MoveObject", client.StatusCode);
1094 public void DeleteObject(string account, string sourceContainer, string objectName)
1096 if (String.IsNullOrWhiteSpace(sourceContainer))
1097 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1098 if (String.IsNullOrWhiteSpace(objectName))
1099 throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
1100 Contract.EndContractBlock();
1102 var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
1103 var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
1105 using (var client = new RestClient(_baseClient))
1107 if (!String.IsNullOrWhiteSpace(account))
1108 client.BaseAddress = GetAccountUrl(account);
1110 client.Headers.Add("X-Move-From", sourceUrl);
1111 client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
1112 client.PutWithRetry(targetUrl, 3);
1114 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
1115 if (!expectedCodes.Contains(client.StatusCode))
1116 throw CreateWebException("DeleteObject", client.StatusCode);
1121 private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
1123 return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
1129 public class ShareAccountInfo
1131 public DateTime? last_modified { get; set; }
1132 public string name { get; set; }