2 /* -----------------------------------------------------------------------
3 * <copyright file="CloudFilesClient.cs" company="GRNet">
5 * Copyright 2011-2012 GRNET S.A. All rights reserved.
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
11 * 1. Redistributions of source code must retain the above
12 * copyright notice, this list of conditions and the following
15 * 2. Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials
18 * provided with the distribution.
21 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
34 * The views and conclusions contained in the software and
35 * documentation are those of the authors and should not be
36 * interpreted as representing official policies, either expressed
37 * or implied, of GRNET S.A.
39 * -----------------------------------------------------------------------
43 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
45 // The class provides methods to upload/download files, delete files, manage containers
49 using System.Collections.Generic;
50 using System.Collections.Specialized;
51 using System.ComponentModel.Composition;
52 using System.Diagnostics;
53 using System.Diagnostics.Contracts;
57 using System.Reflection;
58 using System.Security.Cryptography;
60 using System.Threading;
61 using System.Threading.Tasks;
62 using Newtonsoft.Json;
63 using Pithos.Interfaces;
66 namespace Pithos.Network
68 [Export(typeof(ICloudClient))]
69 public class CloudFilesClient:ICloudClient
71 private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
73 //CloudFilesClient uses *_baseClient* internally to communicate with the server
74 //RestClient provides a REST-friendly interface over the standard WebClient.
75 private RestClient _baseClient;
78 //During authentication the client provides a UserName
79 public string UserName { get; set; }
81 //and and ApiKey to the server
82 public string ApiKey { get; set; }
84 //And receives an authentication Token. This token must be provided in ALL other operations,
85 //in the X-Auth-Token header
86 private string _token;
89 get { return _token; }
93 _baseClient.Headers["X-Auth-Token"] = value;
97 //The client also receives a StorageUrl after authentication. All subsequent operations must
99 public Uri StorageUrl { get; set; }
102 protected Uri RootAddressUri { get; set; }
104 /* private WebProxy _proxy;
105 public WebProxy Proxy
107 get { return _proxy; }
111 if (_baseClient != null)
112 _baseClient.Proxy = value;
117 /* private Uri _proxy;
120 get { return _proxy; }
124 if (_baseClient != null)
125 _baseClient.Proxy = new WebProxy(value);
129 public double DownloadPercentLimit { get; set; }
130 public double UploadPercentLimit { get; set; }
132 public string AuthenticationUrl { get; set; }
135 public string VersionPath
137 get { return UsePithos ? "v1" : "v1.0"; }
140 public bool UsePithos { get; set; }
144 public CloudFilesClient(string userName, string apiKey)
150 public CloudFilesClient(AccountInfo accountInfo)
152 if (accountInfo==null)
153 throw new ArgumentNullException("accountInfo");
154 Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
155 Contract.Ensures(StorageUrl != null);
156 Contract.Ensures(_baseClient != null);
157 Contract.Ensures(RootAddressUri != null);
158 Contract.EndContractBlock();
160 _baseClient = new RestClient
162 BaseAddress = accountInfo.StorageUri.ToString(),
166 StorageUrl = accountInfo.StorageUri;
167 Token = accountInfo.Token;
168 UserName = accountInfo.UserName;
170 //Get the root address (StorageUrl without the account)
171 var storageUrl = StorageUrl.AbsoluteUri;
172 var usernameIndex = storageUrl.LastIndexOf(UserName);
173 var rootUrl = storageUrl.Substring(0, usernameIndex);
174 RootAddressUri = new Uri(rootUrl);
178 public AccountInfo Authenticate()
180 if (String.IsNullOrWhiteSpace(UserName))
181 throw new InvalidOperationException("UserName is empty");
182 if (String.IsNullOrWhiteSpace(ApiKey))
183 throw new InvalidOperationException("ApiKey is empty");
184 if (String.IsNullOrWhiteSpace(AuthenticationUrl))
185 throw new InvalidOperationException("AuthenticationUrl is empty");
186 Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
187 Contract.Ensures(StorageUrl != null);
188 Contract.Ensures(_baseClient != null);
189 Contract.Ensures(RootAddressUri != null);
190 Contract.EndContractBlock();
193 Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
195 var groups = new List<Group>();
197 using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
199 /* if (Proxy != null)
200 authClient.Proxy = Proxy;*/
202 Contract.Assume(authClient.Headers!=null);
204 authClient.Headers.Add("X-Auth-User", UserName);
205 authClient.Headers.Add("X-Auth-Key", ApiKey);
207 authClient.DownloadStringWithRetry(VersionPath, 3);
209 authClient.AssertStatusOK("Authentication failed");
211 var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
212 if (String.IsNullOrWhiteSpace(storageUrl))
213 throw new InvalidOperationException("Failed to obtain storage url");
215 _baseClient = new RestClient
217 BaseAddress = storageUrl,
223 StorageUrl = new Uri(storageUrl);
225 //Get the root address (StorageUrl without the account)
226 var usernameIndex=storageUrl.LastIndexOf(UserName);
227 var rootUrl = storageUrl.Substring(0, usernameIndex);
228 RootAddressUri = new Uri(rootUrl);
230 var token = authClient.GetHeaderValue("X-Auth-Token");
231 if (String.IsNullOrWhiteSpace(token))
232 throw new InvalidOperationException("Failed to obtain token url");
235 /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
236 groups = (from key in keys
237 where key.StartsWith("X-Account-Group-")
238 let name = key.Substring(16)
239 select new Group(name, authClient.ResponseHeaders[key]))
245 Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
246 Debug.Assert(_baseClient!=null);
248 return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};
254 public IList<ContainerInfo> ListContainers(string account)
256 using (var client = new RestClient(_baseClient))
258 if (!String.IsNullOrWhiteSpace(account))
259 client.BaseAddress = GetAccountUrl(account);
261 client.Parameters.Clear();
262 client.Parameters.Add("format", "json");
263 var content = client.DownloadStringWithRetry("", 3);
264 client.AssertStatusOK("List Containers failed");
266 if (client.StatusCode == HttpStatusCode.NoContent)
267 return new List<ContainerInfo>();
268 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
270 foreach (var info in infos)
272 info.Account = account;
279 private string GetAccountUrl(string account)
281 return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
284 public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
286 using (ThreadContext.Stacks["Share"].Push("List Accounts"))
288 if (Log.IsDebugEnabled) Log.DebugFormat("START");
290 using (var client = new RestClient(_baseClient))
292 client.Parameters.Clear();
293 client.Parameters.Add("format", "json");
294 client.IfModifiedSince = since;
296 //Extract the username from the base address
297 client.BaseAddress = RootAddressUri.AbsoluteUri;
299 var content = client.DownloadStringWithRetry(@"", 3);
301 client.AssertStatusOK("ListSharingAccounts failed");
303 //If the result is empty, return an empty list,
304 var infos = String.IsNullOrWhiteSpace(content)
305 ? new List<ShareAccountInfo>()
306 //Otherwise deserialize the account list into a list of ShareAccountInfos
307 : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
309 Log.DebugFormat("END");
315 //Request listing of all objects in a container modified since a specific time.
316 //If the *since* value is missing, return all objects
317 public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
320 using (ThreadContext.Stacks["Share"].Push("List Objects"))
322 if (Log.IsDebugEnabled) Log.DebugFormat("START");
323 //'since' is not used here because we need to have ListObjects return a NoChange result
324 //for all shared accounts,containers
325 var accounts = ListSharingAccounts();
326 var items = from account in accounts
327 let containers = ListContainers(account.name)
328 from container in containers
329 select ListObjects(account.name, container.Name,since);
330 var objects=items.SelectMany(r=> r).ToList();
332 var objects = new List<ObjectInfo>();
333 foreach (var containerObjects in items)
335 objects.AddRange(containerObjects);
338 if (Log.IsDebugEnabled) Log.DebugFormat("END");
343 public void SetTags(ObjectInfo target,IDictionary<string,string> tags)
345 if (String.IsNullOrWhiteSpace(Token))
346 throw new InvalidOperationException("The Token is not set");
347 if (StorageUrl == null)
348 throw new InvalidOperationException("The StorageUrl is not set");
350 throw new ArgumentNullException("target");
351 Contract.EndContractBlock();
353 using (ThreadContext.Stacks["Share"].Push("Share Object"))
355 if (Log.IsDebugEnabled) Log.DebugFormat("START");
357 using (var client = new RestClient(_baseClient))
360 client.BaseAddress = GetAccountUrl(target.Account);
362 client.Parameters.Clear();
363 client.Parameters.Add("update", "");
365 foreach (var tag in tags)
367 var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
368 client.Headers.Add(headerTag, tag.Value);
371 client.DownloadStringWithRetry(target.Container, 3);
374 client.AssertStatusOK("SetTags failed");
375 //If the status is NOT ACCEPTED we have a problem
376 if (client.StatusCode != HttpStatusCode.Accepted)
378 Log.Error("Failed to set tags");
379 throw new Exception("Failed to set tags");
382 if (Log.IsDebugEnabled) Log.DebugFormat("END");
389 public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
391 if (String.IsNullOrWhiteSpace(Token))
392 throw new InvalidOperationException("The Token is not set");
393 if (StorageUrl==null)
394 throw new InvalidOperationException("The StorageUrl is not set");
395 if (String.IsNullOrWhiteSpace(container))
396 throw new ArgumentNullException("container");
397 if (String.IsNullOrWhiteSpace(objectName))
398 throw new ArgumentNullException("objectName");
399 if (String.IsNullOrWhiteSpace(account))
400 throw new ArgumentNullException("account");
401 if (String.IsNullOrWhiteSpace(shareTo))
402 throw new ArgumentNullException("shareTo");
403 Contract.EndContractBlock();
405 using (ThreadContext.Stacks["Share"].Push("Share Object"))
407 if (Log.IsDebugEnabled) Log.DebugFormat("START");
409 using (var client = new RestClient(_baseClient))
412 client.BaseAddress = GetAccountUrl(account);
414 client.Parameters.Clear();
415 client.Parameters.Add("format", "json");
417 string permission = "";
419 permission = String.Format("write={0}", shareTo);
421 permission = String.Format("read={0}", shareTo);
422 client.Headers.Add("X-Object-Sharing", permission);
424 var content = client.DownloadStringWithRetry(container, 3);
426 client.AssertStatusOK("ShareObject failed");
428 //If the result is empty, return an empty list,
429 var infos = String.IsNullOrWhiteSpace(content)
430 ? new List<ObjectInfo>()
431 //Otherwise deserialize the object list into a list of ObjectInfos
432 : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
434 if (Log.IsDebugEnabled) Log.DebugFormat("END");
441 public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
443 if (accountInfo==null)
444 throw new ArgumentNullException("accountInfo");
445 Contract.EndContractBlock();
447 using (ThreadContext.Stacks["Account"].Push("GetPolicies"))
449 if (Log.IsDebugEnabled) Log.DebugFormat("START");
451 using (var client = new RestClient(_baseClient))
453 if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
454 client.BaseAddress = GetAccountUrl(accountInfo.UserName);
456 client.Parameters.Clear();
457 client.Parameters.Add("format", "json");
458 client.Head(String.Empty, 3);
460 var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
461 var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
464 if (long.TryParse(quotaValue, out quota))
465 accountInfo.Quota = quota;
466 if (long.TryParse(bytesValue, out bytes))
467 accountInfo.BytesUsed = bytes;
476 public void UpdateMetadata(ObjectInfo objectInfo)
478 if (objectInfo == null)
479 throw new ArgumentNullException("objectInfo");
480 Contract.EndContractBlock();
482 using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
484 if (Log.IsDebugEnabled) Log.DebugFormat("START");
487 using(var client=new RestClient(_baseClient))
490 client.BaseAddress = GetAccountUrl(objectInfo.Account);
492 client.Parameters.Clear();
496 foreach (var tag in objectInfo.Tags)
498 var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
499 client.Headers.Add(headerTag, tag.Value);
504 var permissions=objectInfo.GetPermissionString();
505 client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);
507 client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
508 client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
509 client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
510 var isPublic = objectInfo.IsPublic.ToString().ToLower();
511 client.Headers.Add("X-Object-Public", isPublic);
514 /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);
515 uriBuilder.Query = "update=";
516 var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/
517 var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);
518 client.PostWithRetry(address,"application/xml");
520 //client.UploadValues(uri,new NameValueCollection());
523 client.AssertStatusOK("UpdateMetadata failed");
524 //If the status is NOT ACCEPTED or OK we have a problem
525 if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
527 Log.Error("Failed to update metadata");
528 throw new Exception("Failed to update metadata");
531 if (Log.IsDebugEnabled) Log.DebugFormat("END");
537 public void UpdateMetadata(ContainerInfo containerInfo)
539 if (containerInfo == null)
540 throw new ArgumentNullException("containerInfo");
541 Contract.EndContractBlock();
543 using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
545 if (Log.IsDebugEnabled) Log.DebugFormat("START");
548 using(var client=new RestClient(_baseClient))
551 client.BaseAddress = GetAccountUrl(containerInfo.Account);
553 client.Parameters.Clear();
557 foreach (var tag in containerInfo.Tags)
559 var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
560 client.Headers.Add(headerTag, tag.Value);
565 foreach (var policy in containerInfo.Policies)
567 var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
568 client.Headers.Add(headerPolicy, policy.Value);
572 var uriBuilder = client.GetAddressBuilder(containerInfo.Name,"");
573 var uri = uriBuilder.Uri;
575 client.UploadValues(uri,new NameValueCollection());
578 client.AssertStatusOK("UpdateMetadata failed");
579 //If the status is NOT ACCEPTED or OK we have a problem
580 if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
582 Log.Error("Failed to update metadata");
583 throw new Exception("Failed to update metadata");
586 if (Log.IsDebugEnabled) Log.DebugFormat("END");
593 public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
595 if (String.IsNullOrWhiteSpace(container))
596 throw new ArgumentNullException("container");
597 Contract.EndContractBlock();
599 using (ThreadContext.Stacks["Objects"].Push("List"))
601 if (Log.IsDebugEnabled) Log.DebugFormat("START");
603 using (var client = new RestClient(_baseClient))
605 if (!String.IsNullOrWhiteSpace(account))
606 client.BaseAddress = GetAccountUrl(account);
608 client.Parameters.Clear();
609 client.Parameters.Add("format", "json");
610 client.IfModifiedSince = since;
611 var content = client.DownloadStringWithRetry(container, 3);
613 client.AssertStatusOK("ListObjects failed");
615 if (client.StatusCode==HttpStatusCode.NotModified)
616 return new[]{new NoModificationInfo(account,container)};
617 //If the result is empty, return an empty list,
618 var infos = String.IsNullOrWhiteSpace(content)
619 ? new List<ObjectInfo>()
620 //Otherwise deserialize the object list into a list of ObjectInfos
621 : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
623 foreach (var info in infos)
625 info.Container = container;
626 info.Account = account;
627 info.StorageUri = this.StorageUrl;
629 if (Log.IsDebugEnabled) Log.DebugFormat("END");
635 public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
637 if (String.IsNullOrWhiteSpace(container))
638 throw new ArgumentNullException("container");
640 if (String.IsNullOrWhiteSpace(folder))
641 throw new ArgumentNullException("folder");
643 Contract.EndContractBlock();
645 using (ThreadContext.Stacks["Objects"].Push("List"))
647 if (Log.IsDebugEnabled) Log.DebugFormat("START");
649 using (var client = new RestClient(_baseClient))
651 if (!String.IsNullOrWhiteSpace(account))
652 client.BaseAddress = GetAccountUrl(account);
654 client.Parameters.Clear();
655 client.Parameters.Add("format", "json");
656 client.Parameters.Add("path", folder);
657 client.IfModifiedSince = since;
658 var content = client.DownloadStringWithRetry(container, 3);
659 client.AssertStatusOK("ListObjects failed");
661 if (client.StatusCode==HttpStatusCode.NotModified)
662 return new[]{new NoModificationInfo(account,container,folder)};
664 var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
665 foreach (var info in infos)
667 info.Account = account;
668 if (info.Container == null)
669 info.Container = container;
670 info.StorageUri = this.StorageUrl;
672 if (Log.IsDebugEnabled) Log.DebugFormat("END");
679 public bool ContainerExists(string account, string container)
681 if (String.IsNullOrWhiteSpace(container))
682 throw new ArgumentNullException("container", "The container property can't be empty");
683 Contract.EndContractBlock();
685 using (ThreadContext.Stacks["Containters"].Push("Exists"))
687 if (Log.IsDebugEnabled) Log.DebugFormat("START");
689 using (var client = new RestClient(_baseClient))
691 if (!String.IsNullOrWhiteSpace(account))
692 client.BaseAddress = GetAccountUrl(account);
694 client.Parameters.Clear();
695 client.Head(container, 3);
698 switch (client.StatusCode)
700 case HttpStatusCode.OK:
701 case HttpStatusCode.NoContent:
704 case HttpStatusCode.NotFound:
708 throw CreateWebException("ContainerExists", client.StatusCode);
710 if (Log.IsDebugEnabled) Log.DebugFormat("END");
718 public bool ObjectExists(string account, string container, string objectName)
720 if (String.IsNullOrWhiteSpace(container))
721 throw new ArgumentNullException("container", "The container property can't be empty");
722 if (String.IsNullOrWhiteSpace(objectName))
723 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
724 Contract.EndContractBlock();
726 using (var client = new RestClient(_baseClient))
728 if (!String.IsNullOrWhiteSpace(account))
729 client.BaseAddress = GetAccountUrl(account);
731 client.Parameters.Clear();
732 client.Head(container + "/" + objectName, 3);
734 switch (client.StatusCode)
736 case HttpStatusCode.OK:
737 case HttpStatusCode.NoContent:
739 case HttpStatusCode.NotFound:
742 throw CreateWebException("ObjectExists", client.StatusCode);
748 public ObjectInfo GetObjectInfo(string account, string container, string objectName)
750 if (String.IsNullOrWhiteSpace(container))
751 throw new ArgumentNullException("container", "The container property can't be empty");
752 if (String.IsNullOrWhiteSpace(objectName))
753 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
754 Contract.EndContractBlock();
756 using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
759 using (var client = new RestClient(_baseClient))
761 if (!String.IsNullOrWhiteSpace(account))
762 client.BaseAddress = GetAccountUrl(account);
765 client.Parameters.Clear();
767 client.Head(container + "/" + objectName, 3);
770 return ObjectInfo.Empty;
772 switch (client.StatusCode)
774 case HttpStatusCode.OK:
775 case HttpStatusCode.NoContent:
776 var keys = client.ResponseHeaders.AllKeys.AsQueryable();
777 var tags = client.GetMeta("X-Object-Meta-");
778 var extensions = (from key in keys
779 where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
780 select new {Name = key, Value = client.ResponseHeaders[key]})
781 .ToDictionary(t => t.Name, t => t.Value);
783 var permissions=client.GetHeaderValue("X-Object-Sharing", true);
786 var info = new ObjectInfo
789 Container = container,
791 ETag = client.GetHeaderValue("ETag"),
792 X_Object_Hash = client.GetHeaderValue("X-Object-Hash"),
793 Content_Type = client.GetHeaderValue("Content-Type"),
794 Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),
796 Last_Modified = client.LastModified,
797 Extensions = extensions,
798 ContentEncoding=client.GetHeaderValue("Content-Encoding",true),
799 ContendDisposition = client.GetHeaderValue("Content-Disposition",true),
800 Manifest=client.GetHeaderValue("X-Object-Manifest",true),
801 PublicUrl=client.GetHeaderValue("X-Object-Public",true),
802 StorageUri=this.StorageUrl,
804 info.SetPermissions(permissions);
806 case HttpStatusCode.NotFound:
807 return ObjectInfo.Empty;
809 throw new WebException(
810 String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
811 objectName, client.StatusCode));
815 catch (RetryException)
817 Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);
818 return ObjectInfo.Empty;
820 catch (WebException e)
823 String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
824 objectName, client.StatusCode), e);
832 public void CreateFolder(string account, string container, string folder)
834 if (String.IsNullOrWhiteSpace(container))
835 throw new ArgumentNullException("container", "The container property can't be empty");
836 if (String.IsNullOrWhiteSpace(folder))
837 throw new ArgumentNullException("folder", "The folder property can't be empty");
838 Contract.EndContractBlock();
840 var folderUrl=String.Format("{0}/{1}",container,folder);
841 using (var client = new RestClient(_baseClient))
843 if (!String.IsNullOrWhiteSpace(account))
844 client.BaseAddress = GetAccountUrl(account);
846 client.Parameters.Clear();
847 client.Headers.Add("Content-Type", @"application/directory");
848 client.Headers.Add("Content-Length", "0");
849 client.PutWithRetry(folderUrl, 3);
851 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
852 throw CreateWebException("CreateFolder", client.StatusCode);
858 public ContainerInfo GetContainerInfo(string account, string container)
860 if (String.IsNullOrWhiteSpace(container))
861 throw new ArgumentNullException("container", "The container property can't be empty");
862 Contract.EndContractBlock();
864 using (var client = new RestClient(_baseClient))
866 if (!String.IsNullOrWhiteSpace(account))
867 client.BaseAddress = GetAccountUrl(account);
869 client.Head(container);
870 switch (client.StatusCode)
872 case HttpStatusCode.OK:
873 case HttpStatusCode.NoContent:
874 var tags = client.GetMeta("X-Container-Meta-");
875 var policies = client.GetMeta("X-Container-Policy-");
877 var containerInfo = new ContainerInfo
881 StorageUrl=this.StorageUrl.ToString(),
883 long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
884 Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
885 BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
886 BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),
887 Last_Modified=client.LastModified,
893 return containerInfo;
894 case HttpStatusCode.NotFound:
895 return ContainerInfo.Empty;
897 throw CreateWebException("GetContainerInfo", client.StatusCode);
902 public void CreateContainer(string account, string container)
904 if (String.IsNullOrWhiteSpace(account))
905 throw new ArgumentNullException("account");
906 if (String.IsNullOrWhiteSpace(container))
907 throw new ArgumentNullException("container");
908 Contract.EndContractBlock();
910 using (var client = new RestClient(_baseClient))
912 if (!String.IsNullOrWhiteSpace(account))
913 client.BaseAddress = GetAccountUrl(account);
915 client.PutWithRetry(container, 3);
916 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
917 if (!expectedCodes.Contains(client.StatusCode))
918 throw CreateWebException("CreateContainer", client.StatusCode);
922 public void DeleteContainer(string account, string container)
924 if (String.IsNullOrWhiteSpace(container))
925 throw new ArgumentNullException("container", "The container property can't be empty");
926 Contract.EndContractBlock();
928 using (var client = new RestClient(_baseClient))
930 if (!String.IsNullOrWhiteSpace(account))
931 client.BaseAddress = GetAccountUrl(account);
933 client.DeleteWithRetry(container, 3);
934 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
935 if (!expectedCodes.Contains(client.StatusCode))
936 throw CreateWebException("DeleteContainer", client.StatusCode);
944 /// <param name="account"></param>
945 /// <param name="container"></param>
946 /// <param name="objectName"></param>
947 /// <param name="fileName"></param>
948 /// <returns></returns>
949 /// <remarks>This method should have no timeout or a very long one</remarks>
950 //Asynchronously download the object specified by *objectName* in a specific *container* to
952 public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken)
954 if (String.IsNullOrWhiteSpace(container))
955 throw new ArgumentNullException("container", "The container property can't be empty");
956 if (String.IsNullOrWhiteSpace(objectName))
957 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
958 Contract.EndContractBlock();
962 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
963 //object to avoid concurrency errors.
965 //Download operations take a long time therefore they have no timeout.
966 using(var client = new RestClient(_baseClient) { Timeout = 0 })
968 if (!String.IsNullOrWhiteSpace(account))
969 client.BaseAddress = GetAccountUrl(account);
971 //The container and objectName are relative names. They are joined with the client's
972 //BaseAddress to create the object's absolute address
973 var builder = client.GetAddressBuilder(container, objectName);
974 var uri = builder.Uri;
976 //Download progress is reported to the Trace log
977 Log.InfoFormat("[GET] START {0}", objectName);
978 client.DownloadProgressChanged += (sender, args) =>
979 Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
980 fileName, args.ProgressPercentage,
982 args.TotalBytesToReceive);
985 //Start downloading the object asynchronously
986 await client.DownloadFileTaskAsync(uri, fileName,cancellationToken);
988 //Once the download completes
989 //Delete the local client object
991 //And report failure or completion
993 catch (Exception exc)
995 Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);
999 Log.InfoFormat("[GET] END {0}", objectName);
1004 public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
1006 if (String.IsNullOrWhiteSpace(container))
1007 throw new ArgumentNullException("container");
1008 if (String.IsNullOrWhiteSpace(objectName))
1009 throw new ArgumentNullException("objectName");
1011 throw new ArgumentNullException("hash");
1012 if (String.IsNullOrWhiteSpace(Token))
1013 throw new InvalidOperationException("Invalid Token");
1014 if (StorageUrl == null)
1015 throw new InvalidOperationException("Invalid Storage Url");
1016 Contract.EndContractBlock();
1019 //Don't use a timeout because putting the hashmap may be a long process
1020 var client = new RestClient(_baseClient) { Timeout = 0 };
1021 if (!String.IsNullOrWhiteSpace(account))
1022 client.BaseAddress = GetAccountUrl(account);
1024 //The container and objectName are relative names. They are joined with the client's
1025 //BaseAddress to create the object's absolute address
1026 var builder = client.GetAddressBuilder(container, objectName);
1027 builder.Query = "format=json&hashmap";
1028 var uri = builder.Uri;
1031 //Send the tree hash as Json to the server
1032 client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
1033 var jsonHash = hash.ToJson();
1034 var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);
1035 if (Log.IsDebugEnabled)
1036 Log.DebugFormat("Hashes:\r\n{0}", jsonHash);
1037 return uploadTask.ContinueWith(t =>
1040 var empty = (IList<string>)new List<string>();
1043 //The server will respond either with 201-created if all blocks were already on the server
1044 if (client.StatusCode == HttpStatusCode.Created)
1046 //in which case we return an empty hash list
1049 //or with a 409-conflict and return the list of missing parts
1050 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
1053 var ex = t.Exception.InnerException;
1054 var we = ex as WebException;
1055 var response = we.Response as HttpWebResponse;
1056 if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
1058 //In case of 409 the missing parts will be in the response content
1059 using (var stream = response.GetResponseStream())
1060 using(var reader=stream.GetLoggedReader(Log))
1062 //We used to have to cleanup the content before returning it because it contains
1063 //error content after the list of hashes
1065 //As of 30/1/2012, the result is a proper Json array so we don't need to read the content
1068 var serializer = new JsonSerializer();
1069 serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);
1070 var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));
1074 //Any other status code is unexpected and the exception should be rethrown
1075 Log.LogError(response);
1080 //Any other status code is unexpected but there was no exception. We can probably continue processing
1081 Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
1089 public async Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)
1091 if (String.IsNullOrWhiteSpace(Token))
1092 throw new InvalidOperationException("Invalid Token");
1093 if (StorageUrl == null)
1094 throw new InvalidOperationException("Invalid Storage Url");
1095 if (String.IsNullOrWhiteSpace(container))
1096 throw new ArgumentNullException("container");
1097 if (relativeUrl == null)
1098 throw new ArgumentNullException("relativeUrl");
1099 if (end.HasValue && end < 0)
1100 throw new ArgumentOutOfRangeException("end");
1102 throw new ArgumentOutOfRangeException("start");
1103 Contract.EndContractBlock();
1105 //Don't use a timeout because putting the hashmap may be a long process
1106 using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end})
1108 if (!String.IsNullOrWhiteSpace(account))
1109 client.BaseAddress = GetAccountUrl(account);
1111 var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
1112 var uri = builder.Uri;
1114 client.DownloadProgressChanged += (sender, args) =>
1115 Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
1116 uri.Segments.Last(), args.ProgressPercentage,
1118 args.TotalBytesToReceive);
1121 var result = await client.DownloadDataTaskAsync(uri, cancellationToken);
1127 public async Task PostBlock(string account, string container, byte[] block, int offset, int count,CancellationToken token)
1129 if (String.IsNullOrWhiteSpace(container))
1130 throw new ArgumentNullException("container");
1132 throw new ArgumentNullException("block");
1133 if (offset < 0 || offset >= block.Length)
1134 throw new ArgumentOutOfRangeException("offset");
1135 if (count < 0 || count > block.Length)
1136 throw new ArgumentOutOfRangeException("count");
1137 if (String.IsNullOrWhiteSpace(Token))
1138 throw new InvalidOperationException("Invalid Token");
1139 if (StorageUrl == null)
1140 throw new InvalidOperationException("Invalid Storage Url");
1141 Contract.EndContractBlock();
1147 //Don't use a timeout because putting the hashmap may be a long process
1148 using (var client = new RestClient(_baseClient) { Timeout = 0 })
1150 if (!String.IsNullOrWhiteSpace(account))
1151 client.BaseAddress = GetAccountUrl(account);
1153 token.Register(client.CancelAsync);
1155 var builder = client.GetAddressBuilder(container, "");
1156 //We are doing an update
1157 builder.Query = "update";
1158 var uri = builder.Uri;
1160 client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
1162 Log.InfoFormat("[BLOCK POST] START");
1164 client.UploadProgressChanged += (sender, args) =>
1165 Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
1166 args.ProgressPercentage, args.BytesSent,
1167 args.TotalBytesToSend);
1168 client.UploadFileCompleted += (sender, args) =>
1169 Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
1171 var buffer = new byte[count];
1172 Buffer.BlockCopy(block, offset, buffer, 0, count);
1174 await client.UploadDataTaskAsync(uri, "POST", buffer);
1175 Log.InfoFormat("[BLOCK POST] END");
1178 catch (TaskCanceledException )
1180 Log.Info("Aborting block");
1183 catch (Exception exc)
1185 Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);
1191 public async Task<TreeHash> GetHashMap(string account, string container, string objectName)
1193 if (String.IsNullOrWhiteSpace(container))
1194 throw new ArgumentNullException("container");
1195 if (String.IsNullOrWhiteSpace(objectName))
1196 throw new ArgumentNullException("objectName");
1197 if (String.IsNullOrWhiteSpace(Token))
1198 throw new InvalidOperationException("Invalid Token");
1199 if (StorageUrl == null)
1200 throw new InvalidOperationException("Invalid Storage Url");
1201 Contract.EndContractBlock();
1205 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
1206 //object to avoid concurrency errors.
1208 //Download operations take a long time therefore they have no timeout.
1209 //TODO: Do they really? this is a hashmap operation, not a download
1211 //Start downloading the object asynchronously
1212 using (var client = new RestClient(_baseClient) { Timeout = 0 })
1214 if (!String.IsNullOrWhiteSpace(account))
1215 client.BaseAddress = GetAccountUrl(account);
1217 //The container and objectName are relative names. They are joined with the client's
1218 //BaseAddress to create the object's absolute address
1219 var builder = client.GetAddressBuilder(container, objectName);
1220 builder.Query = "format=json&hashmap";
1221 var uri = builder.Uri;
1224 var json = await client.DownloadStringTaskAsync(uri);
1225 var treeHash = TreeHash.Parse(json);
1226 Log.InfoFormat("[GET HASH] END {0}", objectName);
1230 catch (Exception exc)
1232 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
1242 /// <param name="account"></param>
1243 /// <param name="container"></param>
1244 /// <param name="objectName"></param>
1245 /// <param name="fileName"></param>
1246 /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
1247 /// <remarks>>This method should have no timeout or a very long one</remarks>
1248 public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream")
1250 if (String.IsNullOrWhiteSpace(container))
1251 throw new ArgumentNullException("container", "The container property can't be empty");
1252 if (String.IsNullOrWhiteSpace(objectName))
1253 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1254 if (String.IsNullOrWhiteSpace(fileName))
1255 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
1257 if (!File.Exists(fileName) && !Directory.Exists(fileName))
1258 throw new FileNotFoundException("The file or directory does not exist",fileName);
1264 using (var client = new RestClient(_baseClient) { Timeout = 0 })
1266 if (!String.IsNullOrWhiteSpace(account))
1267 client.BaseAddress = GetAccountUrl(account);
1269 var builder = client.GetAddressBuilder(container, objectName);
1270 var uri = builder.Uri;
1272 string etag = hash ?? CalculateHash(fileName);
1274 client.Headers.Add("Content-Type", contentType);
1275 client.Headers.Add("ETag", etag);
1278 Log.InfoFormat("[PUT] START {0}", objectName);
1279 client.UploadProgressChanged += (sender, args) =>
1281 using (ThreadContext.Stacks["PUT"].Push("Progress"))
1283 Log.InfoFormat("{0} {1}% {2} of {3}", fileName,
1284 args.ProgressPercentage,
1285 args.BytesSent, args.TotalBytesToSend);
1289 client.UploadFileCompleted += (sender, args) =>
1291 using (ThreadContext.Stacks["PUT"].Push("Progress"))
1293 Log.InfoFormat("Completed {0}", fileName);
1296 if (contentType=="application/directory")
1297 await client.UploadDataTaskAsync(uri, "PUT", new byte[0]);
1299 await client.UploadFileTaskAsync(uri, "PUT", fileName);
1302 Log.InfoFormat("[PUT] END {0}", objectName);
1304 catch (Exception exc)
1306 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
1313 private static string CalculateHash(string fileName)
1315 Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
1316 Contract.EndContractBlock();
1319 using (var hasher = MD5.Create())
1320 using(var stream=File.OpenRead(fileName))
1322 var hashBuilder=new StringBuilder();
1323 foreach (byte b in hasher.ComputeHash(stream))
1324 hashBuilder.Append(b.ToString("x2").ToLower());
1325 hash = hashBuilder.ToString();
1330 public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
1332 if (String.IsNullOrWhiteSpace(sourceContainer))
1333 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1334 if (String.IsNullOrWhiteSpace(oldObjectName))
1335 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
1336 if (String.IsNullOrWhiteSpace(targetContainer))
1337 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
1338 if (String.IsNullOrWhiteSpace(newObjectName))
1339 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
1340 Contract.EndContractBlock();
1342 var targetUrl = targetContainer + "/" + newObjectName;
1343 var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
1345 using (var client = new RestClient(_baseClient))
1347 if (!String.IsNullOrWhiteSpace(account))
1348 client.BaseAddress = GetAccountUrl(account);
1350 client.Headers.Add("X-Move-From", sourceUrl);
1351 client.PutWithRetry(targetUrl, 3);
1353 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1354 if (!expectedCodes.Contains(client.StatusCode))
1355 throw CreateWebException("MoveObject", client.StatusCode);
1359 public void DeleteObject(string account, string sourceContainer, string objectName)
1361 if (String.IsNullOrWhiteSpace(sourceContainer))
1362 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1363 if (String.IsNullOrWhiteSpace(objectName))
1364 throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
1365 Contract.EndContractBlock();
1367 var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
1368 var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
1370 using (var client = new RestClient(_baseClient))
1372 if (!String.IsNullOrWhiteSpace(account))
1373 client.BaseAddress = GetAccountUrl(account);
1375 client.Headers.Add("X-Move-From", sourceUrl);
1376 client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
1377 Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);
1378 client.PutWithRetry(targetUrl, 3);
1380 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
1381 if (!expectedCodes.Contains(client.StatusCode))
1382 throw CreateWebException("DeleteObject", client.StatusCode);
1387 private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
1389 return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
1394 public IEnumerable<ObjectInfo> ListDirectories(ContainerInfo container)
1396 var directories=this.ListObjects(container.Account, container.Name, "/");
1400 public bool CanUpload(string account, ObjectInfo cloudFile)
1402 Contract.Requires(!String.IsNullOrWhiteSpace(account));
1403 Contract.Requires(cloudFile!=null);
1405 using (var client = new RestClient(_baseClient))
1407 if (!String.IsNullOrWhiteSpace(account))
1408 client.BaseAddress = GetAccountUrl(account);
1411 var parts = cloudFile.Name.Split('/');
1412 var folder = String.Join("/", parts,0,parts.Length-1);
1414 var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid());
1416 client.Parameters.Clear();
1419 client.PutWithRetry(fileUrl, 3, @"application/octet-stream");
1421 var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1422 var result=(expectedCodes.Contains(client.StatusCode));
1423 DeleteObject(account, cloudFile.Container, fileUrl);
1434 public class ShareAccountInfo
1436 public DateTime? last_modified { get; set; }
1437 public string name { get; set; }