X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/blobdiff_plain/07ffe53596b17db4876a471907f82317937faad9..cfb091032c1ea51d109dccf68322ce2ee63eaab5:/trunk/Pithos.Network/CloudFilesClient.cs diff --git a/trunk/Pithos.Network/CloudFilesClient.cs b/trunk/Pithos.Network/CloudFilesClient.cs index 4bc2a7c..cef59f5 100644 --- a/trunk/Pithos.Network/CloudFilesClient.cs +++ b/trunk/Pithos.Network/CloudFilesClient.cs @@ -1,1565 +1,1884 @@ -#region -/* ----------------------------------------------------------------------- - * - * - * Copyright 2011-2012 GRNET S.A. All rights reserved. - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * 1. Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * - * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS - * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - * The views and conclusions contained in the software and - * documentation are those of the authors and should not be - * interpreted as representing official policies, either expressed - * or implied, of GRNET S.A. - * - * ----------------------------------------------------------------------- - */ -#endregion - -// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos -// -// The class provides methods to upload/download files, delete files, manage containers - - -using System; -using System.Collections.Generic; -using System.Collections.Specialized; -using System.ComponentModel.Composition; -using System.Diagnostics; -using System.Diagnostics.Contracts; -using System.IO; -using System.Linq; -using System.Net; -using System.Reflection; -using System.Security.Cryptography; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Newtonsoft.Json; -using Pithos.Interfaces; -using log4net; - -namespace Pithos.Network -{ - [Export(typeof(ICloudClient))] - public class CloudFilesClient:ICloudClient - { - private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - - //CloudFilesClient uses *_baseClient* internally to communicate with the server - //RestClient provides a REST-friendly interface over the standard WebClient. - private RestClient _baseClient; - - - //During authentication the client provides a UserName - public string UserName { get; set; } - - //and and ApiKey to the server - public string ApiKey { get; set; } - - //And receives an authentication Token. This token must be provided in ALL other operations, - //in the X-Auth-Token header - private string _token; - private readonly string _emptyGuid = Guid.Empty.ToString(); - - - public string Token - { - get { return _token; } - set - { - _token = value; - _baseClient.Headers["X-Auth-Token"] = value; - } - } - - //The client also receives a StorageUrl after authentication. All subsequent operations must - //use this url - public Uri StorageUrl { get; set; } - - - public Uri RootAddressUri { get; set; } - - /* private WebProxy _proxy; - public WebProxy Proxy - { - get { return _proxy; } - set - { - _proxy = value; - if (_baseClient != null) - _baseClient.Proxy = value; - } - } -*/ - - /* private Uri _proxy; - public Uri Proxy - { - get { return _proxy; } - set - { - _proxy = value; - if (_baseClient != null) - _baseClient.Proxy = new WebProxy(value); - } - }*/ - - public double DownloadPercentLimit { get; set; } - public double UploadPercentLimit { get; set; } - - public string AuthenticationUrl { get; set; } - - - public string VersionPath - { - get { return UsePithos ? "v1" : "v1.0"; } - } - - public bool UsePithos { get; set; } - - - - public CloudFilesClient(string userName, string apiKey) - { - UserName = userName; - ApiKey = apiKey; - } - - public CloudFilesClient(AccountInfo accountInfo) - { - if (accountInfo==null) - throw new ArgumentNullException("accountInfo"); - Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); - Contract.Ensures(StorageUrl != null); - Contract.Ensures(_baseClient != null); - Contract.Ensures(RootAddressUri != null); - Contract.EndContractBlock(); - - _baseClient = new RestClient - { - BaseAddress = accountInfo.StorageUri.ToString(), - Timeout = 30000, - Retries = 3, - }; - StorageUrl = accountInfo.StorageUri; - Token = accountInfo.Token; - UserName = accountInfo.UserName; - - //Get the root address (StorageUrl without the account) - var storageUrl = StorageUrl.AbsoluteUri; - var usernameIndex = storageUrl.LastIndexOf(UserName); - var rootUrl = storageUrl.Substring(0, usernameIndex); - RootAddressUri = new Uri(rootUrl); - } - - - public AccountInfo Authenticate() - { - if (String.IsNullOrWhiteSpace(UserName)) - throw new InvalidOperationException("UserName is empty"); - if (String.IsNullOrWhiteSpace(ApiKey)) - throw new InvalidOperationException("ApiKey is empty"); - if (String.IsNullOrWhiteSpace(AuthenticationUrl)) - throw new InvalidOperationException("AuthenticationUrl is empty"); - Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); - Contract.Ensures(StorageUrl != null); - Contract.Ensures(_baseClient != null); - Contract.Ensures(RootAddressUri != null); - Contract.EndContractBlock(); - - - Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName); - - var groups = new List(); - - using (var authClient = new RestClient{BaseAddress=AuthenticationUrl}) - { - /* if (Proxy != null) - authClient.Proxy = Proxy;*/ - - Contract.Assume(authClient.Headers!=null); - - authClient.Headers.Add("X-Auth-User", UserName); - authClient.Headers.Add("X-Auth-Key", ApiKey); - //TODO: Remove after testing. Added to overcome server auth bug - //authClient.Headers.Add("X-Auth-Token", ApiKey); - - authClient.DownloadStringWithRetry(VersionPath, 3); - - authClient.AssertStatusOK("Authentication failed"); - - var storageUrl = authClient.GetHeaderValue("X-Storage-Url"); - if (String.IsNullOrWhiteSpace(storageUrl)) - throw new InvalidOperationException("Failed to obtain storage url"); - - _baseClient = new RestClient - { - BaseAddress = storageUrl, - Timeout = 10000, - Retries = 3, - //Proxy=Proxy - }; - - StorageUrl = new Uri(storageUrl); - - //Get the root address (StorageUrl without the account) - var usernameIndex=storageUrl.LastIndexOf(UserName); - var rootUrl = storageUrl.Substring(0, usernameIndex); - RootAddressUri = new Uri(rootUrl); - - var token = authClient.GetHeaderValue("X-Auth-Token"); - if (String.IsNullOrWhiteSpace(token)) - throw new InvalidOperationException("Failed to obtain token url"); - Token = token; - - /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable(); - groups = (from key in keys - where key.StartsWith("X-Account-Group-") - let name = key.Substring(16) - select new Group(name, authClient.ResponseHeaders[key])) - .ToList(); - -*/ - } - - Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName); - Debug.Assert(_baseClient!=null); - - return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups}; - - } - - - - public IList ListContainers(string account) - { - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Parameters.Clear(); - client.Parameters.Add("format", "json"); - var content = client.DownloadStringWithRetry("", 3); - client.AssertStatusOK("List Containers failed"); - - if (client.StatusCode == HttpStatusCode.NoContent) - return new List(); - var infos = JsonConvert.DeserializeObject>(content); - - foreach (var info in infos) - { - info.Account = account; - } - return infos; - } - - } - - private string GetAccountUrl(string account) - { - return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri; - } - - public IList ListSharingAccounts(DateTime? since=null) - { - using (ThreadContext.Stacks["Share"].Push("List Accounts")) - { - if (Log.IsDebugEnabled) Log.DebugFormat("START"); - - using (var client = new RestClient(_baseClient)) - { - client.Parameters.Clear(); - client.Parameters.Add("format", "json"); - client.IfModifiedSince = since; - - //Extract the username from the base address - client.BaseAddress = RootAddressUri.AbsoluteUri; - - var content = client.DownloadStringWithRetry(@"", 3); - - client.AssertStatusOK("ListSharingAccounts failed"); - - //If the result is empty, return an empty list, - var infos = String.IsNullOrWhiteSpace(content) - ? new List() - //Otherwise deserialize the account list into a list of ShareAccountInfos - : JsonConvert.DeserializeObject>(content); - - Log.DebugFormat("END"); - return infos; - } - } - } - - - /// - /// Request listing of all objects in a container modified since a specific time. - /// If the *since* value is missing, return all objects - /// - /// Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new - /// and should be polled anyway - /// - /// - /// - public IList ListSharedObjects(HashSet knownContainers,DateTime? since = null ) - { - - using (ThreadContext.Stacks["Share"].Push("List Objects")) - { - if (Log.IsDebugEnabled) Log.DebugFormat("START"); - //'since' is not used here because we need to have ListObjects return a NoChange result - //for all shared accounts,containers - - Func GetKey = c => String.Format("{0}\\{1}", c.Account, c.Name); - - var accounts = ListSharingAccounts(); - var containers = (from account in accounts - let conts = ListContainers(account.name) - from container in conts - select container).ToList(); - var items = from container in containers - let actualSince=knownContainers.Contains(GetKey(container))?since:null - select ListObjects(container.Account , container.Name, actualSince); - var objects=items.SelectMany(r=> r).ToList(); - - //For each object - //Check parents recursively up to (but not including) the container. - //If parents are missing, add them to the list - //Need function to calculate all parent URLs - objects = AddMissingParents(objects); - - //Store any new containers - foreach (var container in containers) - { - knownContainers.Add(GetKey(container)); - } - - - - if (Log.IsDebugEnabled) Log.DebugFormat("END"); - return objects; - } - } - - private List AddMissingParents(List objects) - { - //TODO: Remove short-circuit when we decide to use Missing Parents functionality - //return objects; - - var existingUris = objects.ToDictionary(o => o.Uri, o => o); - foreach (var objectInfo in objects) - { - //Can be null when retrieving objects to show in selective sync - if (objectInfo.Name == null) - continue; - - var parts = objectInfo.Name.Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries); - //If there is no parent, skip - if (parts.Length == 1) - continue; - var baseParts = new[] - { - objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container - }; - for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++) - { - var nameparts = parts.Range(0, partIdx).ToArray(); - var parentName= String.Join("/", nameparts); - - var parentParts = baseParts.Concat(nameparts); - var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts); - - var parentUri = new Uri(parentUrl, UriKind.Absolute); - - ObjectInfo existingInfo; - if (!existingUris.TryGetValue(parentUri,out existingInfo)) - { - var h = parentUrl.GetHashCode(); - var reverse = new string(parentUrl.Reverse().ToArray()); - var rh = reverse.GetHashCode(); - var b1 = BitConverter.GetBytes(h); - var b2 = BitConverter.GetBytes(rh); - var g = new Guid(0,0,0,b1.Concat(b2).ToArray()); - - - existingUris[parentUri] = new ObjectInfo - { - Account = objectInfo.Account, - Container = objectInfo.Container, - Content_Type = @"application/directory", - ETag = Signature.MD5_EMPTY, - X_Object_Hash = Signature.MERKLE_EMPTY, - Name=parentName, - StorageUri=objectInfo.StorageUri, - Bytes = 0, - UUID=g.ToString(), - }; - } - } - } - return existingUris.Values.ToList(); - } - - public void SetTags(ObjectInfo target,IDictionary tags) - { - if (String.IsNullOrWhiteSpace(Token)) - throw new InvalidOperationException("The Token is not set"); - if (StorageUrl == null) - throw new InvalidOperationException("The StorageUrl is not set"); - if (target == null) - throw new ArgumentNullException("target"); - Contract.EndContractBlock(); - - using (ThreadContext.Stacks["Share"].Push("Share Object")) - { - if (Log.IsDebugEnabled) Log.DebugFormat("START"); - - using (var client = new RestClient(_baseClient)) - { - - client.BaseAddress = GetAccountUrl(target.Account); - - client.Parameters.Clear(); - client.Parameters.Add("update", ""); - - foreach (var tag in tags) - { - var headerTag = String.Format("X-Object-Meta-{0}", tag.Key); - client.Headers.Add(headerTag, tag.Value); - } - - client.DownloadStringWithRetry(target.Container, 3); - - - client.AssertStatusOK("SetTags failed"); - //If the status is NOT ACCEPTED we have a problem - if (client.StatusCode != HttpStatusCode.Accepted) - { - Log.Error("Failed to set tags"); - throw new Exception("Failed to set tags"); - } - - if (Log.IsDebugEnabled) Log.DebugFormat("END"); - } - } - - - } - - public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write) - { - if (String.IsNullOrWhiteSpace(Token)) - throw new InvalidOperationException("The Token is not set"); - if (StorageUrl==null) - throw new InvalidOperationException("The StorageUrl is not set"); - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName"); - if (String.IsNullOrWhiteSpace(account)) - throw new ArgumentNullException("account"); - if (String.IsNullOrWhiteSpace(shareTo)) - throw new ArgumentNullException("shareTo"); - Contract.EndContractBlock(); - - using (ThreadContext.Stacks["Share"].Push("Share Object")) - { - if (Log.IsDebugEnabled) Log.DebugFormat("START"); - - using (var client = new RestClient(_baseClient)) - { - - client.BaseAddress = GetAccountUrl(account); - - client.Parameters.Clear(); - client.Parameters.Add("format", "json"); - - string permission = ""; - if (write) - permission = String.Format("write={0}", shareTo); - else if (read) - permission = String.Format("read={0}", shareTo); - client.Headers.Add("X-Object-Sharing", permission); - - var content = client.DownloadStringWithRetry(container, 3); - - client.AssertStatusOK("ShareObject failed"); - - //If the result is empty, return an empty list, - var infos = String.IsNullOrWhiteSpace(content) - ? new List() - //Otherwise deserialize the object list into a list of ObjectInfos - : JsonConvert.DeserializeObject>(content); - - if (Log.IsDebugEnabled) Log.DebugFormat("END"); - } - } - - - } - - public AccountInfo GetAccountPolicies(AccountInfo accountInfo) - { - if (accountInfo==null) - throw new ArgumentNullException("accountInfo"); - Contract.EndContractBlock(); - - using (ThreadContext.Stacks["Account"].Push("GetPolicies")) - { - if (Log.IsDebugEnabled) Log.DebugFormat("START"); - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(accountInfo.UserName)) - client.BaseAddress = GetAccountUrl(accountInfo.UserName); - - client.Parameters.Clear(); - client.Parameters.Add("format", "json"); - client.Head(String.Empty, 3); - - var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"]; - var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"]; - - long quota, bytes; - if (long.TryParse(quotaValue, out quota)) - accountInfo.Quota = quota; - if (long.TryParse(bytesValue, out bytes)) - accountInfo.BytesUsed = bytes; - - return accountInfo; - - } - - } - } - - public void UpdateMetadata(ObjectInfo objectInfo) - { - if (objectInfo == null) - throw new ArgumentNullException("objectInfo"); - Contract.EndContractBlock(); - - using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata")) - { - if (Log.IsDebugEnabled) Log.DebugFormat("START"); - - - using(var client=new RestClient(_baseClient)) - { - - client.BaseAddress = GetAccountUrl(objectInfo.Account); - - client.Parameters.Clear(); - - - //Set Tags - foreach (var tag in objectInfo.Tags) - { - var headerTag = String.Format("X-Object-Meta-{0}", tag.Key); - client.Headers.Add(headerTag, tag.Value); - } - - //Set Permissions - - var permissions=objectInfo.GetPermissionString(); - client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions); - - client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition); - client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding); - client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest); - var isPublic = objectInfo.IsPublic.ToString().ToLower(); - client.Headers.Add("X-Object-Public", isPublic); - - - /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name); - uriBuilder.Query = "update="; - var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/ - var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name); - client.PostWithRetry(address,"application/xml"); - - //client.UploadValues(uri,new NameValueCollection()); - - - client.AssertStatusOK("UpdateMetadata failed"); - //If the status is NOT ACCEPTED or OK we have a problem - if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK)) - { - Log.Error("Failed to update metadata"); - throw new Exception("Failed to update metadata"); - } - - if (Log.IsDebugEnabled) Log.DebugFormat("END"); - } - } - - } - - public void UpdateMetadata(ContainerInfo containerInfo) - { - if (containerInfo == null) - throw new ArgumentNullException("containerInfo"); - Contract.EndContractBlock(); - - using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata")) - { - if (Log.IsDebugEnabled) Log.DebugFormat("START"); - - - using(var client=new RestClient(_baseClient)) - { - - client.BaseAddress = GetAccountUrl(containerInfo.Account); - - client.Parameters.Clear(); - - - //Set Tags - foreach (var tag in containerInfo.Tags) - { - var headerTag = String.Format("X-Container-Meta-{0}", tag.Key); - client.Headers.Add(headerTag, tag.Value); - } - - - //Set Policies - foreach (var policy in containerInfo.Policies) - { - var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key); - client.Headers.Add(headerPolicy, policy.Value); - } - - - var uriBuilder = client.GetAddressBuilder(containerInfo.Name,""); - var uri = uriBuilder.Uri; - - client.UploadValues(uri,new NameValueCollection()); - - - client.AssertStatusOK("UpdateMetadata failed"); - //If the status is NOT ACCEPTED or OK we have a problem - if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK)) - { - Log.Error("Failed to update metadata"); - throw new Exception("Failed to update metadata"); - } - - if (Log.IsDebugEnabled) Log.DebugFormat("END"); - } - } - - } - - - public IList ListObjects(string account, string container, DateTime? since = null) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); - Contract.EndContractBlock(); - - using (ThreadContext.Stacks["Objects"].Push("List")) - { - if (Log.IsDebugEnabled) Log.DebugFormat("START"); - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Parameters.Clear(); - client.Parameters.Add("format", "json"); - client.IfModifiedSince = since; - var content = client.DownloadStringWithRetry(container, 3); - - client.AssertStatusOK("ListObjects failed"); - - if (client.StatusCode==HttpStatusCode.NotModified) - return new[]{new NoModificationInfo(account,container)}; - //If the result is empty, return an empty list, - var infos = String.IsNullOrWhiteSpace(content) - ? new List() - //Otherwise deserialize the object list into a list of ObjectInfos - : JsonConvert.DeserializeObject>(content); - - foreach (var info in infos) - { - info.Container = container; - info.Account = account; - info.StorageUri = this.StorageUrl; - } - if (Log.IsDebugEnabled) Log.DebugFormat("END"); - return infos; - } - } - } - - public IList ListObjects(string account, string container, string folder, DateTime? since = null) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); -/* - if (String.IsNullOrWhiteSpace(folder)) - throw new ArgumentNullException("folder"); -*/ - Contract.EndContractBlock(); - - using (ThreadContext.Stacks["Objects"].Push("List")) - { - if (Log.IsDebugEnabled) Log.DebugFormat("START"); - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Parameters.Clear(); - client.Parameters.Add("format", "json"); - client.Parameters.Add("path", folder); - client.IfModifiedSince = since; - var content = client.DownloadStringWithRetry(container, 3); - client.AssertStatusOK("ListObjects failed"); - - if (client.StatusCode==HttpStatusCode.NotModified) - return new[]{new NoModificationInfo(account,container,folder)}; - - var infos = JsonConvert.DeserializeObject>(content); - foreach (var info in infos) - { - info.Account = account; - if (info.Container == null) - info.Container = container; - info.StorageUri = this.StorageUrl; - } - if (Log.IsDebugEnabled) Log.DebugFormat("END"); - return infos; - } - } - } - - - public bool ContainerExists(string account, string container) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); - Contract.EndContractBlock(); - - using (ThreadContext.Stacks["Containters"].Push("Exists")) - { - if (Log.IsDebugEnabled) Log.DebugFormat("START"); - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Parameters.Clear(); - client.Head(container, 3); - - bool result; - switch (client.StatusCode) - { - case HttpStatusCode.OK: - case HttpStatusCode.NoContent: - result=true; - break; - case HttpStatusCode.NotFound: - result=false; - break; - default: - throw CreateWebException("ContainerExists", client.StatusCode); - } - if (Log.IsDebugEnabled) Log.DebugFormat("END"); - - return result; - } - - } - } - - public bool ObjectExists(string account, string container, string objectName) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName", "The objectName property can't be empty"); - Contract.EndContractBlock(); - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Parameters.Clear(); - client.Head(container + "/" + objectName, 3); - - switch (client.StatusCode) - { - case HttpStatusCode.OK: - case HttpStatusCode.NoContent: - return true; - case HttpStatusCode.NotFound: - return false; - default: - throw CreateWebException("ObjectExists", client.StatusCode); - } - } - - } - - public ObjectInfo GetObjectInfo(string account, string container, string objectName) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName", "The objectName property can't be empty"); - Contract.EndContractBlock(); - - using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo")) - { - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - try - { - client.Parameters.Clear(); - - client.Head(container + "/" + objectName, 3); - - if (client.TimedOut) - return ObjectInfo.Empty; - - switch (client.StatusCode) - { - case HttpStatusCode.OK: - case HttpStatusCode.NoContent: - var keys = client.ResponseHeaders.AllKeys.AsQueryable(); - var tags = client.GetMeta("X-Object-Meta-"); - var extensions = (from key in keys - where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-") - select new {Name = key, Value = client.ResponseHeaders[key]}) - .ToDictionary(t => t.Name, t => t.Value); - - var permissions=client.GetHeaderValue("X-Object-Sharing", true); - - - var info = new ObjectInfo - { - Account = account, - Container = container, - Name = objectName, - ETag = client.GetHeaderValue("ETag"), - UUID=client.GetHeaderValue("X-Object-UUID"), - X_Object_Hash = client.GetHeaderValue("X-Object-Hash"), - Content_Type = client.GetHeaderValue("Content-Type"), - Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)), - Tags = tags, - Last_Modified = client.LastModified, - Extensions = extensions, - ContentEncoding=client.GetHeaderValue("Content-Encoding",true), - ContendDisposition = client.GetHeaderValue("Content-Disposition",true), - Manifest=client.GetHeaderValue("X-Object-Manifest",true), - PublicUrl=client.GetHeaderValue("X-Object-Public",true), - StorageUri=this.StorageUrl, - }; - info.SetPermissions(permissions); - return info; - case HttpStatusCode.NotFound: - return ObjectInfo.Empty; - default: - throw new WebException( - String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", - objectName, client.StatusCode)); - } - - } - catch (RetryException) - { - Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName); - return ObjectInfo.Empty; - } - catch (WebException e) - { - Log.Error( - String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", - objectName, client.StatusCode), e); - throw; - } - } - } - - } - - public void CreateFolder(string account, string container, string folder) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(folder)) - throw new ArgumentNullException("folder", "The folder property can't be empty"); - Contract.EndContractBlock(); - - var folderUrl=String.Format("{0}/{1}",container,folder); - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Parameters.Clear(); - client.Headers.Add("Content-Type", @"application/directory"); - client.Headers.Add("Content-Length", "0"); - client.PutWithRetry(folderUrl, 3); - - if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted) - throw CreateWebException("CreateFolder", client.StatusCode); - } - } - - - - public ContainerInfo GetContainerInfo(string account, string container) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); - Contract.EndContractBlock(); - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Head(container); - switch (client.StatusCode) - { - case HttpStatusCode.OK: - case HttpStatusCode.NoContent: - var tags = client.GetMeta("X-Container-Meta-"); - var policies = client.GetMeta("X-Container-Policy-"); - - var containerInfo = new ContainerInfo - { - Account=account, - Name = container, - StorageUrl=this.StorageUrl.ToString(), - Count = - long.Parse(client.GetHeaderValue("X-Container-Object-Count")), - Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")), - BlockHash = client.GetHeaderValue("X-Container-Block-Hash"), - BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")), - Last_Modified=client.LastModified, - Tags=tags, - Policies=policies - }; - - - return containerInfo; - case HttpStatusCode.NotFound: - return ContainerInfo.Empty; - default: - throw CreateWebException("GetContainerInfo", client.StatusCode); - } - } - } - - public void CreateContainer(string account, string container) - { - if (String.IsNullOrWhiteSpace(account)) - throw new ArgumentNullException("account"); - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); - Contract.EndContractBlock(); - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.PutWithRetry(container, 3); - var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK}; - if (!expectedCodes.Contains(client.StatusCode)) - throw CreateWebException("CreateContainer", client.StatusCode); - } - } - - public void DeleteContainer(string account, string container) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); - Contract.EndContractBlock(); - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.DeleteWithRetry(container, 3); - var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent}; - if (!expectedCodes.Contains(client.StatusCode)) - throw CreateWebException("DeleteContainer", client.StatusCode); - } - - } - - /// - /// - /// - /// - /// - /// - /// - /// - /// This method should have no timeout or a very long one - //Asynchronously download the object specified by *objectName* in a specific *container* to - // a local file - public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName", "The objectName property can't be empty"); - Contract.EndContractBlock(); - - try - { - //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient - //object to avoid concurrency errors. - // - //Download operations take a long time therefore they have no timeout. - using(var client = new RestClient(_baseClient) { Timeout = 0 }) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - //The container and objectName are relative names. They are joined with the client's - //BaseAddress to create the object's absolute address - var builder = client.GetAddressBuilder(container, objectName); - var uri = builder.Uri; - - //Download progress is reported to the Trace log - Log.InfoFormat("[GET] START {0}", objectName); - /*client.DownloadProgressChanged += (sender, args) => - Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}", - fileName, args.ProgressPercentage, - args.BytesReceived, - args.TotalBytesToReceive);*/ - var progress = new Progress(args => - { - Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}", - fileName, args.ProgressPercentage, - args.BytesReceived, - args.TotalBytesToReceive); - DownloadProgressChanged(this, args); - }); - - //Start downloading the object asynchronously - await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false); - - //Once the download completes - //Delete the local client object - } - //And report failure or completion - } - catch (Exception exc) - { - Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc); - throw; - } - - Log.InfoFormat("[GET] END {0}", objectName); - - - } - - public Task> PutHashMap(string account, string container, string objectName, TreeHash hash) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName"); - if (hash==null) - throw new ArgumentNullException("hash"); - if (String.IsNullOrWhiteSpace(Token)) - throw new InvalidOperationException("Invalid Token"); - if (StorageUrl == null) - throw new InvalidOperationException("Invalid Storage Url"); - Contract.EndContractBlock(); - - - //Don't use a timeout because putting the hashmap may be a long process - var client = new RestClient(_baseClient) { Timeout = 0 }; - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - //The container and objectName are relative names. They are joined with the client's - //BaseAddress to create the object's absolute address - var builder = client.GetAddressBuilder(container, objectName); - builder.Query = "format=json&hashmap"; - var uri = builder.Uri; - - - //Send the tree hash as Json to the server - client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream"; - var jsonHash = hash.ToJson(); - - client.Headers.Add("ETag",hash.MD5); - var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash); - if (Log.IsDebugEnabled) - Log.DebugFormat("Hashes:\r\n{0}", jsonHash); - return uploadTask.ContinueWith(t => - { - - var empty = (IList)new List(); - - - //The server will respond either with 201-created if all blocks were already on the server - if (client.StatusCode == HttpStatusCode.Created) - { - //in which case we return an empty hash list - return empty; - } - //or with a 409-conflict and return the list of missing parts - //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception - if (t.IsFaulted) - { - var ex = t.Exception.InnerException; - var we = ex as WebException; - var response = we.Response as HttpWebResponse; - if (response!=null && response.StatusCode==HttpStatusCode.Conflict) - { - //In case of 409 the missing parts will be in the response content - using (var stream = response.GetResponseStream()) - using(var reader=stream.GetLoggedReader(Log)) - { - //We used to have to cleanup the content before returning it because it contains - //error content after the list of hashes - // - //As of 30/1/2012, the result is a proper Json array so we don't need to read the content - //line by line - - var serializer = new JsonSerializer(); - serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member); - var hashes = (List)serializer.Deserialize(reader, typeof(List)); - return hashes; - } - } - //Any other status code is unexpected and the exception should be rethrown - Log.LogError(response); - throw ex; - - } - - //Any other status code is unexpected but there was no exception. We can probably continue processing - Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription); - - return empty; - }); - - } - - - public async Task GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken) - { - if (String.IsNullOrWhiteSpace(Token)) - throw new InvalidOperationException("Invalid Token"); - if (StorageUrl == null) - throw new InvalidOperationException("Invalid Storage Url"); - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); - if (relativeUrl == null) - throw new ArgumentNullException("relativeUrl"); - if (end.HasValue && end < 0) - throw new ArgumentOutOfRangeException("end"); - if (start < 0) - throw new ArgumentOutOfRangeException("start"); - Contract.EndContractBlock(); - - //Don't use a timeout because putting the hashmap may be a long process - using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end}) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - var builder = client.GetAddressBuilder(container, relativeUrl.ToString()); - var uri = builder.Uri; - -/* client.DownloadProgressChanged += (sender, args) => - { - Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}", - uri.Segments.Last(), args.ProgressPercentage, - args.BytesReceived, - args.TotalBytesToReceive); - DownloadProgressChanged(sender, args); - };*/ - var progress = new Progress(args => - { - Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}", - uri.Segments.Last(), args.ProgressPercentage, - args.BytesReceived, - args.TotalBytesToReceive); - DownloadProgressChanged(this, args); - }); - - - var result = await client.DownloadDataTaskAsync(uri, cancellationToken,progress).ConfigureAwait(false); - return result; - } - } - - public event UploadProgressChangedEventHandler UploadProgressChanged; - public event DownloadProgressChangedEventHandler DownloadProgressChanged; - - public async Task PostBlock(string account, string container, byte[] block, int offset, int count,CancellationToken token) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); - if (block == null) - throw new ArgumentNullException("block"); - if (offset < 0 || offset >= block.Length) - throw new ArgumentOutOfRangeException("offset"); - if (count < 0 || count > block.Length) - throw new ArgumentOutOfRangeException("count"); - if (String.IsNullOrWhiteSpace(Token)) - throw new InvalidOperationException("Invalid Token"); - if (StorageUrl == null) - throw new InvalidOperationException("Invalid Storage Url"); - Contract.EndContractBlock(); - - - try - { - - //Don't use a timeout because putting the hashmap may be a long process - using (var client = new RestClient(_baseClient) { Timeout = 0 }) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - token.Register(client.CancelAsync); - - var builder = client.GetAddressBuilder(container, ""); - //We are doing an update - builder.Query = "update"; - var uri = builder.Uri; - - client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream"; - - Log.InfoFormat("[BLOCK POST] START"); - -/* - client.UploadProgressChanged += (sender, args) => - { - Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}", - args.ProgressPercentage, args.BytesSent, - args.TotalBytesToSend); - UploadProgressChanged(sender, args); - }; -*/ - client.UploadFileCompleted += (sender, args) => - Log.InfoFormat("[BLOCK POST PROGRESS] Completed "); - - var progress=new Progress(args=> - { - Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}", - args.ProgressPercentage, args.BytesSent, - args.TotalBytesToSend); - UploadProgressChanged(this, args); - }); - var buffer = new byte[count]; - Buffer.BlockCopy(block, offset, buffer, 0, count); - //Send the block - await client.UploadDataTaskAsync(uri, "POST", buffer,token,progress).ConfigureAwait(false); - Log.InfoFormat("[BLOCK POST] END"); - } - } - catch (TaskCanceledException ) - { - Log.Info("Aborting block"); - throw; - } - catch (Exception exc) - { - Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc); - throw; - } - } - - - public async Task GetHashMap(string account, string container, string objectName) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName"); - if (String.IsNullOrWhiteSpace(Token)) - throw new InvalidOperationException("Invalid Token"); - if (StorageUrl == null) - throw new InvalidOperationException("Invalid Storage Url"); - Contract.EndContractBlock(); - - try - { - //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient - //object to avoid concurrency errors. - // - //Download operations take a long time therefore they have no timeout. - //TODO: Do they really? this is a hashmap operation, not a download - - //Start downloading the object asynchronously - using (var client = new RestClient(_baseClient) { Timeout = 0 }) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - //The container and objectName are relative names. They are joined with the client's - //BaseAddress to create the object's absolute address - var builder = client.GetAddressBuilder(container, objectName); - builder.Query = "format=json&hashmap"; - var uri = builder.Uri; - - - var json = await client.DownloadStringTaskAsync(uri).ConfigureAwait(false); - var treeHash = TreeHash.Parse(json); - Log.InfoFormat("[GET HASH] END {0}", objectName); - return treeHash; - } - } - catch (Exception exc) - { - Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc); - throw; - } - - } - - - /// - /// - /// - /// - /// - /// - /// - /// Optional hash value for the file. If no hash is provided, the method calculates a new hash - /// >This method should have no timeout or a very long one - public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream") - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName", "The objectName property can't be empty"); - if (String.IsNullOrWhiteSpace(fileName)) - throw new ArgumentNullException("fileName", "The fileName property can't be empty"); -/* - if (!File.Exists(fileName) && !Directory.Exists(fileName)) - throw new FileNotFoundException("The file or directory does not exist",fileName); -*/ - - try - { - - using (var client = new RestClient(_baseClient) { Timeout = 0 }) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - var builder = client.GetAddressBuilder(container, objectName); - var uri = builder.Uri; - - string etag = hash ?? CalculateHash(fileName); - - client.Headers.Add("Content-Type", contentType); - client.Headers.Add("ETag", etag); - - - Log.InfoFormat("[PUT] START {0}", objectName); - client.UploadProgressChanged += (sender, args) => - { - using (ThreadContext.Stacks["PUT"].Push("Progress")) - { - Log.InfoFormat("{0} {1}% {2} of {3}", fileName, - args.ProgressPercentage, - args.BytesSent, args.TotalBytesToSend); - } - }; - - client.UploadFileCompleted += (sender, args) => - { - using (ThreadContext.Stacks["PUT"].Push("Progress")) - { - Log.InfoFormat("Completed {0}", fileName); - } - }; - if (contentType=="application/directory") - await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false); - else - await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false); - } - - Log.InfoFormat("[PUT] END {0}", objectName); - } - catch (Exception exc) - { - Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc); - throw; - } - - } - - - private static string CalculateHash(string fileName) - { - Contract.Requires(!String.IsNullOrWhiteSpace(fileName)); - Contract.EndContractBlock(); - - string hash; - using (var hasher = MD5.Create()) - using(var stream=File.OpenRead(fileName)) - { - var hashBuilder=new StringBuilder(); - foreach (byte b in hasher.ComputeHash(stream)) - hashBuilder.Append(b.ToString("x2").ToLower()); - hash = hashBuilder.ToString(); - } - return hash; - } - - public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName) - { - if (String.IsNullOrWhiteSpace(sourceContainer)) - throw new ArgumentNullException("sourceContainer", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(oldObjectName)) - throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty"); - if (String.IsNullOrWhiteSpace(targetContainer)) - throw new ArgumentNullException("targetContainer", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(newObjectName)) - throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty"); - Contract.EndContractBlock(); - - var targetUrl = targetContainer + "/" + newObjectName; - var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName); - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Headers.Add("X-Move-From", sourceUrl); - client.PutWithRetry(targetUrl, 3); - - var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; - if (!expectedCodes.Contains(client.StatusCode)) - throw CreateWebException("MoveObject", client.StatusCode); - } - } - - public void DeleteObject(string account, string sourceContainer, string objectName, bool isDirectory) - { - if (String.IsNullOrWhiteSpace(sourceContainer)) - throw new ArgumentNullException("sourceContainer", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty"); - Contract.EndContractBlock(); - - var targetUrl = FolderConstants.TrashContainer + "/" + objectName; -/* - if (isDirectory) - targetUrl = targetUrl + "?delimiter=/"; -*/ - - var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName); - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Headers.Add("X-Move-From", sourceUrl); - client.AllowedStatusCodes.Add(HttpStatusCode.NotFound); - Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl); - client.PutWithRetry(targetUrl, 3); - - var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound}; - if (!expectedCodes.Contains(client.StatusCode)) - throw CreateWebException("DeleteObject", client.StatusCode); - } - } - - - private static WebException CreateWebException(string operation, HttpStatusCode statusCode) - { - return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode)); - } - - -/* - public IEnumerable ListDirectories(ContainerInfo container) - { - var directories=this.ListObjects(container.Account, container.Name, "/"); - } -*/ - - public bool CanUpload(string account, ObjectInfo cloudFile) - { - Contract.Requires(!String.IsNullOrWhiteSpace(account)); - Contract.Requires(cloudFile!=null); - - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - - var parts = cloudFile.Name.Split('/'); - var folder = String.Join("/", parts,0,parts.Length-1); - - var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid()); - - client.Parameters.Clear(); - try - { - client.PutWithRetry(fileUrl, 3, @"application/octet-stream"); - - var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; - var result=(expectedCodes.Contains(client.StatusCode)); - DeleteObject(account, cloudFile.Container, fileUrl, cloudFile.IsDirectory); - return result; - } - catch - { - return false; - } - } - } - } - - public class ShareAccountInfo - { - public DateTime? last_modified { get; set; } - public string name { get; set; } - } -} +#region +/* ----------------------------------------------------------------------- + * + * + * Copyright 2011-2012 GRNET S.A. All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and + * documentation are those of the authors and should not be + * interpreted as representing official policies, either expressed + * or implied, of GRNET S.A. + * + * ----------------------------------------------------------------------- + */ +#endregion + +// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos +// +// The class provides methods to upload/download files, delete files, manage containers + + +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.ComponentModel.Composition; +using System.Diagnostics; +using System.Diagnostics.Contracts; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Pithos.Interfaces; +using Pithos.Network; +using log4net; + +namespace Pithos.Network +{ + + [Export(typeof(ICloudClient))] + public class CloudFilesClient:ICloudClient,IDisposable + { + private const string TOKEN_HEADER = "X-Auth-Token"; + private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + //CloudFilesClient uses *_baseClient* internally to communicate with the server + //RestClient provides a REST-friendly interface over the standard WebClient. + private RestClient _baseClient; + + private HttpClient _baseHttpClient; + private HttpClient _baseHttpClientNoTimeout; + + + //During authentication the client provides a UserName + public string UserName { get; set; } + + //and and ApiKey to the server + public string ApiKey { get; set; } + + //And receives an authentication Token. This token must be provided in ALL other operations, + //in the X-Auth-Token header + private string _token; + private readonly string _emptyGuid = Guid.Empty.ToString(); + private readonly Uri _emptyUri = new Uri("",UriKind.Relative); + + private HttpClientHandler _httpClientHandler = new HttpClientHandler + { + AllowAutoRedirect = true, + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + UseCookies = true, + }; + + + public string Token + { + get { return _token; } + set + { + _token = value; + _baseClient.Headers[TOKEN_HEADER] = value; + } + } + + //The client also receives a StorageUrl after authentication. All subsequent operations must + //use this url + public Uri StorageUrl { get; set; } + + + public Uri RootAddressUri { get; set; } + + + public double DownloadPercentLimit { get; set; } + public double UploadPercentLimit { get; set; } + + public string AuthenticationUrl { get; set; } + + + public string VersionPath + { + get { return UsePithos ? "v1" : "v1.0"; } + } + + public bool UsePithos { get; set; } + + + + public CloudFilesClient(string userName, string apiKey) + { + UserName = userName; + ApiKey = apiKey; + } + + public CloudFilesClient(AccountInfo accountInfo) + { + if (accountInfo==null) + throw new ArgumentNullException("accountInfo"); + Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); + Contract.Ensures(StorageUrl != null); + Contract.Ensures(_baseClient != null); + Contract.Ensures(RootAddressUri != null); + Contract.EndContractBlock(); + + _baseClient = new RestClient + { + BaseAddress = accountInfo.StorageUri.ToString(), + Timeout = 30000, + Retries = 3, + }; + StorageUrl = accountInfo.StorageUri; + Token = accountInfo.Token; + UserName = accountInfo.UserName; + + //Get the root address (StorageUrl without the account) + var storageUrl = StorageUrl.AbsoluteUri; + var usernameIndex = storageUrl.LastIndexOf(UserName); + var rootUrl = storageUrl.Substring(0, usernameIndex); + RootAddressUri = new Uri(rootUrl); + + var httpClientHandler = new HttpClientHandler + { + AllowAutoRedirect = true, + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + UseCookies = true, + }; + + + _baseHttpClient = new HttpClient(httpClientHandler) + { + BaseAddress = StorageUrl, + Timeout = TimeSpan.FromSeconds(30) + }; + _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, Token); + + _baseHttpClientNoTimeout = new HttpClient(httpClientHandler) + { + BaseAddress = StorageUrl, + Timeout = TimeSpan.FromMilliseconds(-1) + }; + _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, Token); + + + } + + + private static void AssertStatusOK(HttpResponseMessage response, string message) + { + var statusCode = response.StatusCode; + if (statusCode >= HttpStatusCode.BadRequest) + throw new WebException(String.Format("{0} with code {1} - {2}", message, statusCode, response.ReasonPhrase)); + } + + public async Task Authenticate() + { + if (String.IsNullOrWhiteSpace(UserName)) + throw new InvalidOperationException("UserName is empty"); + if (String.IsNullOrWhiteSpace(ApiKey)) + throw new InvalidOperationException("ApiKey is empty"); + if (String.IsNullOrWhiteSpace(AuthenticationUrl)) + throw new InvalidOperationException("AuthenticationUrl is empty"); + Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); + Contract.Ensures(StorageUrl != null); + Contract.Ensures(_baseClient != null); + Contract.Ensures(RootAddressUri != null); + Contract.EndContractBlock(); + + + Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName); + + var groups = new List(); + + using (var authClient = new HttpClient(_httpClientHandler,false){ BaseAddress = new Uri(AuthenticationUrl),Timeout=TimeSpan.FromSeconds(30) }) + { + + authClient.DefaultRequestHeaders.Add("X-Auth-User", UserName); + authClient.DefaultRequestHeaders.Add("X-Auth-Key", ApiKey); + + string storageUrl; + string token; + + using (var response = await authClient.GetAsyncWithRetries(new Uri(VersionPath, UriKind.Relative),3).ConfigureAwait(false)) // .DownloadStringWithRetryRelative(new Uri(VersionPath, UriKind.Relative), 3); + { + AssertStatusOK(response,"Authentication failed"); + + storageUrl = response.Headers.GetFirstValue("X-Storage-Url"); + if (String.IsNullOrWhiteSpace(storageUrl)) + throw new InvalidOperationException("Failed to obtain storage url"); + + token = response.Headers.GetFirstValue(TOKEN_HEADER); + if (String.IsNullOrWhiteSpace(token)) + throw new InvalidOperationException("Failed to obtain token url"); + + } + + + _baseClient = new RestClient + { + BaseAddress = storageUrl, + Timeout = 30000, + Retries = 3, + }; + + StorageUrl = new Uri(storageUrl); + Token = token; + + + + + //Get the root address (StorageUrl without the account) + var usernameIndex=storageUrl.LastIndexOf(UserName); + var rootUrl = storageUrl.Substring(0, usernameIndex); + RootAddressUri = new Uri(rootUrl); + + + _baseHttpClient = new HttpClient(_httpClientHandler,false) + { + BaseAddress = StorageUrl, + Timeout = TimeSpan.FromSeconds(30) + }; + _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, token); + + _baseHttpClientNoTimeout = new HttpClient(_httpClientHandler,false) + { + BaseAddress = StorageUrl, + Timeout = TimeSpan.FromMilliseconds(-1) + }; + _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, token); + + /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable(); + groups = (from key in keys + where key.StartsWith("X-Account-Group-") + let name = key.Substring(16) + select new Group(name, authClient.ResponseHeaders[key])) + .ToList(); + +*/ + } + + Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName); + Debug.Assert(_baseClient!=null); + + return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups}; + + } + + private static void TraceStart(string method, Uri actualAddress) + { + Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress); + } + + private async Task GetStringAsync(Uri targetUri, string errorMessage,DateTimeOffset? since=null) + { + TraceStart("GET",targetUri); + var request = new HttpRequestMessage(HttpMethod.Get, targetUri); + if (since.HasValue) + { + request.Headers.IfModifiedSince = since.Value; + } + using (var response = await _baseHttpClient.SendAsyncWithRetries(request,3).ConfigureAwait(false)) + { + AssertStatusOK(response, errorMessage); + + if (response.StatusCode == HttpStatusCode.NoContent) + return String.Empty; + + var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + return content; + } + } + + public async Task> ListContainers(string account) + { + + var targetUrl = GetTargetUrl(account); + var targetUri = new Uri(String.Format("{0}?format=json", targetUrl)); + var result = await GetStringAsync(targetUri, "List Containers failed").ConfigureAwait(false); + if (String.IsNullOrWhiteSpace(result)) + return new List(); + var infos = JsonConvert.DeserializeObject>(result); + foreach (var info in infos) + { + info.Account = account; + } + return infos; + } + + + private string GetAccountUrl(string account) + { + return RootAddressUri.Combine(account).AbsoluteUri; + } + + public IList ListSharingAccounts(DateTime? since=null) + { + using (ThreadContext.Stacks["Share"].Push("List Accounts")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + var targetUri = new Uri(String.Format("{0}?format=json", RootAddressUri), UriKind.Absolute); + var content=TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListSharingAccounts failed", since).ConfigureAwait(false)).Result; + + //If the result is empty, return an empty list, + var infos = String.IsNullOrWhiteSpace(content) + ? new List() + //Otherwise deserialize the account list into a list of ShareAccountInfos + : JsonConvert.DeserializeObject>(content); + + Log.DebugFormat("END"); + return infos; + } + } + + + /// + /// Request listing of all objects in a container modified since a specific time. + /// If the *since* value is missing, return all objects + /// + /// Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new + /// and should be polled anyway + /// + /// + /// + public IList ListSharedObjects(HashSet knownContainers, DateTimeOffset? since) + { + + using (ThreadContext.Stacks["Share"].Push("List Objects")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + //'since' is not used here because we need to have ListObjects return a NoChange result + //for all shared accounts,containers + + Func getKey = c => String.Format("{0}\\{1}", c.Account, c.Name); + + var containers = (from account in ListSharingAccounts() + let conts = ListContainers(account.name).Result + from container in conts + select container).ToList(); + var items = from container in containers + let actualSince=knownContainers.Contains(getKey(container))?since:null + select ListObjects(container.Account , container.Name, actualSince); + var objects=items.SelectMany(r=> r).ToList(); + + //For each object + //Check parents recursively up to (but not including) the container. + //If parents are missing, add them to the list + //Need function to calculate all parent URLs + objects = AddMissingParents(objects); + + //Store any new containers + foreach (var container in containers) + { + knownContainers.Add(getKey(container)); + } + + + + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + return objects; + } + } + + private List AddMissingParents(List objects) + { + //TODO: Remove short-circuit when we decide to use Missing Parents functionality + //return objects; + + var existingUris = objects.ToDictionary(o => o.Uri, o => o); + foreach (var objectInfo in objects) + { + //Can be null when retrieving objects to show in selective sync + if (objectInfo.Name == null) + continue; + + //No need to unescape here, the parts will be used to create new ObjectInfos + var parts = objectInfo.Name.ToString().Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries); + //If there is no parent, skip + if (parts.Length == 1) + continue; + var baseParts = new[] + { + objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container.ToString() + }; + for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++) + { + var nameparts = parts.Range(0, partIdx).ToArray(); + var parentName= String.Join("/", nameparts); + + var parentParts = baseParts.Concat(nameparts); + var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts); + + var parentUri = new Uri(parentUrl, UriKind.Absolute); + + ObjectInfo existingInfo; + if (!existingUris.TryGetValue(parentUri,out existingInfo)) + { + var h = parentUrl.GetHashCode(); + var reverse = new string(parentUrl.Reverse().ToArray()); + var rh = reverse.GetHashCode(); + var b1 = BitConverter.GetBytes(h); + var b2 = BitConverter.GetBytes(rh); + var g = new Guid(0,0,0,b1.Concat(b2).ToArray()); + + + existingUris[parentUri] = new ObjectInfo + { + Account = objectInfo.Account, + Container = objectInfo.Container, + Content_Type = ObjectInfo.CONTENT_TYPE_DIRECTORY, + ETag = Signature.MERKLE_EMPTY, + X_Object_Hash = Signature.MERKLE_EMPTY, + Name=new Uri(parentName,UriKind.Relative), + StorageUri=objectInfo.StorageUri, + Bytes = 0, + UUID=g.ToString(), + }; + } + } + } + return existingUris.Values.ToList(); + } + + public void SetTags(ObjectInfo target,IDictionary tags) + { + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("The Token is not set"); + if (StorageUrl == null) + throw new InvalidOperationException("The StorageUrl is not set"); + if (target == null) + throw new ArgumentNullException("target"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Share"].Push("Share Object")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { + + client.BaseAddress = GetAccountUrl(target.Account); + + client.Parameters.Clear(); + client.Parameters.Add("update", ""); + + foreach (var tag in tags) + { + var headerTag = String.Format("X-Object-Meta-{0}", tag.Key); + client.Headers.Add(headerTag, tag.Value); + } + + client.DownloadStringWithRetryRelative(target.Container, 3); + + + client.AssertStatusOK("SetTags failed"); + //If the status is NOT ACCEPTED we have a problem + if (client.StatusCode != HttpStatusCode.Accepted) + { + Log.Error("Failed to set tags"); + throw new Exception("Failed to set tags"); + } + + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + } + } + + + } + + public void ShareObject(string account, Uri container, Uri objectName, string shareTo, bool read, bool write) + { + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("The Token is not set"); + if (StorageUrl==null) + throw new InvalidOperationException("The StorageUrl is not set"); + if (container==null) + throw new ArgumentNullException("container"); + if (container.IsAbsoluteUri) + throw new ArgumentException("container"); + if (objectName==null) + throw new ArgumentNullException("objectName"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("objectName"); + if (String.IsNullOrWhiteSpace(account)) + throw new ArgumentNullException("account"); + if (String.IsNullOrWhiteSpace(shareTo)) + throw new ArgumentNullException("shareTo"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Share"].Push("Share Object")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { + + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + + string permission = ""; + if (write) + permission = String.Format("write={0}", shareTo); + else if (read) + permission = String.Format("read={0}", shareTo); + client.Headers.Add("X-Object-Sharing", permission); + + var content = client.DownloadStringWithRetryRelative(container, 3); + + client.AssertStatusOK("ShareObject failed"); + + //If the result is empty, return an empty list, + var infos = String.IsNullOrWhiteSpace(content) + ? new List() + //Otherwise deserialize the object list into a list of ObjectInfos + : JsonConvert.DeserializeObject>(content); + + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + } + } + + + } + + public async Task GetAccountPolicies(AccountInfo accountInfo) + { + if (accountInfo==null) + throw new ArgumentNullException("accountInfo"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Account"].Push("GetPolicies")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + +/* + if (_baseClient == null) + { + _baseClient = new RestClient + { + BaseAddress = accountInfo.StorageUri.ToString(), + Timeout = 30000, + Retries = 3, + }; + } + +*/ + var containerUri = GetTargetUri(accountInfo.UserName); + var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute); + using(var response=await _baseHttpClient.HeadAsyncWithRetries(targetUri,3).ConfigureAwait(false)) + { + + var quotaValue=response.Headers.GetFirstValue("X-Account-Policy-Quota"); + var bytesValue = response.Headers.GetFirstValue("X-Account-Bytes-Used"); + long quota, bytes; + if (long.TryParse(quotaValue, out quota)) + accountInfo.Quota = quota; + if (long.TryParse(bytesValue, out bytes)) + accountInfo.BytesUsed = bytes; + + return accountInfo; + } + + + //using (var client = new RestClient(_baseClient)) + //{ + // if (!String.IsNullOrWhiteSpace(accountInfo.UserName)) + // client.BaseAddress = GetAccountUrl(accountInfo.UserName); + + // client.Parameters.Clear(); + // client.Parameters.Add("format", "json"); + // client.Head(_emptyUri, 3); + + // var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"]; + // var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"]; + + // long quota, bytes; + // if (long.TryParse(quotaValue, out quota)) + // accountInfo.Quota = quota; + // if (long.TryParse(bytesValue, out bytes)) + // accountInfo.BytesUsed = bytes; + + // return accountInfo; + + //} + + } + } + + public void UpdateMetadata(ObjectInfo objectInfo) + { + if (objectInfo == null) + throw new ArgumentNullException("objectInfo"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + + using(var client=new RestClient(_baseClient)) + { + + client.BaseAddress = GetAccountUrl(objectInfo.Account); + + client.Parameters.Clear(); + + + //Set Tags + foreach (var tag in objectInfo.Tags) + { + var headerTag = String.Format("X-Object-Meta-{0}", tag.Key); + client.Headers.Add(headerTag, tag.Value); + } + + //Set Permissions + + var permissions=objectInfo.GetPermissionString(); + client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions); + + client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition); + client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding); + client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest); + var isPublic = objectInfo.IsPublic.ToString().ToLower(); + client.Headers.Add("X-Object-Public", isPublic); + + + var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name); + client.PostWithRetry(new Uri(address,UriKind.Relative),"application/xml"); + + client.AssertStatusOK("UpdateMetadata failed"); + //If the status is NOT ACCEPTED or OK we have a problem + if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK)) + { + Log.Error("Failed to update metadata"); + throw new Exception("Failed to update metadata"); + } + + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + } + } + + } + + public void UpdateMetadata(ContainerInfo containerInfo) + { + if (containerInfo == null) + throw new ArgumentNullException("containerInfo"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + + using(var client=new RestClient(_baseClient)) + { + + client.BaseAddress = GetAccountUrl(containerInfo.Account); + + client.Parameters.Clear(); + + + //Set Tags + foreach (var tag in containerInfo.Tags) + { + var headerTag = String.Format("X-Container-Meta-{0}", tag.Key); + client.Headers.Add(headerTag, tag.Value); + } + + + //Set Policies + foreach (var policy in containerInfo.Policies) + { + var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key); + client.Headers.Add(headerPolicy, policy.Value); + } + + + var uriBuilder = client.GetAddressBuilder(containerInfo.Name,_emptyUri); + var uri = uriBuilder.Uri; + + client.UploadValues(uri,new NameValueCollection()); + + + client.AssertStatusOK("UpdateMetadata failed"); + //If the status is NOT ACCEPTED or OK we have a problem + if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK)) + { + Log.Error("Failed to update metadata"); + throw new Exception("Failed to update metadata"); + } + + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + } + } + + } + + + + + public IList ListObjects(string account, Uri container, DateTimeOffset? since = null) + { + if (container==null) + throw new ArgumentNullException("container"); + if (container.IsAbsoluteUri) + throw new ArgumentException("container"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Objects"].Push("List")) + { + + var containerUri = GetTargetUri(account).Combine(container); + var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute); + + var content =TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result; + + //304 will result in an empty string. Empty containers return an empty json array + if (String.IsNullOrWhiteSpace(content)) + return new[] {new NoModificationInfo(account, container)}; + + //If the result is empty, return an empty list, + var infos = String.IsNullOrWhiteSpace(content) + ? new List() + //Otherwise deserialize the object list into a list of ObjectInfos + : JsonConvert.DeserializeObject>(content); + + foreach (var info in infos) + { + info.Container = container; + info.Account = account; + info.StorageUri = StorageUrl; + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + return infos; + } + } + + public IList ListObjects(string account, Uri container, Uri folder, DateTimeOffset? since = null) + { + if (container==null) + throw new ArgumentNullException("container"); + if (container.IsAbsoluteUri) + throw new ArgumentException("container"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Objects"].Push("List")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + var containerUri = GetTargetUri(account).Combine(container); + var targetUri = new Uri(String.Format("{0}?format=json&path={1}", containerUri,folder), UriKind.Absolute); + var content = TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result; + + //304 will result in an empty string. Empty containers return an empty json array + if (String.IsNullOrWhiteSpace(content)) + return new[] { new NoModificationInfo(account, container) }; + + + var infos = JsonConvert.DeserializeObject>(content); + foreach (var info in infos) + { + info.Account = account; + if (info.Container == null) + info.Container = container; + info.StorageUri = StorageUrl; + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + return infos; +/* + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + client.Parameters.Add("path", folder.ToString()); + client.IfModifiedSince = since; + var content = client.DownloadStringWithRetryRelative(container, 3); + client.AssertStatusOK("ListObjects failed"); + + if (client.StatusCode==HttpStatusCode.NotModified) + return new[]{new NoModificationInfo(account,container,folder)}; + + var infos = JsonConvert.DeserializeObject>(content); + foreach (var info in infos) + { + info.Account = account; + if (info.Container == null) + info.Container = container; + info.StorageUri = StorageUrl; + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + return infos; + } +*/ + } + } + + + public bool ContainerExists(string account, Uri container) + { + if (container==null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException( "The container must be relative","container"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Containters"].Push("Exists")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + var targetUri = GetTargetUri(account).Combine(container); + + using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result) + { + + bool result; + switch (response.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + result = true; + break; + case HttpStatusCode.NotFound: + result = false; + break; + default: + throw CreateWebException("ContainerExists", response.StatusCode); + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + + return result; + } +/* + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Head(container, 3); + + bool result; + switch (client.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + result=true; + break; + case HttpStatusCode.NotFound: + result=false; + break; + default: + throw CreateWebException("ContainerExists", client.StatusCode); + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + + return result; + } +*/ + + } + } + + private Uri GetTargetUri(string account) + { + return new Uri(GetTargetUrl(account),UriKind.Absolute); + } + + private string GetTargetUrl(string account) + { + return String.IsNullOrWhiteSpace(account) + ? _baseHttpClient.BaseAddress.ToString() + : GetAccountUrl(account); + } + + public bool ObjectExists(string account, Uri container, Uri objectName) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (objectName == null) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); + Contract.EndContractBlock(); + + var targetUri=GetTargetUri(account).Combine(container).Combine(objectName); + + using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result) + { + switch (response.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + return true; + case HttpStatusCode.NotFound: + return false; + default: + throw CreateWebException("ObjectExists", response.StatusCode); + } + } + +/* + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Head(container.Combine(objectName), 3); + + switch (client.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + return true; + case HttpStatusCode.NotFound: + return false; + default: + throw CreateWebException("ObjectExists", client.StatusCode); + } + } +*/ + + } + + public async Task GetObjectInfo(string account, Uri container, Uri objectName) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative", "container"); + if (objectName == null) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative", "objectName"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo")) + { + + var targetUri = GetTargetUri(account).Combine(container).Combine(objectName); + try + { + using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true)) + { + switch (response.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + var tags = response.Headers.GetMeta("X-Object-Meta-"); + var extensions = (from header in response.Headers + where + header.Key.StartsWith("X-Object-") && + !header.Key.StartsWith("X-Object-Meta-") + select new {Name = header.Key, Value = header.Value.FirstOrDefault()}) + .ToDictionary(t => t.Name, t => t.Value); + + var permissions = response.Headers.GetFirstValue("X-Object-Sharing"); + + + var info = new ObjectInfo + { + Account = account, + Container = container, + Name = objectName, + ETag = response.Headers.ETag.NullSafe(e=>e.Tag), + UUID = response.Headers.GetFirstValue("X-Object-UUID"), + X_Object_Hash = response.Headers.GetFirstValue("X-Object-Hash"), + Content_Type = response.Headers.GetFirstValue("Content-Type"), + Bytes = Convert.ToInt64(response.Content.Headers.ContentLength), + Tags = tags, + Last_Modified = response.Content.Headers.LastModified, + Extensions = extensions, + ContentEncoding = + response.Content.Headers.ContentEncoding.FirstOrDefault(), + ContendDisposition = + response.Content.Headers.ContentDisposition.NullSafe(c=>c.ToString()), + Manifest = response.Headers.GetFirstValue("X-Object-Manifest"), + PublicUrl = response.Headers.GetFirstValue("X-Object-Public"), + StorageUri = StorageUrl, + }; + info.SetPermissions(permissions); + return info; + case HttpStatusCode.NotFound: + return ObjectInfo.Empty; + default: + throw new WebException( + String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", + objectName, response.StatusCode)); + } + } + } + catch (RetryException) + { + Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.", objectName); + return ObjectInfo.Empty; + } + catch (WebException e) + { + Log.Error( + String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status {1}", + objectName, e.Status), e); + throw; + } + } /* using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + try + { + client.Parameters.Clear(); + + client.Head(container.Combine(objectName), 3); + + if (client.TimedOut) + return ObjectInfo.Empty; + + switch (client.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + var keys = client.ResponseHeaders.AllKeys.AsQueryable(); + var tags = client.GetMeta("X-Object-Meta-"); + var extensions = (from key in keys + where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-") + select new {Name = key, Value = client.ResponseHeaders[key]}) + .ToDictionary(t => t.Name, t => t.Value); + + var permissions=client.GetHeaderValue("X-Object-Sharing", true); + + + var info = new ObjectInfo + { + Account = account, + Container = container, + Name = objectName, + ETag = client.GetHeaderValue("ETag"), + UUID=client.GetHeaderValue("X-Object-UUID"), + X_Object_Hash = client.GetHeaderValue("X-Object-Hash"), + Content_Type = client.GetHeaderValue("Content-Type"), + Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)), + Tags = tags, + Last_Modified = client.LastModified, + Extensions = extensions, + ContentEncoding=client.GetHeaderValue("Content-Encoding",true), + ContendDisposition = client.GetHeaderValue("Content-Disposition",true), + Manifest=client.GetHeaderValue("X-Object-Manifest",true), + PublicUrl=client.GetHeaderValue("X-Object-Public",true), + StorageUri=StorageUrl, + }; + info.SetPermissions(permissions); + return info; + case HttpStatusCode.NotFound: + return ObjectInfo.Empty; + default: + throw new WebException( + String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", + objectName, client.StatusCode)); + } + + } + catch (RetryException) + { + Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName); + return ObjectInfo.Empty; + } + catch (WebException e) + { + Log.Error( + String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", + objectName, client.StatusCode), e); + throw; + } + } */ + + + } + + + + public void CreateFolder(string account, Uri container, Uri folder) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (folder == null) + throw new ArgumentNullException("folder", "The objectName property can't be empty"); + if (folder.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","folder"); + Contract.EndContractBlock(); + + var folderUri=container.Combine(folder); + var targetUri = GetTargetUri(account).Combine(folderUri); + var message = new HttpRequestMessage(HttpMethod.Put, targetUri); + + message.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY); + message.Headers.Add("Content-Length", "0"); + using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result) + { + if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted) + throw CreateWebException("CreateFolder", response.StatusCode); + } +/* + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY); + client.Headers.Add("Content-Length", "0"); + client.PutWithRetry(folderUri, 3); + + if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted) + throw CreateWebException("CreateFolder", client.StatusCode); + } +*/ + } + + private Dictionary GetMeta(HttpResponseMessage response,string metaPrefix) + { + if (String.IsNullOrWhiteSpace(metaPrefix)) + throw new ArgumentNullException("metaPrefix"); + Contract.EndContractBlock(); + + var dict = (from header in response.Headers + where header.Key.StartsWith(metaPrefix) + select new { Name = header.Key, Value = String.Join(",", header.Value) }) + .ToDictionary(t => t.Name, t => t.Value); + + + return dict; + } + + + public ContainerInfo GetContainerInfo(string account, Uri container) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + Contract.EndContractBlock(); + + var targetUri = GetTargetUri(account).Combine(container); + using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result) + { + if (Log.IsDebugEnabled) + Log.DebugFormat("ContainerInfo data: {0}\n{1}",response,response.Content.ReadAsStringAsync().Result); + switch (response.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + var tags = GetMeta(response,"X-Container-Meta-"); + var policies = GetMeta(response,"X-Container-Policy-"); + + var containerInfo = new ContainerInfo + { + Account = account, + Name = container, + StorageUrl = StorageUrl.ToString(), + Count =long.Parse(response.Headers.GetFirstValue("X-Container-Object-Count")), + Bytes = long.Parse(response.Headers.GetFirstValue("X-Container-Bytes-Used")), + BlockHash = response.Headers.GetFirstValue("X-Container-Block-Hash"), + BlockSize = + int.Parse(response.Headers.GetFirstValue("X-Container-Block-Size")), + Last_Modified = response.Content.Headers.LastModified, + Tags = tags, + Policies = policies + }; + + + return containerInfo; + case HttpStatusCode.NotFound: + return ContainerInfo.Empty; + default: + throw CreateWebException("GetContainerInfo", response.StatusCode); + } + } + } + + public void CreateContainer(string account, Uri container) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + Contract.EndContractBlock(); + + var targetUri=GetTargetUri(account).Combine(container); + var message = new HttpRequestMessage(HttpMethod.Put, targetUri); + message.Headers.Add("Content-Length", "0"); + using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result) + { + var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK}; + if (!expectedCodes.Contains(response.StatusCode)) + throw CreateWebException("CreateContainer", response.StatusCode); + } +/* + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.PutWithRetry(container, 3); + var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK}; + if (!expectedCodes.Contains(client.StatusCode)) + throw CreateWebException("CreateContainer", client.StatusCode); + } +*/ + } + + public async Task WipeContainer(string account, Uri container) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative", "container"); + Contract.EndContractBlock(); + + await DeleteContainer(account, new Uri(String.Format("{0}?delimiter=/", container), UriKind.Relative)).ConfigureAwait(false); + } + + + public async Task DeleteContainer(string account, Uri container) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + Contract.EndContractBlock(); + + var targetUri = GetTargetUri(account).Combine(container); + var message = new HttpRequestMessage(HttpMethod.Delete, targetUri); + using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false)) + { + var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent }; + if (!expectedCodes.Contains(response.StatusCode)) + throw CreateWebException("DeleteContainer", response.StatusCode); + } + + } + + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// This method should have no timeout or a very long one + //Asynchronously download the object specified by *objectName* in a specific *container* to + // a local file + public async Task GetObject(string account, Uri container, Uri objectName, string fileName,CancellationToken cancellationToken) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (objectName == null) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); + Contract.EndContractBlock(); + + + try + { + //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient + //object to avoid concurrency errors. + // + //Download operations take a long time therefore they have no timeout. + using(var client = new RestClient(_baseClient) { Timeout = 0 }) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + //The container and objectName are relative names. They are joined with the client's + //BaseAddress to create the object's absolute address + var builder = client.GetAddressBuilder(container, objectName); + var uri = builder.Uri; + + //Download progress is reported to the Trace log + Log.InfoFormat("[GET] START {0}", objectName); + /*client.DownloadProgressChanged += (sender, args) => + Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}", + fileName, args.ProgressPercentage, + args.BytesReceived, + args.TotalBytesToReceive);*/ + var progress = new Progress(args => + { + Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}", + fileName, args.ProgressPercentage, + args.BytesReceived, + args.TotalBytesToReceive); + if (DownloadProgressChanged!=null) + DownloadProgressChanged(this, new DownloadArgs(args)); + }); + + //Start downloading the object asynchronously + await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false); + + //Once the download completes + //Delete the local client object + } + //And report failure or completion + } + catch (Exception exc) + { + Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc); + throw; + } + + Log.InfoFormat("[GET] END {0}", objectName); + + + } + + public async Task> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (objectName == null) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); + if (hash == null) + throw new ArgumentNullException("hash"); + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + Contract.EndContractBlock(); + + + + //The container and objectName are relative names. They are joined with the client's + //BaseAddress to create the object's absolute address + + var targetUri = GetTargetUri(account).Combine(container).Combine(objectName); + + + var uri = new Uri(String.Format("{0}?format=json&hashmap",targetUri),UriKind.Absolute); + + + //Send the tree hash as Json to the server + var jsonHash = hash.ToJson(); + if (Log.IsDebugEnabled) + Log.DebugFormat("Hashes:\r\n{0}", jsonHash); + + var message = new HttpRequestMessage(HttpMethod.Put, uri) + { + Content = new StringContent(jsonHash) + }; + message.Headers.Add("ETag",hash.TopHash.ToHashString()); + + //Don't use a timeout because putting the hashmap may be a long process + + using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false)) + { + var empty = (IList)new List(); + + switch (response.StatusCode) + { + case HttpStatusCode.Created: + //The server will respond either with 201-created if all blocks were already on the server + return empty; + case HttpStatusCode.Conflict: + //or with a 409-conflict and return the list of missing parts + using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + using(var reader=stream.GetLoggedReader(Log)) + { + var serializer = new JsonSerializer(); + serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member); + var hashes = (List)serializer.Deserialize(reader, typeof(List)); + return hashes; + } + default: + //All other cases are unexpected + //Ensure that failure codes raise exceptions + response.EnsureSuccessStatusCode(); + //And log any other codes as warngings, but continute processing + Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",response.StatusCode,response.ReasonPhrase); + return empty; + } + } + + } + + + public async Task GetBlock(string account, Uri container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken) + { + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (relativeUrl == null) + throw new ArgumentNullException("relativeUrl"); + if (end.HasValue && end < 0) + throw new ArgumentOutOfRangeException("end"); + if (start < 0) + throw new ArgumentOutOfRangeException("start"); + Contract.EndContractBlock(); + + + var targetUri = GetTargetUri(account).Combine(container).Combine(relativeUrl); + var message = new HttpRequestMessage(HttpMethod.Get, targetUri); + message.Headers.Range=new RangeHeaderValue(start,end); + + //Don't use a timeout because putting the hashmap may be a long process + + IProgress progress = new Progress(args => + { + Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}", + targetUri.Segments.Last(), args.ProgressPercentage, + args.BytesReceived, + args.TotalBytesToReceive); + + if (DownloadProgressChanged!=null) + DownloadProgressChanged(this, args); + }); + + + using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false,HttpCompletionOption.ResponseHeadersRead, + cancellationToken).ConfigureAwait(false)) + using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + using(var targetStream=new MemoryStream()) + { + + long totalSize = response.Content.Headers.ContentLength ?? 0; + long total = 0; + var buffer = new byte[65536]; + int read; + while ((read = await stream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) != 0) + { + total += read; + progress.Report(new DownloadArgs(total, totalSize)); + await targetStream.WriteAsync(buffer, 0, read).ConfigureAwait(false); + } + + var result = targetStream.ToArray(); + return result; + } + + } + + public event EventHandler UploadProgressChanged; + public event EventHandler DownloadProgressChanged; + + + public async Task PostBlock(string account, Uri container, byte[] block, int offset, int count,string blockHash,CancellationToken token) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (block == null) + throw new ArgumentNullException("block"); + if (offset < 0 || offset >= block.Length) + throw new ArgumentOutOfRangeException("offset"); + if (count < 0 || count > block.Length) + throw new ArgumentOutOfRangeException("count"); + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + Contract.EndContractBlock(); + + + try + { + var containerUri = GetTargetUri(account).Combine(container); + var targetUri = new Uri(String.Format("{0}?update", containerUri)); + + + //Don't use a timeout because putting the hashmap may be a long process + + + Log.InfoFormat("[BLOCK POST] START"); + + + var progress = new Progress(args => + { + Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}", + args.ProgressPercentage, + args.BytesSent, + args.TotalBytesToSend); + if (UploadProgressChanged != null) + UploadProgressChanged(this,args); + }); + + var message = new HttpRequestMessage(HttpMethod.Post, targetUri) + { + Content = new ByteArrayContentWithProgress(block, offset, count,progress) + }; + message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream"); + + //Send the block + using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3,false,HttpCompletionOption.ResponseContentRead,token).ConfigureAwait(false)) + { + Log.InfoFormat("[BLOCK POST PROGRESS] Completed "); + response.EnsureSuccessStatusCode(); + var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + var cleanHash = responseHash.TrimEnd(); + Debug.Assert(blockHash==cleanHash); + if (responseHash.Equals(cleanHash,StringComparison.OrdinalIgnoreCase)) + Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]",account,container,blockHash,responseHash); + } + Log.InfoFormat("[BLOCK POST] END"); + } + catch (TaskCanceledException ) + { + Log.Info("Aborting block"); + throw; + } + catch (Exception exc) + { + Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc); + throw; + } + } + + + public async Task GetHashMap(string account, Uri container, Uri objectName) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (objectName == null) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + Contract.EndContractBlock(); + + try + { + + var objectUri = GetTargetUri(account).Combine(container).Combine(objectName); + var targetUri = new Uri(String.Format("{0}?format=json&hashmap", objectUri)); + + //Start downloading the object asynchronously + var json = await GetStringAsync(targetUri, "").ConfigureAwait(false); + var treeHash = TreeHash.Parse(json); + Log.InfoFormat("[GET HASH] END {0}", objectName); + return treeHash; + + } + catch (Exception exc) + { + Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc); + throw; + } + + } + + + /// + /// + /// + /// + /// + /// + /// + /// Optional hash value for the file. If no hash is provided, the method calculates a new hash + /// + /// >This method should have no timeout or a very long one + public async Task PutObject(string account, Uri container, Uri objectName, string fileName, string hash = Signature.MERKLE_EMPTY, string contentType = "application/octet-stream") + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (objectName == null) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); + if (String.IsNullOrWhiteSpace(fileName)) + throw new ArgumentNullException("fileName", "The fileName property can't be empty"); + try + { + + using (var client = new RestClient(_baseClient) { Timeout = 0 }) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + var builder = client.GetAddressBuilder(container, objectName); + var uri = builder.Uri; + + string etag = hash ; + + client.Headers.Add("Content-Type", contentType); + if (contentType!=ObjectInfo.CONTENT_TYPE_DIRECTORY) + client.Headers.Add("ETag", etag); + + + Log.InfoFormat("[PUT] START {0}", objectName); + client.UploadProgressChanged += (sender, args) => + { + using (ThreadContext.Stacks["PUT"].Push("Progress")) + { + Log.InfoFormat("{0} {1}% {2} of {3}", fileName, + args.ProgressPercentage, + args.BytesSent, args.TotalBytesToSend); + } + }; + + client.UploadFileCompleted += (sender, args) => + { + using (ThreadContext.Stacks["PUT"].Push("Progress")) + { + Log.InfoFormat("Completed {0}", fileName); + } + }; + + if (contentType==ObjectInfo.CONTENT_TYPE_DIRECTORY) + await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false); + else + await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false); + } + + Log.InfoFormat("[PUT] END {0}", objectName); + } + catch (Exception exc) + { + Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc); + throw; + } + + } + + public void MoveObject(string account, Uri sourceContainer, Uri oldObjectName, Uri targetContainer, Uri newObjectName) + { + if (sourceContainer == null) + throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty"); + if (sourceContainer.IsAbsoluteUri) + throw new ArgumentException("The sourceContainer must be relative","sourceContainer"); + if (oldObjectName == null) + throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty"); + if (oldObjectName.IsAbsoluteUri) + throw new ArgumentException("The oldObjectName must be relative","oldObjectName"); + if (targetContainer == null) + throw new ArgumentNullException("targetContainer", "The targetContainer property can't be empty"); + if (targetContainer.IsAbsoluteUri) + throw new ArgumentException("The targetContainer must be relative","targetContainer"); + if (newObjectName == null) + throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty"); + if (newObjectName.IsAbsoluteUri) + throw new ArgumentException("The newObjectName must be relative","newObjectName"); + Contract.EndContractBlock(); + + var baseUri = GetTargetUri(account); + var targetUri = baseUri.Combine(targetContainer).Combine(newObjectName); + var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, oldObjectName),UriKind.Relative); + + var message = new HttpRequestMessage(HttpMethod.Put, targetUri); + message.Headers.Add("X-Move-From", sourceUri.ToString()); + using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result) + { + var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; + if (!expectedCodes.Contains(response.StatusCode)) + throw CreateWebException("MoveObject", response.StatusCode); + } + } + + public async Task DeleteObject(string account, Uri sourceContainer, Uri objectName, bool isDirectory) + { + if (sourceContainer == null) + throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty"); + if (sourceContainer.IsAbsoluteUri) + throw new ArgumentException("The sourceContainer must be relative","sourceContainer"); + if (objectName == null) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); + Contract.EndContractBlock(); + + + + var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, objectName),UriKind.Relative); + + + if (objectName.OriginalString.EndsWith(".ignore")) + using(var response = await _baseHttpClient.DeleteAsync(sourceUri)){} + else + { + var relativeUri = new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName), + UriKind.Relative); + +/* + var relativeUri = isDirectory + ? new Uri( + String.Format("{0}/{1}?delimiter=/", FolderConstants.TrashContainer, + objectName), UriKind.Relative) + : new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName), + UriKind.Relative); + +*/ + var targetUri = GetTargetUri(account).Combine(relativeUri); + + + var message = new HttpRequestMessage(HttpMethod.Put, targetUri); + message.Headers.Add("X-Move-From", sourceUri.ToString()); + + Log.InfoFormat("[TRASH] [{0}] to [{1}]", sourceUri, targetUri); + using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3)) + { + var expectedCodes = new[] + { + HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created, + HttpStatusCode.NotFound + }; + if (!expectedCodes.Contains(response.StatusCode)) + throw CreateWebException("DeleteObject", response.StatusCode); + } + } +/* + + + var targetUrl = FolderConstants.TrashContainer + "/" + objectName; +/* + if (isDirectory) + targetUrl = targetUrl + "?delimiter=/"; +#1# + + var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Headers.Add("X-Move-From", sourceUrl); + client.AllowedStatusCodes.Add(HttpStatusCode.NotFound); + Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl); + client.PutWithRetry(new Uri(targetUrl,UriKind.Relative), 3); + + var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound}; + if (!expectedCodes.Contains(client.StatusCode)) + throw CreateWebException("DeleteObject", client.StatusCode); + } +*/ + } + + + private static WebException CreateWebException(string operation, HttpStatusCode statusCode) + { + return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode)); + } + + + public async Task CanUpload(string account, ObjectInfo cloudFile) + { + Contract.Requires(!String.IsNullOrWhiteSpace(account)); + Contract.Requires(cloudFile!=null); + + var parts = cloudFile.Name.ToString().Split('/'); + var folder = String.Join("/", parts,0,parts.Length-1); + + var fileName = String.Format("{0}/{1}.pithos.ignore", folder, Guid.NewGuid()); + var fileUri=fileName.ToEscapedUri(); + + try + { + var relativeUri = cloudFile.Container.Combine(fileUri); + var targetUri = GetTargetUri(account).Combine(relativeUri); + var message = new HttpRequestMessage(HttpMethod.Put, targetUri); + message.Content.Headers.ContentType =new MediaTypeHeaderValue("application/octet-stream"); + var response=await _baseHttpClient.SendAsyncWithRetries(message, 3); + var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; + var result=(expectedCodes.Contains(response.StatusCode)); + await DeleteObject(account, cloudFile.Container, fileUri, cloudFile.IsDirectory); + return result; + } + catch + { + return false; + } + + } + + ~CloudFilesClient() + { + Dispose(false); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + if (_httpClientHandler!=null) + _httpClientHandler.Dispose(); + if (_baseClient!=null) + _baseClient.Dispose(); + if(_baseHttpClient!=null) + _baseHttpClient.Dispose(); + if (_baseHttpClientNoTimeout!=null) + _baseHttpClientNoTimeout.Dispose(); + } + _httpClientHandler = null; + _baseClient = null; + _baseHttpClient = null; + _baseHttpClientNoTimeout = null; + } + } + + public class ShareAccountInfo + { + public DateTime? last_modified { get; set; } + public string name { get; set; } + } +}