X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/blobdiff_plain/0eea575a3cc68873de42e46ed2963bde3d046425..97edb52fa8c0dbe6b4856c3972b13436907b4b25:/trunk/Pithos.Network/CloudFilesClient.cs diff --git a/trunk/Pithos.Network/CloudFilesClient.cs b/trunk/Pithos.Network/CloudFilesClient.cs index b24937f..c4496c9 100644 --- a/trunk/Pithos.Network/CloudFilesClient.cs +++ b/trunk/Pithos.Network/CloudFilesClient.cs @@ -1,48 +1,136 @@ -using System; +#region +/* ----------------------------------------------------------------------- + * + * + * Copyright 2011-2012 GRNET S.A. All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and + * documentation are those of the authors and should not be + * interpreted as representing official policies, either expressed + * or implied, of GRNET S.A. + * + * ----------------------------------------------------------------------- + */ +#endregion + +// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos +// +// The class provides methods to upload/download files, delete files, manage containers + + +using System; using System.Collections.Generic; +using System.Collections.Specialized; using System.ComponentModel.Composition; using System.Diagnostics; using System.Diagnostics.Contracts; using System.IO; using System.Linq; using System.Net; +using System.Reflection; using System.Security.Cryptography; using System.Text; +using System.Threading; using System.Threading.Tasks; -using Hammock; -using Hammock.Caching; -using Hammock.Retries; -using Hammock.Serialization; -using Hammock.Tasks; -using Hammock.Web; using Newtonsoft.Json; using Pithos.Interfaces; +using log4net; namespace Pithos.Network { [Export(typeof(ICloudClient))] public class CloudFilesClient:ICloudClient { - string _rackSpaceAuthUrl = "https://auth.api.rackspacecloud.com"; - private string _pithosAuthUrl = "https://pithos.grnet.gr"; + private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - private RestClient _client; - private readonly TimeSpan _shortTimeout = TimeSpan.FromSeconds(10); - private readonly int _retries = 5; - private RetryPolicy _retryPolicy; - public string ApiKey { get; set; } + //CloudFilesClient uses *_baseClient* internally to communicate with the server + //RestClient provides a REST-friendly interface over the standard WebClient. + private RestClient _baseClient; + + + //During authentication the client provides a UserName public string UserName { get; set; } + + //and and ApiKey to the server + public string ApiKey { get; set; } + + //And receives an authentication Token. This token must be provided in ALL other operations, + //in the X-Auth-Token header + private string _token; + public string Token + { + get { return _token; } + set + { + _token = value; + _baseClient.Headers["X-Auth-Token"] = value; + } + } + + //The client also receives a StorageUrl after authentication. All subsequent operations must + //use this url public Uri StorageUrl { get; set; } - public string Token { get; set; } - public Uri Proxy { get; set; } - public double DownloadPercentLimit { get; set; } - public double UploadPercentLimit { get; set; } - - public string AuthUrl + + protected Uri RootAddressUri { get; set; } + + /* private WebProxy _proxy; + public WebProxy Proxy { - get { return UsePithos ? _pithosAuthUrl : _rackSpaceAuthUrl; } + get { return _proxy; } + set + { + _proxy = value; + if (_baseClient != null) + _baseClient.Proxy = value; + } } +*/ + + /* private Uri _proxy; + public Uri Proxy + { + get { return _proxy; } + set + { + _proxy = value; + if (_baseClient != null) + _baseClient.Proxy = new WebProxy(value); + } + }*/ + + public double DownloadPercentLimit { get; set; } + public double UploadPercentLimit { get; set; } + + public string AuthenticationUrl { get; set; } + public string VersionPath { @@ -51,357 +139,1106 @@ namespace Pithos.Network public bool UsePithos { get; set; } - public void Authenticate(string userName,string apiKey) - { - Trace.TraceInformation("[AUTHENTICATE] Start for {0}", userName); - if (String.IsNullOrWhiteSpace(userName)) - throw new ArgumentNullException("userName", "The userName property can't be empty"); - if (String.IsNullOrWhiteSpace(apiKey)) - throw new ArgumentNullException("apiKey", "The apiKey property can't be empty"); + + public CloudFilesClient(string userName, string apiKey) + { UserName = userName; ApiKey = apiKey; - - var proxy = Proxy != null ? Proxy.ToString() : null; - if (UsePithos) - { - Token = "0000"; - string storageUrl = String.Format("{0}/{1}/{2}", AuthUrl, VersionPath, UserName); - StorageUrl = new Uri(storageUrl); - } - else + } + + public CloudFilesClient(AccountInfo accountInfo) + { + if (accountInfo==null) + throw new ArgumentNullException("accountInfo"); + Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); + Contract.Ensures(StorageUrl != null); + Contract.Ensures(_baseClient != null); + Contract.Ensures(RootAddressUri != null); + Contract.EndContractBlock(); + + _baseClient = new RestClient { + BaseAddress = accountInfo.StorageUri.ToString(), + Timeout = 10000, + Retries = 3, + }; + StorageUrl = accountInfo.StorageUri; + Token = accountInfo.Token; + UserName = accountInfo.UserName; + + //Get the root address (StorageUrl without the account) + var storageUrl = StorageUrl.AbsoluteUri; + var usernameIndex = storageUrl.LastIndexOf(UserName); + var rootUrl = storageUrl.Substring(0, usernameIndex); + RootAddressUri = new Uri(rootUrl); + } + + + public AccountInfo Authenticate() + { + if (String.IsNullOrWhiteSpace(UserName)) + throw new InvalidOperationException("UserName is empty"); + if (String.IsNullOrWhiteSpace(ApiKey)) + throw new InvalidOperationException("ApiKey is empty"); + if (String.IsNullOrWhiteSpace(AuthenticationUrl)) + throw new InvalidOperationException("AuthenticationUrl is empty"); + Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); + Contract.Ensures(StorageUrl != null); + Contract.Ensures(_baseClient != null); + Contract.Ensures(RootAddressUri != null); + Contract.EndContractBlock(); + + + Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName); + + var groups = new List(); - string authUrl = String.Format("{0}/{1}", AuthUrl, VersionPath); - var authClient = new RestClient {Path = authUrl, Proxy = proxy}; + using (var authClient = new RestClient{BaseAddress=AuthenticationUrl}) + { + /* if (Proxy != null) + authClient.Proxy = Proxy;*/ - authClient.AddHeader("X-Auth-User", UserName); - authClient.AddHeader("X-Auth-Key", ApiKey); + Contract.Assume(authClient.Headers!=null); - var response = authClient.Request(); + authClient.Headers.Add("X-Auth-User", UserName); + authClient.Headers.Add("X-Auth-Key", ApiKey); - ThrowIfNotStatusOK(response, "Authentication failed"); + authClient.DownloadStringWithRetry(VersionPath, 3); - var keys = response.Headers.AllKeys.AsQueryable(); + authClient.AssertStatusOK("Authentication failed"); - string storageUrl = GetHeaderValue("X-Storage-Url", response, keys); + var storageUrl = authClient.GetHeaderValue("X-Storage-Url"); if (String.IsNullOrWhiteSpace(storageUrl)) throw new InvalidOperationException("Failed to obtain storage url"); - StorageUrl = new Uri(storageUrl); + + _baseClient = new RestClient + { + BaseAddress = storageUrl, + Timeout = 10000, + Retries = 3, + //Proxy=Proxy + }; - var token = GetHeaderValue("X-Auth-Token", response, keys); + StorageUrl = new Uri(storageUrl); + + //Get the root address (StorageUrl without the account) + var usernameIndex=storageUrl.LastIndexOf(UserName); + var rootUrl = storageUrl.Substring(0, usernameIndex); + RootAddressUri = new Uri(rootUrl); + + var token = authClient.GetHeaderValue("X-Auth-Token"); if (String.IsNullOrWhiteSpace(token)) throw new InvalidOperationException("Failed to obtain token url"); Token = token; + + /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable(); + groups = (from key in keys + where key.StartsWith("X-Account-Group-") + let name = key.Substring(16) + select new Group(name, authClient.ResponseHeaders[key])) + .ToList(); + +*/ } - _retryPolicy = new RetryPolicy { RetryCount = _retries }; - _retryPolicy.RetryConditions.Add(new TimeoutRetryCondition()); + Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName); + Debug.Assert(_baseClient!=null); - _client = new RestClient { Authority = StorageUrl.AbsoluteUri, Path = UserName, Proxy = proxy }; - _client.FileProgress += OnFileProgress; - - _client.AddHeader("X-Auth-Token", Token); - /*if (UsePithos) + return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups}; + + } + + + + public IList ListContainers(string account) + { + using (var client = new RestClient(_baseClient)) { - _client.AddHeader("X-Auth-User", UserName); - _client.AddHeader("X-Auth-Key",ApiKey); - }*/ + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + var content = client.DownloadStringWithRetry("", 3); + client.AssertStatusOK("List Containers failed"); + + if (client.StatusCode == HttpStatusCode.NoContent) + return new List(); + var infos = JsonConvert.DeserializeObject>(content); + + foreach (var info in infos) + { + info.Account = account; + } + return infos; + } - Trace.TraceInformation("[AUTHENTICATE] End for {0}", userName); } - private void OnFileProgress(object sender, FileProgressEventArgs e) + private string GetAccountUrl(string account) { - Trace.TraceInformation("[PROGRESS] {0} {1:p} {2} of {3}",e.FileName,(double)e.BytesWritten/e.TotalBytes, e.BytesWritten,e.TotalBytes); + return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri; } - public IList ListContainers() - { - //Workaround for Hammock quirk: Hammock always - //appends a / unless a Path is specified. - - //Create a request with a complete path - var request = new RestRequest { Path = StorageUrl.ToString(), RetryPolicy = _retryPolicy,Timeout = _shortTimeout }; - request.AddParameter("format","json"); - //Create a client clone - var client = new RestClient{Proxy=Proxy.ToString()}; - foreach (var header in _client.GetAllHeaders()) + public IList ListSharingAccounts(DateTime? since=null) + { + using (ThreadContext.Stacks["Share"].Push("List Accounts")) { - client.AddHeader(header.Name,header.Value); - } + if (Log.IsDebugEnabled) Log.DebugFormat("START"); - var response = client.Request(request); + using (var client = new RestClient(_baseClient)) + { + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + client.IfModifiedSince = since; - if (response.StatusCode == HttpStatusCode.NoContent) - return new List(); + //Extract the username from the base address + client.BaseAddress = RootAddressUri.AbsoluteUri; - ThrowIfNotStatusOK(response, "List Containers failed"); + var content = client.DownloadStringWithRetry(@"", 3); + + client.AssertStatusOK("ListSharingAccounts failed"); + + //If the result is empty, return an empty list, + var infos = String.IsNullOrWhiteSpace(content) + ? new List() + //Otherwise deserialize the account list into a list of ShareAccountInfos + : JsonConvert.DeserializeObject>(content); + + Log.DebugFormat("END"); + return infos; + } + } + } + + //Request listing of all objects in a container modified since a specific time. + //If the *since* value is missing, return all objects + public IList ListSharedObjects(DateTime? since = null) + { + + using (ThreadContext.Stacks["Share"].Push("List Objects")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + //'since' is not used here because we need to have ListObjects return a NoChange result + //for all shared accounts,containers + var accounts = ListSharingAccounts(); + var items = from account in accounts + let containers = ListContainers(account.name) + from container in containers + select ListObjects(account.name, container.Name,since); + var objects=items.SelectMany(r=> r).ToList(); +/* + var objects = new List(); + foreach (var containerObjects in items) + { + objects.AddRange(containerObjects); + } +*/ + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + return objects; + } + } + + public void SetTags(ObjectInfo target,IDictionary tags) + { + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("The Token is not set"); + if (StorageUrl == null) + throw new InvalidOperationException("The StorageUrl is not set"); + if (target == null) + throw new ArgumentNullException("target"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Share"].Push("Share Object")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { + + client.BaseAddress = GetAccountUrl(target.Account); + + client.Parameters.Clear(); + client.Parameters.Add("update", ""); + + foreach (var tag in tags) + { + var headerTag = String.Format("X-Object-Meta-{0}", tag.Key); + client.Headers.Add(headerTag, tag.Value); + } + + client.DownloadStringWithRetry(target.Container, 3); + + + client.AssertStatusOK("SetTags failed"); + //If the status is NOT ACCEPTED we have a problem + if (client.StatusCode != HttpStatusCode.Accepted) + { + Log.Error("Failed to set tags"); + throw new Exception("Failed to set tags"); + } + + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + } + } - var infos=JsonConvert.DeserializeObject>(response.Content); - - return infos; } - public IList ListObjects(string container) + public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write) { + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("The Token is not set"); + if (StorageUrl==null) + throw new InvalidOperationException("The StorageUrl is not set"); if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); + throw new ArgumentNullException("container"); + if (String.IsNullOrWhiteSpace(objectName)) + throw new ArgumentNullException("objectName"); + if (String.IsNullOrWhiteSpace(account)) + throw new ArgumentNullException("account"); + if (String.IsNullOrWhiteSpace(shareTo)) + throw new ArgumentNullException("shareTo"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Share"].Push("Share Object")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { - Trace.TraceInformation("[START] ListObjects"); + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + + string permission = ""; + if (write) + permission = String.Format("write={0}", shareTo); + else if (read) + permission = String.Format("read={0}", shareTo); + client.Headers.Add("X-Object-Sharing", permission); + + var content = client.DownloadStringWithRetry(container, 3); + + client.AssertStatusOK("ShareObject failed"); + + //If the result is empty, return an empty list, + var infos = String.IsNullOrWhiteSpace(content) + ? new List() + //Otherwise deserialize the object list into a list of ObjectInfos + : JsonConvert.DeserializeObject>(content); + + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + } + } - var request = new RestRequest { Path = container, RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) }; - request.AddParameter("format", "json"); - var response = _client.Request(request); - - var infos = InfosFromContent(response); - Trace.TraceInformation("[END] ListObjects"); - return infos; } + public AccountInfo GetAccountPolicies(AccountInfo accountInfo) + { + if (accountInfo==null) + throw new ArgumentNullException("accountInfo"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Account"].Push("GetPolicies")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(accountInfo.UserName)) + client.BaseAddress = GetAccountUrl(accountInfo.UserName); + + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + client.Head(String.Empty, 3); + + var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"]; + var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"]; + + long quota, bytes; + if (long.TryParse(quotaValue, out quota)) + accountInfo.Quota = quota; + if (long.TryParse(bytesValue, out bytes)) + accountInfo.BytesUsed = bytes; + + return accountInfo; + } - public IList ListObjects(string container,string folder) + } + } + + public void UpdateMetadata(ObjectInfo objectInfo) { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); + if (objectInfo == null) + throw new ArgumentNullException("objectInfo"); + Contract.EndContractBlock(); - Trace.TraceInformation("[START] ListObjects"); + using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); - var request = new RestRequest { Path = container,RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) }; - request.AddParameter("format", "json"); - request.AddParameter("path", folder); - var response = _client.Request(request); - - var infos = InfosFromContent(response); - Trace.TraceInformation("[END] ListObjects"); - return infos; + using(var client=new RestClient(_baseClient)) + { + + client.BaseAddress = GetAccountUrl(objectInfo.Account); + + client.Parameters.Clear(); + + + //Set Tags + foreach (var tag in objectInfo.Tags) + { + var headerTag = String.Format("X-Object-Meta-{0}", tag.Key); + client.Headers.Add(headerTag, tag.Value); + } + + //Set Permissions + + var permissions=objectInfo.GetPermissionString(); + client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions); + + client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition); + client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding); + client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest); + var isPublic = objectInfo.IsPublic.ToString().ToLower(); + client.Headers.Add("X-Object-Public", isPublic); + + + /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name); + uriBuilder.Query = "update="; + var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/ + var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name); + client.PostWithRetry(address,"application/xml"); + + //client.UploadValues(uri,new NameValueCollection()); + + + client.AssertStatusOK("UpdateMetadata failed"); + //If the status is NOT ACCEPTED or OK we have a problem + if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK)) + { + Log.Error("Failed to update metadata"); + throw new Exception("Failed to update metadata"); + } + + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + } + } + } - private static IList InfosFromContent(RestResponse response) + public void UpdateMetadata(ContainerInfo containerInfo) { - if (response.TimedOut) - return new List(); + if (containerInfo == null) + throw new ArgumentNullException("containerInfo"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + + using(var client=new RestClient(_baseClient)) + { + + client.BaseAddress = GetAccountUrl(containerInfo.Account); + + client.Parameters.Clear(); + - if (response.StatusCode == 0) - return new List(); + //Set Tags + foreach (var tag in containerInfo.Tags) + { + var headerTag = String.Format("X-Container-Meta-{0}", tag.Key); + client.Headers.Add(headerTag, tag.Value); + } - if (response.StatusCode == HttpStatusCode.NoContent) - return new List(); + + //Set Policies + foreach (var policy in containerInfo.Policies) + { + var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key); + client.Headers.Add(headerPolicy, policy.Value); + } + + + var uriBuilder = client.GetAddressBuilder(containerInfo.Name,""); + var uri = uriBuilder.Uri; + + client.UploadValues(uri,new NameValueCollection()); + + + client.AssertStatusOK("UpdateMetadata failed"); + //If the status is NOT ACCEPTED or OK we have a problem + if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK)) + { + Log.Error("Failed to update metadata"); + throw new Exception("Failed to update metadata"); + } + + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + } + } + + } - var statusCode = (int)response.StatusCode; - if (statusCode < 200 || statusCode >= 300) + public IList ListObjects(string account, string container, DateTime? since = null) + { + if (String.IsNullOrWhiteSpace(container)) + throw new ArgumentNullException("container"); + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Objects"].Push("List")) { - Trace.TraceWarning("ListObjects failed with code {0} - {1}", response.StatusCode, response.StatusDescription); - return new List(); + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + client.IfModifiedSince = since; + var content = client.DownloadStringWithRetry(container, 3); + + client.AssertStatusOK("ListObjects failed"); + + if (client.StatusCode==HttpStatusCode.NotModified) + return new[]{new NoModificationInfo(account,container)}; + //If the result is empty, return an empty list, + var infos = String.IsNullOrWhiteSpace(content) + ? new List() + //Otherwise deserialize the object list into a list of ObjectInfos + : JsonConvert.DeserializeObject>(content); + + foreach (var info in infos) + { + info.Container = container; + info.Account = account; + info.StorageUri = this.StorageUrl; + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + return infos; + } } + } + + public IList ListObjects(string account, string container, string folder, DateTime? since = null) + { + if (String.IsNullOrWhiteSpace(container)) + throw new ArgumentNullException("container"); +/* + if (String.IsNullOrWhiteSpace(folder)) + throw new ArgumentNullException("folder"); +*/ + Contract.EndContractBlock(); + + using (ThreadContext.Stacks["Objects"].Push("List")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); - var infos = JsonConvert.DeserializeObject>(response.Content); - return infos; + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + client.Parameters.Add("path", folder); + client.IfModifiedSince = since; + var content = client.DownloadStringWithRetry(container, 3); + client.AssertStatusOK("ListObjects failed"); + + if (client.StatusCode==HttpStatusCode.NotModified) + return new[]{new NoModificationInfo(account,container,folder)}; + + var infos = JsonConvert.DeserializeObject>(content); + foreach (var info in infos) + { + info.Account = account; + if (info.Container == null) + info.Container = container; + info.StorageUri = this.StorageUrl; + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + return infos; + } + } } - public bool ContainerExists(string container) + + public bool ContainerExists(string account, string container) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); + Contract.EndContractBlock(); - var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout }; - var response = _client.Request(request); - - switch(response.StatusCode) + using (ThreadContext.Stacks["Containters"].Push("Exists")) { - case HttpStatusCode.NoContent: - return true; - case HttpStatusCode.NotFound: - return false; - default: - throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode)); + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Head(container, 3); + + bool result; + switch (client.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + result=true; + break; + case HttpStatusCode.NotFound: + result=false; + break; + default: + throw CreateWebException("ContainerExists", client.StatusCode); + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + + return result; + } + } } - public bool ObjectExists(string container,string objectName) + public bool ObjectExists(string account, string container, string objectName) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); if (String.IsNullOrWhiteSpace(objectName)) throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + Contract.EndContractBlock(); + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); - var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head,RetryPolicy = _retryPolicy, Timeout = _shortTimeout }; - var response = _client.Request(request); + client.Parameters.Clear(); + client.Head(container + "/" + objectName, 3); - switch (response.StatusCode) - { - case HttpStatusCode.OK: - case HttpStatusCode.NoContent: - return true; - case HttpStatusCode.NotFound: - return false; - default: - throw new WebException(String.Format("ObjectExists failed with unexpected status code {0}", response.StatusCode)); + switch (client.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + return true; + case HttpStatusCode.NotFound: + return false; + default: + throw CreateWebException("ObjectExists", client.StatusCode); + } } - + } - public ObjectInfo GetObjectInfo(string container, string objectName) + public ObjectInfo GetObjectInfo(string account, string container, string objectName) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); if (String.IsNullOrWhiteSpace(objectName)) throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + Contract.EndContractBlock(); + using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo")) + { - var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout }; - var response = _client.Request(request); - - if (response.TimedOut) - return ObjectInfo.Empty; - - switch (response.StatusCode) - { - case HttpStatusCode.OK: - case HttpStatusCode.NoContent: - var keys = response.Headers.AllKeys.AsQueryable(); - return new ObjectInfo - { - Name = objectName, - Bytes = long.Parse(GetHeaderValue("Content-Length", response, keys)), - Hash = GetHeaderValue("ETag", response, keys), - Content_Type = GetHeaderValue("Content-Type", response, keys) - }; - case HttpStatusCode.NotFound: - return ObjectInfo.Empty; - default: - if (request.RetryState.RepeatCount > 0) + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + try { - Trace.TraceWarning("[RETRY FAIL] GetObjectInfo for {0} failed after {1} retries", - objectName, request.RetryState.RepeatCount); + client.Parameters.Clear(); + + client.Head(container + "/" + objectName, 3); + + if (client.TimedOut) + return ObjectInfo.Empty; + + switch (client.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + var keys = client.ResponseHeaders.AllKeys.AsQueryable(); + var tags = client.GetMeta("X-Object-Meta-"); + var extensions = (from key in keys + where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-") + select new {Name = key, Value = client.ResponseHeaders[key]}) + .ToDictionary(t => t.Name, t => t.Value); + + var permissions=client.GetHeaderValue("X-Object-Sharing", true); + + + var info = new ObjectInfo + { + Account = account, + Container = container, + Name = objectName, + ETag = client.GetHeaderValue("ETag"), + X_Object_Hash = client.GetHeaderValue("X-Object-Hash"), + Content_Type = client.GetHeaderValue("Content-Type"), + Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)), + Tags = tags, + Last_Modified = client.LastModified, + Extensions = extensions, + ContentEncoding=client.GetHeaderValue("Content-Encoding",true), + ContendDisposition = client.GetHeaderValue("Content-Disposition",true), + Manifest=client.GetHeaderValue("X-Object-Manifest",true), + PublicUrl=client.GetHeaderValue("X-Object-Public",true), + StorageUri=this.StorageUrl, + }; + info.SetPermissions(permissions); + return info; + case HttpStatusCode.NotFound: + return ObjectInfo.Empty; + default: + throw new WebException( + String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", + objectName, client.StatusCode)); + } + + } + catch (RetryException) + { + Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName); return ObjectInfo.Empty; } - if (response.InnerException != null) - throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode), response.InnerException); - throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode)); + catch (WebException e) + { + Log.Error( + String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", + objectName, client.StatusCode), e); + throw; + } + } } + } - public void CreateFolder(string container, string folder) + public void CreateFolder(string account, string container, string folder) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); if (String.IsNullOrWhiteSpace(folder)) throw new ArgumentNullException("folder", "The folder property can't be empty"); + Contract.EndContractBlock(); var folderUrl=String.Format("{0}/{1}",container,folder); - var request = new RestRequest { Path = folderUrl, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout }; - request.AddHeader("Content-Type", @"application/directory"); - request.AddHeader("Content-Length", "0"); - - var response = _client.Request(request); + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); - if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted) - throw new WebException(String.Format("CreateFolder failed with unexpected status code {0}", response.StatusCode)); + client.Parameters.Clear(); + client.Headers.Add("Content-Type", @"application/directory"); + client.Headers.Add("Content-Length", "0"); + client.PutWithRetry(folderUrl, 3); + if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted) + throw CreateWebException("CreateFolder", client.StatusCode); + } } - public ContainerInfo GetContainerInfo(string container) + + + public ContainerInfo GetContainerInfo(string account, string container) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); + Contract.EndContractBlock(); - var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout }; - var response = _client.Request(request); - - switch(response.StatusCode) + using (var client = new RestClient(_baseClient)) { - case HttpStatusCode.NoContent: - var keys = response.Headers.AllKeys.AsQueryable(); - var containerInfo = new ContainerInfo - { - Name = container, - Count =long.Parse(GetHeaderValue("X-Container-Object-Count", response, keys)), - Bytes =long.Parse(GetHeaderValue("X-Container-Bytes-Used", response, keys)) - }; - return containerInfo; - case HttpStatusCode.NotFound: - return ContainerInfo.Empty; - default: - throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode)); + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Head(container); + switch (client.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + var tags = client.GetMeta("X-Container-Meta-"); + var policies = client.GetMeta("X-Container-Policy-"); + + var containerInfo = new ContainerInfo + { + Account=account, + Name = container, + StorageUrl=this.StorageUrl.ToString(), + Count = + long.Parse(client.GetHeaderValue("X-Container-Object-Count")), + Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")), + BlockHash = client.GetHeaderValue("X-Container-Block-Hash"), + BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")), + Last_Modified=client.LastModified, + Tags=tags, + Policies=policies + }; + + + return containerInfo; + case HttpStatusCode.NotFound: + return ContainerInfo.Empty; + default: + throw CreateWebException("GetContainerInfo", client.StatusCode); + } } } - public void CreateContainer(string container) + public void CreateContainer(string account, string container) { + if (String.IsNullOrWhiteSpace(account)) + throw new ArgumentNullException("account"); if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); + throw new ArgumentNullException("container"); + Contract.EndContractBlock(); - var request = new RestRequest { Path = container, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout }; - - var response = _client.Request(request); - - if (response.StatusCode!=HttpStatusCode.Created && response.StatusCode!=HttpStatusCode.Accepted ) - throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}", response.StatusCode)); + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.PutWithRetry(container, 3); + var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK}; + if (!expectedCodes.Contains(client.StatusCode)) + throw CreateWebException("CreateContainer", client.StatusCode); + } } - public void DeleteContainer(string container) + public void DeleteContainer(string account, string container) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); + Contract.EndContractBlock(); - var request = new RestRequest { Path = container, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout }; - var response = _client.Request(request); + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); - if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent) - return; - else - throw new WebException(String.Format("DeleteContainer failed with unexpected status code {0}", response.StatusCode)); + client.DeleteWithRetry(container, 3); + var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent}; + if (!expectedCodes.Contains(client.StatusCode)) + throw CreateWebException("DeleteContainer", client.StatusCode); + } } /// /// /// + /// /// /// + /// /// - /// >This method should have no timeout or a very long one - public Stream GetObject(string container, string objectName) + /// This method should have no timeout or a very long one + //Asynchronously download the object specified by *objectName* in a specific *container* to + // a local file + public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + Contract.EndContractBlock(); - var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Get }; -/* - if (DownloadPercentLimit > 0) - request.TaskOptions = new TaskOptions { RateLimitPercent = DownloadPercentLimit }; -*/ - - var response = _client.Request(request); - - if (response.StatusCode == HttpStatusCode.NotFound) - throw new FileNotFoundException(); - if (response.StatusCode == HttpStatusCode.OK) + try + { + //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient + //object to avoid concurrency errors. + // + //Download operations take a long time therefore they have no timeout. + using(var client = new RestClient(_baseClient) { Timeout = 0 }) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + //The container and objectName are relative names. They are joined with the client's + //BaseAddress to create the object's absolute address + var builder = client.GetAddressBuilder(container, objectName); + var uri = builder.Uri; + + //Download progress is reported to the Trace log + Log.InfoFormat("[GET] START {0}", objectName); + client.DownloadProgressChanged += (sender, args) => + Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}", + fileName, args.ProgressPercentage, + args.BytesReceived, + args.TotalBytesToReceive); + + + //Start downloading the object asynchronously + await client.DownloadFileTaskAsync(uri, fileName,cancellationToken); + + //Once the download completes + //Delete the local client object + } + //And report failure or completion + } + catch (Exception exc) { - return response.ContentStream; + Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc); + throw; } - else - throw new WebException(String.Format("GetObject failed with unexpected status code {0}", response.StatusCode)); + + Log.InfoFormat("[GET] END {0}", objectName); + + } + public Task> PutHashMap(string account, string container, string objectName, TreeHash hash) + { + if (String.IsNullOrWhiteSpace(container)) + throw new ArgumentNullException("container"); + if (String.IsNullOrWhiteSpace(objectName)) + throw new ArgumentNullException("objectName"); + if (hash==null) + throw new ArgumentNullException("hash"); + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + Contract.EndContractBlock(); + + + //Don't use a timeout because putting the hashmap may be a long process + var client = new RestClient(_baseClient) { Timeout = 0 }; + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + //The container and objectName are relative names. They are joined with the client's + //BaseAddress to create the object's absolute address + var builder = client.GetAddressBuilder(container, objectName); + builder.Query = "format=json&hashmap"; + var uri = builder.Uri; + + + //Send the tree hash as Json to the server + client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream"; + var jsonHash = hash.ToJson(); + var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash); + if (Log.IsDebugEnabled) + Log.DebugFormat("Hashes:\r\n{0}", jsonHash); + return uploadTask.ContinueWith(t => + { + + var empty = (IList)new List(); + + + //The server will respond either with 201-created if all blocks were already on the server + if (client.StatusCode == HttpStatusCode.Created) + { + //in which case we return an empty hash list + return empty; + } + //or with a 409-conflict and return the list of missing parts + //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception + if (t.IsFaulted) + { + var ex = t.Exception.InnerException; + var we = ex as WebException; + var response = we.Response as HttpWebResponse; + if (response!=null && response.StatusCode==HttpStatusCode.Conflict) + { + //In case of 409 the missing parts will be in the response content + using (var stream = response.GetResponseStream()) + using(var reader=stream.GetLoggedReader(Log)) + { + //We used to have to cleanup the content before returning it because it contains + //error content after the list of hashes + // + //As of 30/1/2012, the result is a proper Json array so we don't need to read the content + //line by line + + var serializer = new JsonSerializer(); + serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member); + var hashes = (List)serializer.Deserialize(reader, typeof(List)); + return hashes; + } + } + //Any other status code is unexpected and the exception should be rethrown + Log.LogError(response); + throw ex; + + } + + //Any other status code is unexpected but there was no exception. We can probably continue processing + Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription); + + return empty; + }); + + } + + + public async Task GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken) + { + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + if (String.IsNullOrWhiteSpace(container)) + throw new ArgumentNullException("container"); + if (relativeUrl == null) + throw new ArgumentNullException("relativeUrl"); + if (end.HasValue && end < 0) + throw new ArgumentOutOfRangeException("end"); + if (start < 0) + throw new ArgumentOutOfRangeException("start"); + Contract.EndContractBlock(); + + //Don't use a timeout because putting the hashmap may be a long process + using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end}) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + var builder = client.GetAddressBuilder(container, relativeUrl.ToString()); + var uri = builder.Uri; + + client.DownloadProgressChanged += (sender, args) => + Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}", + uri.Segments.Last(), args.ProgressPercentage, + args.BytesReceived, + args.TotalBytesToReceive); + + + var result = await client.DownloadDataTaskAsync(uri, cancellationToken); + return result; + } + } + + + public async Task PostBlock(string account, string container, byte[] block, int offset, int count) + { + if (String.IsNullOrWhiteSpace(container)) + throw new ArgumentNullException("container"); + if (block == null) + throw new ArgumentNullException("block"); + if (offset < 0 || offset >= block.Length) + throw new ArgumentOutOfRangeException("offset"); + if (count < 0 || count > block.Length) + throw new ArgumentOutOfRangeException("count"); + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + Contract.EndContractBlock(); + + + try + { + + //Don't use a timeout because putting the hashmap may be a long process + using (var client = new RestClient(_baseClient) { Timeout = 0 }) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + var builder = client.GetAddressBuilder(container, ""); + //We are doing an update + builder.Query = "update"; + var uri = builder.Uri; + + client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream"; + + Log.InfoFormat("[BLOCK POST] START"); + + client.UploadProgressChanged += (sender, args) => + Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}", + args.ProgressPercentage, args.BytesSent, + args.TotalBytesToSend); + client.UploadFileCompleted += (sender, args) => + Log.InfoFormat("[BLOCK POST PROGRESS] Completed "); + + var buffer = new byte[count]; + Buffer.BlockCopy(block, offset, buffer, 0, count); + //Send the block + await client.UploadDataTaskAsync(uri, "POST", buffer); + Log.InfoFormat("[BLOCK POST] END"); + } + } + catch (Exception exc) + { + Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc); + throw; + } + } + + + public async Task GetHashMap(string account, string container, string objectName) + { + if (String.IsNullOrWhiteSpace(container)) + throw new ArgumentNullException("container"); + if (String.IsNullOrWhiteSpace(objectName)) + throw new ArgumentNullException("objectName"); + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + Contract.EndContractBlock(); + + try + { + //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient + //object to avoid concurrency errors. + // + //Download operations take a long time therefore they have no timeout. + //TODO: Do they really? this is a hashmap operation, not a download + + //Start downloading the object asynchronously + using (var client = new RestClient(_baseClient) { Timeout = 0 }) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + //The container and objectName are relative names. They are joined with the client's + //BaseAddress to create the object's absolute address + var builder = client.GetAddressBuilder(container, objectName); + builder.Query = "format=json&hashmap"; + var uri = builder.Uri; + + + var json = await client.DownloadStringTaskAsync(uri); + var treeHash = TreeHash.Parse(json); + Log.InfoFormat("[GET HASH] END {0}", objectName); + return treeHash; + } + } + catch (Exception exc) + { + Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc); + throw; + } + + } + + /// /// /// + /// /// /// /// /// Optional hash value for the file. If no hash is provided, the method calculates a new hash /// >This method should have no timeout or a very long one - public Task PutObject(string container, string objectName, string fileName, string hash = null) + public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream") { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); @@ -409,75 +1246,68 @@ namespace Pithos.Network throw new ArgumentNullException("objectName", "The objectName property can't be empty"); if (String.IsNullOrWhiteSpace(fileName)) throw new ArgumentNullException("fileName", "The fileName property can't be empty"); - if (!File.Exists(fileName)) - throw new FileNotFoundException("The file does not exist",fileName); - +/* + if (!File.Exists(fileName) && !Directory.Exists(fileName)) + throw new FileNotFoundException("The file or directory does not exist",fileName); +*/ try { - var url = String.Join("/",new[]{_client.Authority,container,objectName}); - var uri = new Uri(url); - - var client = new WebClient(); - string etag = hash ?? CalculateHash(fileName); - client.Headers.Add("Content-Type", "application/octet-stream"); - client.Headers.Add("ETag", etag); - - if(!String.IsNullOrWhiteSpace(_client.Proxy)) - client.Proxy = new WebProxy(_client.Proxy); - - CopyHeaders(_client, client); - - Trace.TraceInformation("[PUT] START {0}", objectName); - client.UploadProgressChanged += (sender, args) => + using (var client = new RestClient(_baseClient) { Timeout = 0 }) { - Trace.TraceInformation("[PROGRESS] {0} {1}% {2} of {3}", fileName, args.ProgressPercentage, args.BytesSent, args.TotalBytesToSend); - }; - - return client.UploadFileTask(uri, "PUT", fileName) - .ContinueWith(upload=> - { - client.Dispose(); - - if (upload.IsFaulted) - { - Trace.TraceError("[PUT] FAIL for {0} with \r{1}",objectName,upload.Exception); - } - else - Trace.TraceInformation("[PUT] END {0}", objectName); - }); + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + var builder = client.GetAddressBuilder(container, objectName); + var uri = builder.Uri; + + string etag = hash ?? CalculateHash(fileName); + + client.Headers.Add("Content-Type", contentType); + client.Headers.Add("ETag", etag); + + + Log.InfoFormat("[PUT] START {0}", objectName); + client.UploadProgressChanged += (sender, args) => + { + using (ThreadContext.Stacks["PUT"].Push("Progress")) + { + Log.InfoFormat("{0} {1}% {2} of {3}", fileName, + args.ProgressPercentage, + args.BytesSent, args.TotalBytesToSend); + } + }; + + client.UploadFileCompleted += (sender, args) => + { + using (ThreadContext.Stacks["PUT"].Push("Progress")) + { + Log.InfoFormat("Completed {0}", fileName); + } + }; + if (contentType=="application/directory") + await client.UploadDataTaskAsync(uri, "PUT", new byte[0]); + else + await client.UploadFileTaskAsync(uri, "PUT", fileName); + } + + Log.InfoFormat("[PUT] END {0}", objectName); } catch (Exception exc) { - Trace.TraceError("[PUT] END {0} with {1}", objectName, exc); + Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc); throw; } } - - /// - /// Copies headers from a Hammock RestClient to a WebClient - /// - /// The RestClient from which the headers are copied - /// The WebClient to which the headers are copied - private static void CopyHeaders(RestClient source, WebClient target) - { - Contract.Requires(source!=null,"source can't be null"); - Contract.Requires(target != null, "target can't be null"); - if (source == null) - throw new ArgumentNullException("source", "source can't be null"); - if (source == null) - throw new ArgumentNullException("target", "target can't be null"); - - foreach (var header in source.GetAllHeaders()) - { - target.Headers.Add(header.Name, header.Value); - } - } - + + private static string CalculateHash(string fileName) { + Contract.Requires(!String.IsNullOrWhiteSpace(fileName)); + Contract.EndContractBlock(); + string hash; using (var hasher = MD5.Create()) using(var stream=File.OpenRead(fileName)) @@ -489,59 +1319,114 @@ namespace Pithos.Network } return hash; } - - public void DeleteObject(string container, string objectName) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName", "The objectName property can't be empty"); - - var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout }; - var response = _client.Request(request); - - if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent) - return; - else - throw new WebException(String.Format("DeleteObject failed with unexpected status code {0}", response.StatusCode)); - - } - - public void MoveObject(string container, string oldObjectName, string newObjectName) + + public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName) { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); + if (String.IsNullOrWhiteSpace(sourceContainer)) + throw new ArgumentNullException("sourceContainer", "The container property can't be empty"); if (String.IsNullOrWhiteSpace(oldObjectName)) throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty"); + if (String.IsNullOrWhiteSpace(targetContainer)) + throw new ArgumentNullException("targetContainer", "The container property can't be empty"); if (String.IsNullOrWhiteSpace(newObjectName)) throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty"); + Contract.EndContractBlock(); - var request = new RestRequest { Path = container + "/" + newObjectName, Method = WebMethod.Put }; - request.AddHeader("X-Copy-From",String.Format("/{0}/{1}",container,oldObjectName)); - request.AddPostContent(new byte[]{}); - var response = _client.Request(request); + var targetUrl = targetContainer + "/" + newObjectName; + var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName); - if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.NoContent || response.StatusCode==HttpStatusCode.Created) + using (var client = new RestClient(_baseClient)) { - this.DeleteObject(container,oldObjectName); - } - else - throw new WebException(String.Format("MoveObject failed with unexpected status code {0}", response.StatusCode)); + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Headers.Add("X-Move-From", sourceUrl); + client.PutWithRetry(targetUrl, 3); + + var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; + if (!expectedCodes.Contains(client.StatusCode)) + throw CreateWebException("MoveObject", client.StatusCode); + } + } + + public void DeleteObject(string account, string sourceContainer, string objectName) + { + if (String.IsNullOrWhiteSpace(sourceContainer)) + throw new ArgumentNullException("sourceContainer", "The container property can't be empty"); + if (String.IsNullOrWhiteSpace(objectName)) + throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty"); + Contract.EndContractBlock(); + + var targetUrl = FolderConstants.TrashContainer + "/" + objectName; + var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Headers.Add("X-Move-From", sourceUrl); + client.AllowedStatusCodes.Add(HttpStatusCode.NotFound); + Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl); + client.PutWithRetry(targetUrl, 3); + + var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound}; + if (!expectedCodes.Contains(client.StatusCode)) + throw CreateWebException("DeleteObject", client.StatusCode); + } } - private string GetHeaderValue(string headerName, RestResponse response, IQueryable keys) + + private static WebException CreateWebException(string operation, HttpStatusCode statusCode) { - if (keys.Any(key => key == headerName)) - return response.Headers[headerName]; - else - throw new WebException(String.Format("The {0} header is missing",headerName)); + return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode)); } - private static void ThrowIfNotStatusOK(RestResponse response, string message) + +/* + public IEnumerable ListDirectories(ContainerInfo container) { - int status = (int)response.StatusCode; - if (status < 200 || status >= 300) - throw new WebException(String.Format("{0} with code {1}",message, status)); + var directories=this.ListObjects(container.Account, container.Name, "/"); + } +*/ + + public bool CanUpload(string account, ObjectInfo cloudFile) + { + Contract.Requires(!String.IsNullOrWhiteSpace(account)); + Contract.Requires(cloudFile!=null); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + + var parts = cloudFile.Name.Split('/'); + var folder = String.Join("/", parts,0,parts.Length-1); + + var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid()); + + client.Parameters.Clear(); + try + { + client.PutWithRetry(fileUrl, 3, @"application/octet-stream"); + + var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; + var result=(expectedCodes.Contains(client.StatusCode)); + DeleteObject(account, cloudFile.Container, fileUrl); + return result; + } + catch + { + return false; + } + } } } + + public class ShareAccountInfo + { + public DateTime? last_modified { get; set; } + public string name { get; set; } + } }