X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/blobdiff_plain/c561991c5d879822668e8997210c77f87119baec..cfb091032c1ea51d109dccf68322ce2ee63eaab5:/trunk/Pithos.Network/CloudFilesClient.cs diff --git a/trunk/Pithos.Network/CloudFilesClient.cs b/trunk/Pithos.Network/CloudFilesClient.cs index 9032fc3..cef59f5 100644 --- a/trunk/Pithos.Network/CloudFilesClient.cs +++ b/trunk/Pithos.Network/CloudFilesClient.cs @@ -54,25 +54,31 @@ using System.Diagnostics.Contracts; using System.IO; using System.Linq; using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; using System.Reflection; -using System.Security.Cryptography; -using System.Text; using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; using Pithos.Interfaces; +using Pithos.Network; using log4net; namespace Pithos.Network { + [Export(typeof(ICloudClient))] - public class CloudFilesClient:ICloudClient + public class CloudFilesClient:ICloudClient,IDisposable { + private const string TOKEN_HEADER = "X-Auth-Token"; private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); //CloudFilesClient uses *_baseClient* internally to communicate with the server //RestClient provides a REST-friendly interface over the standard WebClient. private RestClient _baseClient; + + private HttpClient _baseHttpClient; + private HttpClient _baseHttpClientNoTimeout; //During authentication the client provides a UserName @@ -85,6 +91,14 @@ namespace Pithos.Network //in the X-Auth-Token header private string _token; private readonly string _emptyGuid = Guid.Empty.ToString(); + private readonly Uri _emptyUri = new Uri("",UriKind.Relative); + + private HttpClientHandler _httpClientHandler = new HttpClientHandler + { + AllowAutoRedirect = true, + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + UseCookies = true, + }; public string Token @@ -93,7 +107,7 @@ namespace Pithos.Network set { _token = value; - _baseClient.Headers["X-Auth-Token"] = value; + _baseClient.Headers[TOKEN_HEADER] = value; } } @@ -104,30 +118,6 @@ namespace Pithos.Network public Uri RootAddressUri { get; set; } - /* private WebProxy _proxy; - public WebProxy Proxy - { - get { return _proxy; } - set - { - _proxy = value; - if (_baseClient != null) - _baseClient.Proxy = value; - } - } -*/ - - /* private Uri _proxy; - public Uri Proxy - { - get { return _proxy; } - set - { - _proxy = value; - if (_baseClient != null) - _baseClient.Proxy = new WebProxy(value); - } - }*/ public double DownloadPercentLimit { get; set; } public double UploadPercentLimit { get; set; } @@ -175,10 +165,41 @@ namespace Pithos.Network var usernameIndex = storageUrl.LastIndexOf(UserName); var rootUrl = storageUrl.Substring(0, usernameIndex); RootAddressUri = new Uri(rootUrl); + + var httpClientHandler = new HttpClientHandler + { + AllowAutoRedirect = true, + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + UseCookies = true, + }; + + + _baseHttpClient = new HttpClient(httpClientHandler) + { + BaseAddress = StorageUrl, + Timeout = TimeSpan.FromSeconds(30) + }; + _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, Token); + + _baseHttpClientNoTimeout = new HttpClient(httpClientHandler) + { + BaseAddress = StorageUrl, + Timeout = TimeSpan.FromMilliseconds(-1) + }; + _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, Token); + + } - public AccountInfo Authenticate() + private static void AssertStatusOK(HttpResponseMessage response, string message) + { + var statusCode = response.StatusCode; + if (statusCode >= HttpStatusCode.BadRequest) + throw new WebException(String.Format("{0} with code {1} - {2}", message, statusCode, response.ReasonPhrase)); + } + + public async Task Authenticate() { if (String.IsNullOrWhiteSpace(UserName)) throw new InvalidOperationException("UserName is empty"); @@ -197,47 +218,64 @@ namespace Pithos.Network var groups = new List(); - using (var authClient = new RestClient{BaseAddress=AuthenticationUrl}) + using (var authClient = new HttpClient(_httpClientHandler,false){ BaseAddress = new Uri(AuthenticationUrl),Timeout=TimeSpan.FromSeconds(30) }) { - /* if (Proxy != null) - authClient.Proxy = Proxy;*/ - Contract.Assume(authClient.Headers!=null); + authClient.DefaultRequestHeaders.Add("X-Auth-User", UserName); + authClient.DefaultRequestHeaders.Add("X-Auth-Key", ApiKey); - authClient.Headers.Add("X-Auth-User", UserName); - authClient.Headers.Add("X-Auth-Key", ApiKey); - //TODO: Remove after testing. Added to overcome server auth bug - //authClient.Headers.Add("X-Auth-Token", ApiKey); + string storageUrl; + string token; + + using (var response = await authClient.GetAsyncWithRetries(new Uri(VersionPath, UriKind.Relative),3).ConfigureAwait(false)) // .DownloadStringWithRetryRelative(new Uri(VersionPath, UriKind.Relative), 3); + { + AssertStatusOK(response,"Authentication failed"); + + storageUrl = response.Headers.GetFirstValue("X-Storage-Url"); + if (String.IsNullOrWhiteSpace(storageUrl)) + throw new InvalidOperationException("Failed to obtain storage url"); - authClient.DownloadStringWithRetry(VersionPath, 3); + token = response.Headers.GetFirstValue(TOKEN_HEADER); + if (String.IsNullOrWhiteSpace(token)) + throw new InvalidOperationException("Failed to obtain token url"); + + } - authClient.AssertStatusOK("Authentication failed"); - var storageUrl = authClient.GetHeaderValue("X-Storage-Url"); - if (String.IsNullOrWhiteSpace(storageUrl)) - throw new InvalidOperationException("Failed to obtain storage url"); - _baseClient = new RestClient { BaseAddress = storageUrl, - Timeout = 10000, - Retries = 3, - //Proxy=Proxy + Timeout = 30000, + Retries = 3, }; StorageUrl = new Uri(storageUrl); + Token = token; + + + //Get the root address (StorageUrl without the account) var usernameIndex=storageUrl.LastIndexOf(UserName); var rootUrl = storageUrl.Substring(0, usernameIndex); RootAddressUri = new Uri(rootUrl); - var token = authClient.GetHeaderValue("X-Auth-Token"); - if (String.IsNullOrWhiteSpace(token)) - throw new InvalidOperationException("Failed to obtain token url"); - Token = token; - /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable(); + _baseHttpClient = new HttpClient(_httpClientHandler,false) + { + BaseAddress = StorageUrl, + Timeout = TimeSpan.FromSeconds(30) + }; + _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, token); + + _baseHttpClientNoTimeout = new HttpClient(_httpClientHandler,false) + { + BaseAddress = StorageUrl, + Timeout = TimeSpan.FromMilliseconds(-1) + }; + _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, token); + + /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable(); groups = (from key in keys where key.StartsWith("X-Account-Group-") let name = key.Substring(16) @@ -254,36 +292,51 @@ namespace Pithos.Network } + private static void TraceStart(string method, Uri actualAddress) + { + Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress); + } - - public IList ListContainers(string account) + private async Task GetStringAsync(Uri targetUri, string errorMessage,DateTimeOffset? since=null) { - using (var client = new RestClient(_baseClient)) + TraceStart("GET",targetUri); + var request = new HttpRequestMessage(HttpMethod.Get, targetUri); + if (since.HasValue) { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Parameters.Clear(); - client.Parameters.Add("format", "json"); - var content = client.DownloadStringWithRetry("", 3); - client.AssertStatusOK("List Containers failed"); + request.Headers.IfModifiedSince = since.Value; + } + using (var response = await _baseHttpClient.SendAsyncWithRetries(request,3).ConfigureAwait(false)) + { + AssertStatusOK(response, errorMessage); - if (client.StatusCode == HttpStatusCode.NoContent) - return new List(); - var infos = JsonConvert.DeserializeObject>(content); - - foreach (var info in infos) - { - info.Account = account; - } - return infos; + if (response.StatusCode == HttpStatusCode.NoContent) + return String.Empty; + + var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + return content; } + } + public async Task> ListContainers(string account) + { + + var targetUrl = GetTargetUrl(account); + var targetUri = new Uri(String.Format("{0}?format=json", targetUrl)); + var result = await GetStringAsync(targetUri, "List Containers failed").ConfigureAwait(false); + if (String.IsNullOrWhiteSpace(result)) + return new List(); + var infos = JsonConvert.DeserializeObject>(result); + foreach (var info in infos) + { + info.Account = account; + } + return infos; } + private string GetAccountUrl(string account) { - return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri; + return RootAddressUri.Combine(account).AbsoluteUri; } public IList ListSharingAccounts(DateTime? since=null) @@ -292,28 +345,17 @@ namespace Pithos.Network { if (Log.IsDebugEnabled) Log.DebugFormat("START"); - using (var client = new RestClient(_baseClient)) - { - client.Parameters.Clear(); - client.Parameters.Add("format", "json"); - client.IfModifiedSince = since; - - //Extract the username from the base address - client.BaseAddress = RootAddressUri.AbsoluteUri; - - var content = client.DownloadStringWithRetry(@"", 3); + var targetUri = new Uri(String.Format("{0}?format=json", RootAddressUri), UriKind.Absolute); + var content=TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListSharingAccounts failed", since).ConfigureAwait(false)).Result; - client.AssertStatusOK("ListSharingAccounts failed"); + //If the result is empty, return an empty list, + var infos = String.IsNullOrWhiteSpace(content) + ? new List() + //Otherwise deserialize the account list into a list of ShareAccountInfos + : JsonConvert.DeserializeObject>(content); - //If the result is empty, return an empty list, - var infos = String.IsNullOrWhiteSpace(content) - ? new List() - //Otherwise deserialize the account list into a list of ShareAccountInfos - : JsonConvert.DeserializeObject>(content); - - Log.DebugFormat("END"); - return infos; - } + Log.DebugFormat("END"); + return infos; } } @@ -327,7 +369,7 @@ namespace Pithos.Network /// /// /// - public IList ListSharedObjects(HashSet knownContainers,DateTime? since = null ) + public IList ListSharedObjects(HashSet knownContainers, DateTimeOffset? since) { using (ThreadContext.Stacks["Share"].Push("List Objects")) @@ -336,15 +378,14 @@ namespace Pithos.Network //'since' is not used here because we need to have ListObjects return a NoChange result //for all shared accounts,containers - Func GetKey = c => String.Format("{0}\\{1}", c.Account, c.Name); + Func getKey = c => String.Format("{0}\\{1}", c.Account, c.Name); - var accounts = ListSharingAccounts(); - var containers = (from account in accounts - let conts = ListContainers(account.name) + var containers = (from account in ListSharingAccounts() + let conts = ListContainers(account.name).Result from container in conts select container).ToList(); var items = from container in containers - let actualSince=knownContainers.Contains(GetKey(container))?since:null + let actualSince=knownContainers.Contains(getKey(container))?since:null select ListObjects(container.Account , container.Name, actualSince); var objects=items.SelectMany(r=> r).ToList(); @@ -357,7 +398,7 @@ namespace Pithos.Network //Store any new containers foreach (var container in containers) { - knownContainers.Add(GetKey(container)); + knownContainers.Add(getKey(container)); } @@ -379,13 +420,14 @@ namespace Pithos.Network if (objectInfo.Name == null) continue; - var parts = objectInfo.Name.Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries); + //No need to unescape here, the parts will be used to create new ObjectInfos + var parts = objectInfo.Name.ToString().Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries); //If there is no parent, skip if (parts.Length == 1) continue; var baseParts = new[] { - objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container + objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container.ToString() }; for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++) { @@ -412,10 +454,10 @@ namespace Pithos.Network { Account = objectInfo.Account, Container = objectInfo.Container, - Content_Type = @"application/directory", - ETag = Signature.MD5_EMPTY, + Content_Type = ObjectInfo.CONTENT_TYPE_DIRECTORY, + ETag = Signature.MERKLE_EMPTY, X_Object_Hash = Signature.MERKLE_EMPTY, - Name=parentName, + Name=new Uri(parentName,UriKind.Relative), StorageUri=objectInfo.StorageUri, Bytes = 0, UUID=g.ToString(), @@ -454,7 +496,7 @@ namespace Pithos.Network client.Headers.Add(headerTag, tag.Value); } - client.DownloadStringWithRetry(target.Container, 3); + client.DownloadStringWithRetryRelative(target.Container, 3); client.AssertStatusOK("SetTags failed"); @@ -472,16 +514,20 @@ namespace Pithos.Network } - public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write) + public void ShareObject(string account, Uri container, Uri objectName, string shareTo, bool read, bool write) { if (String.IsNullOrWhiteSpace(Token)) throw new InvalidOperationException("The Token is not set"); if (StorageUrl==null) throw new InvalidOperationException("The StorageUrl is not set"); - if (String.IsNullOrWhiteSpace(container)) + if (container==null) throw new ArgumentNullException("container"); - if (String.IsNullOrWhiteSpace(objectName)) + if (container.IsAbsoluteUri) + throw new ArgumentException("container"); + if (objectName==null) throw new ArgumentNullException("objectName"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("objectName"); if (String.IsNullOrWhiteSpace(account)) throw new ArgumentNullException("account"); if (String.IsNullOrWhiteSpace(shareTo)) @@ -507,7 +553,7 @@ namespace Pithos.Network permission = String.Format("read={0}", shareTo); client.Headers.Add("X-Object-Sharing", permission); - var content = client.DownloadStringWithRetry(container, 3); + var content = client.DownloadStringWithRetryRelative(container, 3); client.AssertStatusOK("ShareObject failed"); @@ -524,7 +570,7 @@ namespace Pithos.Network } - public AccountInfo GetAccountPolicies(AccountInfo accountInfo) + public async Task GetAccountPolicies(AccountInfo accountInfo) { if (accountInfo==null) throw new ArgumentNullException("accountInfo"); @@ -534,38 +580,57 @@ namespace Pithos.Network { if (Log.IsDebugEnabled) Log.DebugFormat("START"); +/* if (_baseClient == null) { _baseClient = new RestClient { BaseAddress = accountInfo.StorageUri.ToString(), - Timeout = 10000, + Timeout = 30000, Retries = 3, }; } - using (var client = new RestClient(_baseClient)) +*/ + var containerUri = GetTargetUri(accountInfo.UserName); + var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute); + using(var response=await _baseHttpClient.HeadAsyncWithRetries(targetUri,3).ConfigureAwait(false)) { - if (!String.IsNullOrWhiteSpace(accountInfo.UserName)) - client.BaseAddress = GetAccountUrl(accountInfo.UserName); - - client.Parameters.Clear(); - client.Parameters.Add("format", "json"); - client.Head(String.Empty, 3); - - var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"]; - var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"]; - + + var quotaValue=response.Headers.GetFirstValue("X-Account-Policy-Quota"); + var bytesValue = response.Headers.GetFirstValue("X-Account-Bytes-Used"); long quota, bytes; if (long.TryParse(quotaValue, out quota)) accountInfo.Quota = quota; if (long.TryParse(bytesValue, out bytes)) accountInfo.BytesUsed = bytes; - - return accountInfo; + return accountInfo; } + + //using (var client = new RestClient(_baseClient)) + //{ + // if (!String.IsNullOrWhiteSpace(accountInfo.UserName)) + // client.BaseAddress = GetAccountUrl(accountInfo.UserName); + + // client.Parameters.Clear(); + // client.Parameters.Add("format", "json"); + // client.Head(_emptyUri, 3); + + // var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"]; + // var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"]; + + // long quota, bytes; + // if (long.TryParse(quotaValue, out quota)) + // accountInfo.Quota = quota; + // if (long.TryParse(bytesValue, out bytes)) + // accountInfo.BytesUsed = bytes; + + // return accountInfo; + + //} + } } @@ -607,15 +672,9 @@ namespace Pithos.Network client.Headers.Add("X-Object-Public", isPublic); - /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name); - uriBuilder.Query = "update="; - var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/ var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name); - client.PostWithRetry(address,"application/xml"); + client.PostWithRetry(new Uri(address,UriKind.Relative),"application/xml"); - //client.UploadValues(uri,new NameValueCollection()); - - client.AssertStatusOK("UpdateMetadata failed"); //If the status is NOT ACCEPTED or OK we have a problem if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK)) @@ -665,7 +724,7 @@ namespace Pithos.Network } - var uriBuilder = client.GetAddressBuilder(containerInfo.Name,""); + var uriBuilder = client.GetAddressBuilder(containerInfo.Name,_emptyUri); var uri = uriBuilder.Uri; client.UploadValues(uri,new NameValueCollection()); @@ -685,63 +744,78 @@ namespace Pithos.Network } + + - public IList ListObjects(string account, string container, DateTime? since = null) + public IList ListObjects(string account, Uri container, DateTimeOffset? since = null) { - if (String.IsNullOrWhiteSpace(container)) + if (container==null) throw new ArgumentNullException("container"); + if (container.IsAbsoluteUri) + throw new ArgumentException("container"); Contract.EndContractBlock(); using (ThreadContext.Stacks["Objects"].Push("List")) { - if (Log.IsDebugEnabled) Log.DebugFormat("START"); - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Parameters.Clear(); - client.Parameters.Add("format", "json"); - client.IfModifiedSince = since; - var content = client.DownloadStringWithRetry(container, 3); - - client.AssertStatusOK("ListObjects failed"); - - if (client.StatusCode==HttpStatusCode.NotModified) - return new[]{new NoModificationInfo(account,container)}; - //If the result is empty, return an empty list, - var infos = String.IsNullOrWhiteSpace(content) - ? new List() - //Otherwise deserialize the object list into a list of ObjectInfos - : JsonConvert.DeserializeObject>(content); - - foreach (var info in infos) - { - info.Container = container; - info.Account = account; - info.StorageUri = this.StorageUrl; - } - if (Log.IsDebugEnabled) Log.DebugFormat("END"); - return infos; - } + var containerUri = GetTargetUri(account).Combine(container); + var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute); + + var content =TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result; + + //304 will result in an empty string. Empty containers return an empty json array + if (String.IsNullOrWhiteSpace(content)) + return new[] {new NoModificationInfo(account, container)}; + + //If the result is empty, return an empty list, + var infos = String.IsNullOrWhiteSpace(content) + ? new List() + //Otherwise deserialize the object list into a list of ObjectInfos + : JsonConvert.DeserializeObject>(content); + + foreach (var info in infos) + { + info.Container = container; + info.Account = account; + info.StorageUri = StorageUrl; + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + return infos; } } - public IList ListObjects(string account, string container, string folder, DateTime? since = null) + public IList ListObjects(string account, Uri container, Uri folder, DateTimeOffset? since = null) { - if (String.IsNullOrWhiteSpace(container)) + if (container==null) throw new ArgumentNullException("container"); -/* - if (String.IsNullOrWhiteSpace(folder)) - throw new ArgumentNullException("folder"); -*/ + if (container.IsAbsoluteUri) + throw new ArgumentException("container"); Contract.EndContractBlock(); using (ThreadContext.Stacks["Objects"].Push("List")) { if (Log.IsDebugEnabled) Log.DebugFormat("START"); + var containerUri = GetTargetUri(account).Combine(container); + var targetUri = new Uri(String.Format("{0}?format=json&path={1}", containerUri,folder), UriKind.Absolute); + var content = TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result; + + //304 will result in an empty string. Empty containers return an empty json array + if (String.IsNullOrWhiteSpace(content)) + return new[] { new NoModificationInfo(account, container) }; + + + var infos = JsonConvert.DeserializeObject>(content); + foreach (var info in infos) + { + info.Account = account; + if (info.Container == null) + info.Container = container; + info.StorageUri = StorageUrl; + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + return infos; +/* using (var client = new RestClient(_baseClient)) { if (!String.IsNullOrWhiteSpace(account)) @@ -749,9 +823,9 @@ namespace Pithos.Network client.Parameters.Clear(); client.Parameters.Add("format", "json"); - client.Parameters.Add("path", folder); + client.Parameters.Add("path", folder.ToString()); client.IfModifiedSince = since; - var content = client.DownloadStringWithRetry(container, 3); + var content = client.DownloadStringWithRetryRelative(container, 3); client.AssertStatusOK("ListObjects failed"); if (client.StatusCode==HttpStatusCode.NotModified) @@ -763,25 +837,51 @@ namespace Pithos.Network info.Account = account; if (info.Container == null) info.Container = container; - info.StorageUri = this.StorageUrl; + info.StorageUri = StorageUrl; } if (Log.IsDebugEnabled) Log.DebugFormat("END"); return infos; } +*/ } } - public bool ContainerExists(string account, string container) + public bool ContainerExists(string account, Uri container) { - if (String.IsNullOrWhiteSpace(container)) + if (container==null) throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException( "The container must be relative","container"); Contract.EndContractBlock(); using (ThreadContext.Stacks["Containters"].Push("Exists")) { if (Log.IsDebugEnabled) Log.DebugFormat("START"); + var targetUri = GetTargetUri(account).Combine(container); + + using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result) + { + + bool result; + switch (response.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + result = true; + break; + case HttpStatusCode.NotFound: + result = false; + break; + default: + throw CreateWebException("ContainerExists", response.StatusCode); + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + + return result; + } +/* using (var client = new RestClient(_baseClient)) { if (!String.IsNullOrWhiteSpace(account)) @@ -807,25 +907,59 @@ namespace Pithos.Network return result; } +*/ } } - public bool ObjectExists(string account, string container, string objectName) + private Uri GetTargetUri(string account) + { + return new Uri(GetTargetUrl(account),UriKind.Absolute); + } + + private string GetTargetUrl(string account) { - if (String.IsNullOrWhiteSpace(container)) + return String.IsNullOrWhiteSpace(account) + ? _baseHttpClient.BaseAddress.ToString() + : GetAccountUrl(account); + } + + public bool ObjectExists(string account, Uri container, Uri objectName) + { + if (container == null) throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (objectName == null) throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); Contract.EndContractBlock(); + var targetUri=GetTargetUri(account).Combine(container).Combine(objectName); + + using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result) + { + switch (response.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + return true; + case HttpStatusCode.NotFound: + return false; + default: + throw CreateWebException("ObjectExists", response.StatusCode); + } + } + +/* using (var client = new RestClient(_baseClient)) { if (!String.IsNullOrWhiteSpace(account)) client.BaseAddress = GetAccountUrl(account); client.Parameters.Clear(); - client.Head(container + "/" + objectName, 3); + client.Head(container.Combine(objectName), 3); switch (client.StatusCode) { @@ -838,21 +972,90 @@ namespace Pithos.Network throw CreateWebException("ObjectExists", client.StatusCode); } } +*/ } - public ObjectInfo GetObjectInfo(string account, string container, string objectName) + public async Task GetObjectInfo(string account, Uri container, Uri objectName) { - if (String.IsNullOrWhiteSpace(container)) + if (container == null) throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative", "container"); + if (objectName == null) throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative", "objectName"); Contract.EndContractBlock(); using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo")) - { + { - using (var client = new RestClient(_baseClient)) + var targetUri = GetTargetUri(account).Combine(container).Combine(objectName); + try + { + using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true)) + { + switch (response.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + var tags = response.Headers.GetMeta("X-Object-Meta-"); + var extensions = (from header in response.Headers + where + header.Key.StartsWith("X-Object-") && + !header.Key.StartsWith("X-Object-Meta-") + select new {Name = header.Key, Value = header.Value.FirstOrDefault()}) + .ToDictionary(t => t.Name, t => t.Value); + + var permissions = response.Headers.GetFirstValue("X-Object-Sharing"); + + + var info = new ObjectInfo + { + Account = account, + Container = container, + Name = objectName, + ETag = response.Headers.ETag.NullSafe(e=>e.Tag), + UUID = response.Headers.GetFirstValue("X-Object-UUID"), + X_Object_Hash = response.Headers.GetFirstValue("X-Object-Hash"), + Content_Type = response.Headers.GetFirstValue("Content-Type"), + Bytes = Convert.ToInt64(response.Content.Headers.ContentLength), + Tags = tags, + Last_Modified = response.Content.Headers.LastModified, + Extensions = extensions, + ContentEncoding = + response.Content.Headers.ContentEncoding.FirstOrDefault(), + ContendDisposition = + response.Content.Headers.ContentDisposition.NullSafe(c=>c.ToString()), + Manifest = response.Headers.GetFirstValue("X-Object-Manifest"), + PublicUrl = response.Headers.GetFirstValue("X-Object-Public"), + StorageUri = StorageUrl, + }; + info.SetPermissions(permissions); + return info; + case HttpStatusCode.NotFound: + return ObjectInfo.Empty; + default: + throw new WebException( + String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", + objectName, response.StatusCode)); + } + } + } + catch (RetryException) + { + Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.", objectName); + return ObjectInfo.Empty; + } + catch (WebException e) + { + Log.Error( + String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status {1}", + objectName, e.Status), e); + throw; + } + } /* using (var client = new RestClient(_baseClient)) { if (!String.IsNullOrWhiteSpace(account)) client.BaseAddress = GetAccountUrl(account); @@ -860,7 +1063,7 @@ namespace Pithos.Network { client.Parameters.Clear(); - client.Head(container + "/" + objectName, 3); + client.Head(container.Combine(objectName), 3); if (client.TimedOut) return ObjectInfo.Empty; @@ -896,7 +1099,7 @@ namespace Pithos.Network ContendDisposition = client.GetHeaderValue("Content-Disposition",true), Manifest=client.GetHeaderValue("X-Object-Manifest",true), PublicUrl=client.GetHeaderValue("X-Object-Public",true), - StorageUri=this.StorageUrl, + StorageUri=StorageUrl, }; info.SetPermissions(permissions); return info; @@ -921,89 +1124,132 @@ namespace Pithos.Network objectName, client.StatusCode), e); throw; } - } - } + } */ + } - public void CreateFolder(string account, string container, string folder) + + + public void CreateFolder(string account, Uri container, Uri folder) { - if (String.IsNullOrWhiteSpace(container)) + if (container == null) throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(folder)) - throw new ArgumentNullException("folder", "The folder property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (folder == null) + throw new ArgumentNullException("folder", "The objectName property can't be empty"); + if (folder.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","folder"); Contract.EndContractBlock(); - var folderUrl=String.Format("{0}/{1}",container,folder); + var folderUri=container.Combine(folder); + var targetUri = GetTargetUri(account).Combine(folderUri); + var message = new HttpRequestMessage(HttpMethod.Put, targetUri); + + message.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY); + message.Headers.Add("Content-Length", "0"); + using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result) + { + if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted) + throw CreateWebException("CreateFolder", response.StatusCode); + } +/* using (var client = new RestClient(_baseClient)) { if (!String.IsNullOrWhiteSpace(account)) client.BaseAddress = GetAccountUrl(account); client.Parameters.Clear(); - client.Headers.Add("Content-Type", @"application/directory"); + client.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY); client.Headers.Add("Content-Length", "0"); - client.PutWithRetry(folderUrl, 3); + client.PutWithRetry(folderUri, 3); if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted) throw CreateWebException("CreateFolder", client.StatusCode); } +*/ + } + + private Dictionary GetMeta(HttpResponseMessage response,string metaPrefix) + { + if (String.IsNullOrWhiteSpace(metaPrefix)) + throw new ArgumentNullException("metaPrefix"); + Contract.EndContractBlock(); + + var dict = (from header in response.Headers + where header.Key.StartsWith(metaPrefix) + select new { Name = header.Key, Value = String.Join(",", header.Value) }) + .ToDictionary(t => t.Name, t => t.Value); + + + return dict; } - - public ContainerInfo GetContainerInfo(string account, string container) + public ContainerInfo GetContainerInfo(string account, Uri container) { - if (String.IsNullOrWhiteSpace(container)) + if (container == null) throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); Contract.EndContractBlock(); - using (var client = new RestClient(_baseClient)) + var targetUri = GetTargetUri(account).Combine(container); + using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result) { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - client.Head(container); - switch (client.StatusCode) + if (Log.IsDebugEnabled) + Log.DebugFormat("ContainerInfo data: {0}\n{1}",response,response.Content.ReadAsStringAsync().Result); + switch (response.StatusCode) { case HttpStatusCode.OK: case HttpStatusCode.NoContent: - var tags = client.GetMeta("X-Container-Meta-"); - var policies = client.GetMeta("X-Container-Policy-"); + var tags = GetMeta(response,"X-Container-Meta-"); + var policies = GetMeta(response,"X-Container-Policy-"); var containerInfo = new ContainerInfo { - Account=account, + Account = account, Name = container, - StorageUrl=this.StorageUrl.ToString(), - Count = - long.Parse(client.GetHeaderValue("X-Container-Object-Count")), - Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")), - BlockHash = client.GetHeaderValue("X-Container-Block-Hash"), - BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")), - Last_Modified=client.LastModified, - Tags=tags, - Policies=policies + StorageUrl = StorageUrl.ToString(), + Count =long.Parse(response.Headers.GetFirstValue("X-Container-Object-Count")), + Bytes = long.Parse(response.Headers.GetFirstValue("X-Container-Bytes-Used")), + BlockHash = response.Headers.GetFirstValue("X-Container-Block-Hash"), + BlockSize = + int.Parse(response.Headers.GetFirstValue("X-Container-Block-Size")), + Last_Modified = response.Content.Headers.LastModified, + Tags = tags, + Policies = policies }; - + return containerInfo; case HttpStatusCode.NotFound: return ContainerInfo.Empty; default: - throw CreateWebException("GetContainerInfo", client.StatusCode); + throw CreateWebException("GetContainerInfo", response.StatusCode); } - } + } } - public void CreateContainer(string account, string container) + public void CreateContainer(string account, Uri container) { - if (String.IsNullOrWhiteSpace(account)) - throw new ArgumentNullException("account"); - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); Contract.EndContractBlock(); + var targetUri=GetTargetUri(account).Combine(container); + var message = new HttpRequestMessage(HttpMethod.Put, targetUri); + message.Headers.Add("Content-Length", "0"); + using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result) + { + var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK}; + if (!expectedCodes.Contains(response.StatusCode)) + throw CreateWebException("CreateContainer", response.StatusCode); + } +/* using (var client = new RestClient(_baseClient)) { if (!String.IsNullOrWhiteSpace(account)) @@ -1014,23 +1260,36 @@ namespace Pithos.Network if (!expectedCodes.Contains(client.StatusCode)) throw CreateWebException("CreateContainer", client.StatusCode); } +*/ } - public void DeleteContainer(string account, string container) + public async Task WipeContainer(string account, Uri container) { - if (String.IsNullOrWhiteSpace(container)) + if (container == null) throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative", "container"); Contract.EndContractBlock(); - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); + await DeleteContainer(account, new Uri(String.Format("{0}?delimiter=/", container), UriKind.Relative)).ConfigureAwait(false); + } - client.DeleteWithRetry(container, 3); - var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent}; - if (!expectedCodes.Contains(client.StatusCode)) - throw CreateWebException("DeleteContainer", client.StatusCode); + + public async Task DeleteContainer(string account, Uri container) + { + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + Contract.EndContractBlock(); + + var targetUri = GetTargetUri(account).Combine(container); + var message = new HttpRequestMessage(HttpMethod.Delete, targetUri); + using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false)) + { + var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent }; + if (!expectedCodes.Contains(response.StatusCode)) + throw CreateWebException("DeleteContainer", response.StatusCode); } } @@ -1042,17 +1301,23 @@ namespace Pithos.Network /// /// /// + /// /// /// This method should have no timeout or a very long one //Asynchronously download the object specified by *objectName* in a specific *container* to // a local file - public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken) + public async Task GetObject(string account, Uri container, Uri objectName, string fileName,CancellationToken cancellationToken) { - if (String.IsNullOrWhiteSpace(container)) + if (container == null) throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (objectName == null) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); Contract.EndContractBlock(); + try { @@ -1084,7 +1349,7 @@ namespace Pithos.Network args.BytesReceived, args.TotalBytesToReceive); if (DownloadProgressChanged!=null) - DownloadProgressChanged(this, args); + DownloadProgressChanged(this, new DownloadArgs(args)); }); //Start downloading the object asynchronously @@ -1106,13 +1371,17 @@ namespace Pithos.Network } - public Task> PutHashMap(string account, string container, string objectName, TreeHash hash) + public async Task> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash) { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName"); - if (hash==null) + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (objectName == null) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); + if (hash == null) throw new ArgumentNullException("hash"); if (String.IsNullOrWhiteSpace(Token)) throw new InvalidOperationException("Invalid Token"); @@ -1120,87 +1389,72 @@ namespace Pithos.Network throw new InvalidOperationException("Invalid Storage Url"); Contract.EndContractBlock(); - - //Don't use a timeout because putting the hashmap may be a long process - var client = new RestClient(_baseClient) { Timeout = 0 }; - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); + //The container and objectName are relative names. They are joined with the client's //BaseAddress to create the object's absolute address - var builder = client.GetAddressBuilder(container, objectName); - builder.Query = "format=json&hashmap"; - var uri = builder.Uri; + var targetUri = GetTargetUri(account).Combine(container).Combine(objectName); + + + var uri = new Uri(String.Format("{0}?format=json&hashmap",targetUri),UriKind.Absolute); + //Send the tree hash as Json to the server - client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream"; var jsonHash = hash.ToJson(); - - client.Headers.Add("ETag",hash.MD5); - var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash); if (Log.IsDebugEnabled) Log.DebugFormat("Hashes:\r\n{0}", jsonHash); - return uploadTask.ContinueWith(t => + + var message = new HttpRequestMessage(HttpMethod.Put, uri) { + Content = new StringContent(jsonHash) + }; + message.Headers.Add("ETag",hash.TopHash.ToHashString()); + + //Don't use a timeout because putting the hashmap may be a long process + using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false)) + { var empty = (IList)new List(); - - //The server will respond either with 201-created if all blocks were already on the server - if (client.StatusCode == HttpStatusCode.Created) - { - //in which case we return an empty hash list - return empty; - } - //or with a 409-conflict and return the list of missing parts - //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception - if (t.IsFaulted) + switch (response.StatusCode) { - var ex = t.Exception.InnerException; - var we = ex as WebException; - var response = we.Response as HttpWebResponse; - if (response!=null && response.StatusCode==HttpStatusCode.Conflict) - { - //In case of 409 the missing parts will be in the response content - using (var stream = response.GetResponseStream()) + case HttpStatusCode.Created: + //The server will respond either with 201-created if all blocks were already on the server + return empty; + case HttpStatusCode.Conflict: + //or with a 409-conflict and return the list of missing parts + using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) using(var reader=stream.GetLoggedReader(Log)) - { - //We used to have to cleanup the content before returning it because it contains - //error content after the list of hashes - // - //As of 30/1/2012, the result is a proper Json array so we don't need to read the content - //line by line - + { var serializer = new JsonSerializer(); serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member); var hashes = (List)serializer.Deserialize(reader, typeof(List)); return hashes; } - } - //Any other status code is unexpected and the exception should be rethrown - Log.LogError(response); - throw ex; - + default: + //All other cases are unexpected + //Ensure that failure codes raise exceptions + response.EnsureSuccessStatusCode(); + //And log any other codes as warngings, but continute processing + Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",response.StatusCode,response.ReasonPhrase); + return empty; } - - //Any other status code is unexpected but there was no exception. We can probably continue processing - Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription); - - return empty; - }); + } } - public async Task GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken) + public async Task GetBlock(string account, Uri container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken) { if (String.IsNullOrWhiteSpace(Token)) throw new InvalidOperationException("Invalid Token"); if (StorageUrl == null) throw new InvalidOperationException("Invalid Storage Url"); - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); if (relativeUrl == null) throw new ArgumentNullException("relativeUrl"); if (end.HasValue && end < 0) @@ -1209,46 +1463,58 @@ namespace Pithos.Network throw new ArgumentOutOfRangeException("start"); Contract.EndContractBlock(); - //Don't use a timeout because putting the hashmap may be a long process - using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end}) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - var builder = client.GetAddressBuilder(container, relativeUrl.ToString()); - var uri = builder.Uri; + var targetUri = GetTargetUri(account).Combine(container).Combine(relativeUrl); + var message = new HttpRequestMessage(HttpMethod.Get, targetUri); + message.Headers.Range=new RangeHeaderValue(start,end); -/* client.DownloadProgressChanged += (sender, args) => - { - Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}", - uri.Segments.Last(), args.ProgressPercentage, - args.BytesReceived, - args.TotalBytesToReceive); - DownloadProgressChanged(sender, args); - };*/ - var progress = new Progress(args => + //Don't use a timeout because putting the hashmap may be a long process + + IProgress progress = new Progress(args => { Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}", - uri.Segments.Last(), args.ProgressPercentage, + targetUri.Segments.Last(), args.ProgressPercentage, args.BytesReceived, args.TotalBytesToReceive); + if (DownloadProgressChanged!=null) - DownloadProgressChanged(this, args); + DownloadProgressChanged(this, args); }); - var result = await client.DownloadDataTaskAsync(uri, cancellationToken,progress).ConfigureAwait(false); + using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false,HttpCompletionOption.ResponseHeadersRead, + cancellationToken).ConfigureAwait(false)) + using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + using(var targetStream=new MemoryStream()) + { + + long totalSize = response.Content.Headers.ContentLength ?? 0; + long total = 0; + var buffer = new byte[65536]; + int read; + while ((read = await stream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) != 0) + { + total += read; + progress.Report(new DownloadArgs(total, totalSize)); + await targetStream.WriteAsync(buffer, 0, read).ConfigureAwait(false); + } + + var result = targetStream.ToArray(); return result; } + } - public event UploadProgressChangedEventHandler UploadProgressChanged; - public event DownloadProgressChangedEventHandler DownloadProgressChanged; + public event EventHandler UploadProgressChanged; + public event EventHandler DownloadProgressChanged; + - public async Task PostBlock(string account, string container, byte[] block, int offset, int count,CancellationToken token) + public async Task PostBlock(string account, Uri container, byte[] block, int offset, int count,string blockHash,CancellationToken token) { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); if (block == null) throw new ArgumentNullException("block"); if (offset < 0 || offset >= block.Length) @@ -1264,48 +1530,44 @@ namespace Pithos.Network try { + var containerUri = GetTargetUri(account).Combine(container); + var targetUri = new Uri(String.Format("{0}?update", containerUri)); + //Don't use a timeout because putting the hashmap may be a long process - using (var client = new RestClient(_baseClient) { Timeout = 0 }) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - var builder = client.GetAddressBuilder(container, ""); - //We are doing an update - builder.Query = "update"; - var uri = builder.Uri; - client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream"; + Log.InfoFormat("[BLOCK POST] START"); - Log.InfoFormat("[BLOCK POST] START"); -/* - client.UploadProgressChanged += (sender, args) => - { - Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}", - args.ProgressPercentage, args.BytesSent, - args.TotalBytesToSend); - UploadProgressChanged(sender, args); - }; -*/ - client.UploadFileCompleted += (sender, args) => - Log.InfoFormat("[BLOCK POST PROGRESS] Completed "); + var progress = new Progress(args => + { + Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}", + args.ProgressPercentage, + args.BytesSent, + args.TotalBytesToSend); + if (UploadProgressChanged != null) + UploadProgressChanged(this,args); + }); - var progress=new Progress(args=> - { - Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}", - args.ProgressPercentage, args.BytesSent, - args.TotalBytesToSend); - if (UploadProgressChanged!=null) - UploadProgressChanged(this, args); - }); - var buffer = new byte[count]; - Buffer.BlockCopy(block, offset, buffer, 0, count); - //Send the block - await client.UploadDataTaskAsync(uri, "POST", buffer,token,progress).ConfigureAwait(false); - Log.InfoFormat("[BLOCK POST] END"); + var message = new HttpRequestMessage(HttpMethod.Post, targetUri) + { + Content = new ByteArrayContentWithProgress(block, offset, count,progress) + }; + message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream"); + + //Send the block + using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3,false,HttpCompletionOption.ResponseContentRead,token).ConfigureAwait(false)) + { + Log.InfoFormat("[BLOCK POST PROGRESS] Completed "); + response.EnsureSuccessStatusCode(); + var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + var cleanHash = responseHash.TrimEnd(); + Debug.Assert(blockHash==cleanHash); + if (responseHash.Equals(cleanHash,StringComparison.OrdinalIgnoreCase)) + Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]",account,container,blockHash,responseHash); } + Log.InfoFormat("[BLOCK POST] END"); } catch (TaskCanceledException ) { @@ -1320,12 +1582,16 @@ namespace Pithos.Network } - public async Task GetHashMap(string account, string container, string objectName) + public async Task GetHashMap(string account, Uri container, Uri objectName) { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName"); + if (container == null) + throw new ArgumentNullException("container", "The container property can't be empty"); + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (objectName == null) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); if (String.IsNullOrWhiteSpace(Token)) throw new InvalidOperationException("Invalid Token"); if (StorageUrl == null) @@ -1334,30 +1600,16 @@ namespace Pithos.Network try { - //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient - //object to avoid concurrency errors. - // - //Download operations take a long time therefore they have no timeout. - //TODO: Do they really? this is a hashmap operation, not a download - - //Start downloading the object asynchronously - using (var client = new RestClient(_baseClient) { Timeout = 0 }) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - //The container and objectName are relative names. They are joined with the client's - //BaseAddress to create the object's absolute address - var builder = client.GetAddressBuilder(container, objectName); - builder.Query = "format=json&hashmap"; - var uri = builder.Uri; + var objectUri = GetTargetUri(account).Combine(container).Combine(objectName); + var targetUri = new Uri(String.Format("{0}?format=json&hashmap", objectUri)); + //Start downloading the object asynchronously + var json = await GetStringAsync(targetUri, "").ConfigureAwait(false); + var treeHash = TreeHash.Parse(json); + Log.InfoFormat("[GET HASH] END {0}", objectName); + return treeHash; - var json = await client.DownloadStringTaskAsync(uri).ConfigureAwait(false); - var treeHash = TreeHash.Parse(json); - Log.InfoFormat("[GET HASH] END {0}", objectName); - return treeHash; - } } catch (Exception exc) { @@ -1376,20 +1628,20 @@ namespace Pithos.Network /// /// /// Optional hash value for the file. If no hash is provided, the method calculates a new hash + /// /// >This method should have no timeout or a very long one - public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream") + public async Task PutObject(string account, Uri container, Uri objectName, string fileName, string hash = Signature.MERKLE_EMPTY, string contentType = "application/octet-stream") { - if (String.IsNullOrWhiteSpace(container)) + if (container == null) throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) + if (container.IsAbsoluteUri) + throw new ArgumentException("The container must be relative","container"); + if (objectName == null) throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); if (String.IsNullOrWhiteSpace(fileName)) throw new ArgumentNullException("fileName", "The fileName property can't be empty"); -/* - if (!File.Exists(fileName) && !Directory.Exists(fileName)) - throw new FileNotFoundException("The file or directory does not exist",fileName); -*/ - try { @@ -1401,10 +1653,11 @@ namespace Pithos.Network var builder = client.GetAddressBuilder(container, objectName); var uri = builder.Uri; - string etag = hash ?? CalculateHash(fileName); + string etag = hash ; client.Headers.Add("Content-Type", contentType); - client.Headers.Add("ETag", etag); + if (contentType!=ObjectInfo.CONTENT_TYPE_DIRECTORY) + client.Headers.Add("ETag", etag); Log.InfoFormat("[PUT] START {0}", objectName); @@ -1424,8 +1677,9 @@ namespace Pithos.Network { Log.InfoFormat("Completed {0}", fileName); } - }; - if (contentType=="application/directory") + }; + + if (contentType==ObjectInfo.CONTENT_TYPE_DIRECTORY) await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false); else await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false); @@ -1440,67 +1694,100 @@ namespace Pithos.Network } } - - private static string CalculateHash(string fileName) + public void MoveObject(string account, Uri sourceContainer, Uri oldObjectName, Uri targetContainer, Uri newObjectName) { - Contract.Requires(!String.IsNullOrWhiteSpace(fileName)); + if (sourceContainer == null) + throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty"); + if (sourceContainer.IsAbsoluteUri) + throw new ArgumentException("The sourceContainer must be relative","sourceContainer"); + if (oldObjectName == null) + throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty"); + if (oldObjectName.IsAbsoluteUri) + throw new ArgumentException("The oldObjectName must be relative","oldObjectName"); + if (targetContainer == null) + throw new ArgumentNullException("targetContainer", "The targetContainer property can't be empty"); + if (targetContainer.IsAbsoluteUri) + throw new ArgumentException("The targetContainer must be relative","targetContainer"); + if (newObjectName == null) + throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty"); + if (newObjectName.IsAbsoluteUri) + throw new ArgumentException("The newObjectName must be relative","newObjectName"); Contract.EndContractBlock(); - string hash; - using (var hasher = MD5.Create()) - using(var stream=File.OpenRead(fileName)) + var baseUri = GetTargetUri(account); + var targetUri = baseUri.Combine(targetContainer).Combine(newObjectName); + var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, oldObjectName),UriKind.Relative); + + var message = new HttpRequestMessage(HttpMethod.Put, targetUri); + message.Headers.Add("X-Move-From", sourceUri.ToString()); + using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result) { - var hashBuilder=new StringBuilder(); - foreach (byte b in hasher.ComputeHash(stream)) - hashBuilder.Append(b.ToString("x2").ToLower()); - hash = hashBuilder.ToString(); + var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; + if (!expectedCodes.Contains(response.StatusCode)) + throw CreateWebException("MoveObject", response.StatusCode); } - return hash; } - - public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName) + + public async Task DeleteObject(string account, Uri sourceContainer, Uri objectName, bool isDirectory) { - if (String.IsNullOrWhiteSpace(sourceContainer)) - throw new ArgumentNullException("sourceContainer", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(oldObjectName)) - throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty"); - if (String.IsNullOrWhiteSpace(targetContainer)) - throw new ArgumentNullException("targetContainer", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(newObjectName)) - throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty"); + if (sourceContainer == null) + throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty"); + if (sourceContainer.IsAbsoluteUri) + throw new ArgumentException("The sourceContainer must be relative","sourceContainer"); + if (objectName == null) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + if (objectName.IsAbsoluteUri) + throw new ArgumentException("The objectName must be relative","objectName"); Contract.EndContractBlock(); - var targetUrl = targetContainer + "/" + newObjectName; - var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName); - using (var client = new RestClient(_baseClient)) + + var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, objectName),UriKind.Relative); + + + if (objectName.OriginalString.EndsWith(".ignore")) + using(var response = await _baseHttpClient.DeleteAsync(sourceUri)){} + else { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); + var relativeUri = new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName), + UriKind.Relative); - client.Headers.Add("X-Move-From", Uri.EscapeUriString(sourceUrl)); - client.PutWithRetry(targetUrl, 3); +/* + var relativeUri = isDirectory + ? new Uri( + String.Format("{0}/{1}?delimiter=/", FolderConstants.TrashContainer, + objectName), UriKind.Relative) + : new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName), + UriKind.Relative); - var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; - if (!expectedCodes.Contains(client.StatusCode)) - throw CreateWebException("MoveObject", client.StatusCode); - } - } +*/ + var targetUri = GetTargetUri(account).Combine(relativeUri); - public void DeleteObject(string account, string sourceContainer, string objectName, bool isDirectory) - { - if (String.IsNullOrWhiteSpace(sourceContainer)) - throw new ArgumentNullException("sourceContainer", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty"); - Contract.EndContractBlock(); + + var message = new HttpRequestMessage(HttpMethod.Put, targetUri); + message.Headers.Add("X-Move-From", sourceUri.ToString()); + + Log.InfoFormat("[TRASH] [{0}] to [{1}]", sourceUri, targetUri); + using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3)) + { + var expectedCodes = new[] + { + HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created, + HttpStatusCode.NotFound + }; + if (!expectedCodes.Contains(response.StatusCode)) + throw CreateWebException("DeleteObject", response.StatusCode); + } + } +/* + var targetUrl = FolderConstants.TrashContainer + "/" + objectName; /* if (isDirectory) targetUrl = targetUrl + "?delimiter=/"; -*/ +#1# var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName); @@ -1509,15 +1796,16 @@ namespace Pithos.Network if (!String.IsNullOrWhiteSpace(account)) client.BaseAddress = GetAccountUrl(account); - client.Headers.Add("X-Move-From", Uri.EscapeUriString(sourceUrl)); + client.Headers.Add("X-Move-From", sourceUrl); client.AllowedStatusCodes.Add(HttpStatusCode.NotFound); Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl); - client.PutWithRetry(targetUrl, 3); + client.PutWithRetry(new Uri(targetUrl,UriKind.Relative), 3); var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound}; if (!expectedCodes.Contains(client.StatusCode)) throw CreateWebException("DeleteObject", client.StatusCode); } +*/ } @@ -1527,44 +1815,64 @@ namespace Pithos.Network } -/* - public IEnumerable ListDirectories(ContainerInfo container) - { - var directories=this.ListObjects(container.Account, container.Name, "/"); - } -*/ - - public bool CanUpload(string account, ObjectInfo cloudFile) + public async Task CanUpload(string account, ObjectInfo cloudFile) { Contract.Requires(!String.IsNullOrWhiteSpace(account)); Contract.Requires(cloudFile!=null); - using (var client = new RestClient(_baseClient)) - { - if (!String.IsNullOrWhiteSpace(account)) - client.BaseAddress = GetAccountUrl(account); - - - var parts = cloudFile.Name.Split('/'); + var parts = cloudFile.Name.ToString().Split('/'); var folder = String.Join("/", parts,0,parts.Length-1); - var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid()); + var fileName = String.Format("{0}/{1}.pithos.ignore", folder, Guid.NewGuid()); + var fileUri=fileName.ToEscapedUri(); - client.Parameters.Clear(); try { - client.PutWithRetry(fileUrl, 3, @"application/octet-stream"); - + var relativeUri = cloudFile.Container.Combine(fileUri); + var targetUri = GetTargetUri(account).Combine(relativeUri); + var message = new HttpRequestMessage(HttpMethod.Put, targetUri); + message.Content.Headers.ContentType =new MediaTypeHeaderValue("application/octet-stream"); + var response=await _baseHttpClient.SendAsyncWithRetries(message, 3); var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; - var result=(expectedCodes.Contains(client.StatusCode)); - DeleteObject(account, cloudFile.Container, fileUrl, cloudFile.IsDirectory); + var result=(expectedCodes.Contains(response.StatusCode)); + await DeleteObject(account, cloudFile.Container, fileUri, cloudFile.IsDirectory); return result; } catch { return false; } + + } + + ~CloudFilesClient() + { + Dispose(false); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + if (_httpClientHandler!=null) + _httpClientHandler.Dispose(); + if (_baseClient!=null) + _baseClient.Dispose(); + if(_baseHttpClient!=null) + _baseHttpClient.Dispose(); + if (_baseHttpClientNoTimeout!=null) + _baseHttpClientNoTimeout.Dispose(); } + _httpClientHandler = null; + _baseClient = null; + _baseHttpClient = null; + _baseHttpClientNoTimeout = null; } }