X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/blobdiff_plain/637bc368f9c73dc17636a6426c0e0257c7ccb0db..42800be8ef2daeeb921d32691126dad3ac297f38:/trunk/Pithos.Network/CloudFilesClient.cs diff --git a/trunk/Pithos.Network/CloudFilesClient.cs b/trunk/Pithos.Network/CloudFilesClient.cs index 8ddb658..889c8dc 100644 --- a/trunk/Pithos.Network/CloudFilesClient.cs +++ b/trunk/Pithos.Network/CloudFilesClient.cs @@ -1,4 +1,9 @@ -using System; +// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos +// +// The class provides methods to upload/download files, delete files, manage containers + + +using System; using System.Collections.Generic; using System.ComponentModel.Composition; using System.Diagnostics.Contracts; @@ -7,34 +12,64 @@ using System.Linq; using System.Net; using System.Security.Cryptography; using System.Text; -using Hammock; -using Hammock.Caching; -using Hammock.Retries; -using Hammock.Serialization; -using Hammock.Web; +using System.Threading.Tasks; using Newtonsoft.Json; using Pithos.Interfaces; +using log4net; namespace Pithos.Network { [Export(typeof(ICloudClient))] public class CloudFilesClient:ICloudClient { - string _rackSpaceAuthUrl = "https://auth.api.rackspacecloud.com"; - private string _pithosAuthUrl = "http://pithos.dev.grnet.gr"; + //CloudFilesClient uses *_baseClient* internally to communicate with the server + //RestClient provides a REST-friendly interface over the standard WebClient. + private RestClient _baseClient; + - private RestClient _client; - private readonly TimeSpan _timeout = TimeSpan.FromSeconds(10); - private readonly int _retries = 5; - public string ApiKey { get; set; } + //During authentication the client provides a UserName public string UserName { get; set; } - public Uri StorageUrl { get; set; } - public string Token { get; set; } - public string AuthUrl + //and and ApiKey to the server + public string ApiKey { get; set; } + + //And receives an authentication Token. This token must be provided in ALL other operations, + //in the X-Auth-Token header + private string _token; + public string Token + { + get { return _token; } + set + { + _token = value; + _baseClient.Headers["X-Auth-Token"] = value; + } + } + + //The client also receives a StorageUrl after authentication. All subsequent operations must + //use this url + public Uri StorageUrl { get; set; } + + + protected Uri RootAddressUri { get; set; } + + private Uri _proxy; + public Uri Proxy { - get { return UsePithos ? _pithosAuthUrl : _rackSpaceAuthUrl; } + get { return _proxy; } + set + { + _proxy = value; + if (_baseClient != null) + _baseClient.Proxy = new WebProxy(value); + } } + + public double DownloadPercentLimit { get; set; } + public double UploadPercentLimit { get; set; } + + public string AuthenticationUrl { get; set; } + public string VersionPath { @@ -43,306 +78,923 @@ namespace Pithos.Network public bool UsePithos { get; set; } - public void Authenticate(string userName,string apiKey) - { - if (String.IsNullOrWhiteSpace(userName)) - throw new ArgumentNullException("userName","The userName property can't be empty"); - if (String.IsNullOrWhiteSpace(apiKey)) - throw new ArgumentNullException("apiKey", "The apiKey property can't be empty"); - + private static readonly ILog Log = LogManager.GetLogger("CloudFilesClient"); + + public CloudFilesClient(string userName, string apiKey) + { UserName = userName; ApiKey = apiKey; + } - string authUrl = UsePithos ? String.Format("{0}/{1}/{2}", AuthUrl, VersionPath,UserName) - : String.Format("{0}/{1}", AuthUrl, VersionPath); - var authClient = new RestClient{Path=authUrl}; - authClient.AddHeader("X-Auth-User", UserName); - authClient.AddHeader("X-Auth-Key", ApiKey); - - var response=authClient.Request(); - - ThrowIfNotStatusOK(response, "Authentication failed"); - - var keys = response.Headers.AllKeys.AsQueryable(); + public CloudFilesClient(AccountInfo accountInfo) + { + if (accountInfo==null) + throw new ArgumentNullException("accountInfo"); + Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); + Contract.Ensures(StorageUrl != null); + Contract.Ensures(_baseClient != null); + Contract.Ensures(RootAddressUri != null); + Contract.EndContractBlock(); + + _baseClient = new RestClient + { + BaseAddress = accountInfo.StorageUri.ToString(), + Timeout = 10000, + Retries = 3 + }; + StorageUrl = accountInfo.StorageUri; + Token = accountInfo.Token; + UserName = accountInfo.UserName; + + //Get the root address (StorageUrl without the account) + var storageUrl = StorageUrl.AbsoluteUri; + var usernameIndex = storageUrl.LastIndexOf(UserName); + var rootUrl = storageUrl.Substring(0, usernameIndex); + RootAddressUri = new Uri(rootUrl); + } - string storageUrl =UsePithos? - String.Format("{0}/{1}/{2}",AuthUrl,VersionPath,UserName) - :GetHeaderValue("X-Storage-Url", response, keys); - - if (String.IsNullOrWhiteSpace(storageUrl)) - throw new InvalidOperationException("Failed to obtain storage url"); - StorageUrl = new Uri(storageUrl); - if (!UsePithos) + public AccountInfo Authenticate() + { + if (String.IsNullOrWhiteSpace(UserName)) + throw new InvalidOperationException("UserName is empty"); + if (String.IsNullOrWhiteSpace(ApiKey)) + throw new InvalidOperationException("ApiKey is empty"); + if (String.IsNullOrWhiteSpace(AuthenticationUrl)) + throw new InvalidOperationException("AuthenticationUrl is empty"); + Contract.Ensures(!String.IsNullOrWhiteSpace(Token)); + Contract.Ensures(StorageUrl != null); + Contract.Ensures(_baseClient != null); + Contract.Ensures(RootAddressUri != null); + Contract.EndContractBlock(); + + + Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName); + + using (var authClient = new RestClient{BaseAddress=AuthenticationUrl}) { - var token = GetHeaderValue("X-Auth-Token", response, keys); + if (Proxy != null) + authClient.Proxy = new WebProxy(Proxy); + + Contract.Assume(authClient.Headers!=null); + + authClient.Headers.Add("X-Auth-User", UserName); + authClient.Headers.Add("X-Auth-Key", ApiKey); + + authClient.DownloadStringWithRetry(VersionPath, 3); + + authClient.AssertStatusOK("Authentication failed"); + + var storageUrl = authClient.GetHeaderValue("X-Storage-Url"); + if (String.IsNullOrWhiteSpace(storageUrl)) + throw new InvalidOperationException("Failed to obtain storage url"); + + _baseClient = new RestClient + { + BaseAddress = storageUrl, + Timeout = 10000, + Retries = 3 + }; + + StorageUrl = new Uri(storageUrl); + + //Get the root address (StorageUrl without the account) + var usernameIndex=storageUrl.LastIndexOf(UserName); + var rootUrl = storageUrl.Substring(0, usernameIndex); + RootAddressUri = new Uri(rootUrl); + + var token = authClient.GetHeaderValue("X-Auth-Token"); if (String.IsNullOrWhiteSpace(token)) throw new InvalidOperationException("Failed to obtain token url"); Token = token; - } - else - Token = "0000"; - var retryPolicy = new RetryPolicy { RetryCount = _retries }; - retryPolicy.RetryConditions.Add(new TimeoutRetryCondition()); + } - _client = new RestClient { Authority = StorageUrl.AbsoluteUri, RetryPolicy = retryPolicy, Timeout = _timeout,Path=UserName }; + Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName); - _client.AddHeader("X-Auth-Token", Token); - if (UsePithos) + + return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName}; + + } + + + + public IList ListContainers(string account) + { + using (var client = new RestClient(_baseClient)) { - _client.AddHeader("X-Auth-User", UserName); - _client.AddHeader("X-Auth-Key",ApiKey); + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + var content = client.DownloadStringWithRetry("", 3); + client.AssertStatusOK("List Containers failed"); + + if (client.StatusCode == HttpStatusCode.NoContent) + return new List(); + var infos = JsonConvert.DeserializeObject>(content); + + foreach (var info in infos) + { + info.Account = account; + } + return infos; } + } + private string GetAccountUrl(string account) + { + return new Uri(this.RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri; } - public IList ListContainers() - { - //Workaround for Hammock quirk: Hammock always - //appends a / unless a Path is specified. - - //Create a request with a complete path - var request = new RestRequest{Path=StorageUrl.ToString()}; - request.AddParameter("format","json"); - //Create a client clone - var client = new RestClient(); - foreach (var header in _client.GetAllHeaders()) + public IList ListSharingAccounts(DateTime? since=null) + { + using (log4net.ThreadContext.Stacks["Share"].Push("List Accounts")) { - client.AddHeader(header.Name,header.Value); - } + if (Log.IsDebugEnabled) Log.DebugFormat("START"); - var response = client.Request(request); + using (var client = new RestClient(_baseClient)) + { + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + client.IfModifiedSince = since; - ThrowIfNotStatusOK(response, "List Containers failed"); + //Extract the username from the base address + client.BaseAddress = RootAddressUri.AbsoluteUri; - if (response.StatusCode == HttpStatusCode.NoContent) - return new List(); + var content = client.DownloadStringWithRetry(@"", 3); - var infos=JsonConvert.DeserializeObject>(response.Content); - - return infos; + client.AssertStatusOK("ListSharingAccounts failed"); + + //If the result is empty, return an empty list, + var infos = String.IsNullOrWhiteSpace(content) + ? new List() + //Otherwise deserialize the account list into a list of ShareAccountInfos + : JsonConvert.DeserializeObject>(content); + + Log.DebugFormat("END"); + return infos; + } + } + } + + //Request listing of all objects in a container modified since a specific time. + //If the *since* value is missing, return all objects + public IList ListSharedObjects(DateTime? since = null) + { + + using (log4net.ThreadContext.Stacks["Share"].Push("List Objects")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + var objects = new List(); + var accounts = ListSharingAccounts(since); + foreach (var account in accounts) + { + var containers = ListContainers(account.name); + foreach (var container in containers) + { + var containerObjects = ListObjects(account.name, container.Name, null); + objects.AddRange(containerObjects); + } + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + return objects; + } } - public IList ListObjects(string container) + public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write) { + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("The Token is not set"); + if (StorageUrl==null) + throw new InvalidOperationException("The StorageUrl is not set"); if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); + throw new ArgumentNullException("container"); + if (String.IsNullOrWhiteSpace(objectName)) + throw new ArgumentNullException("objectName"); + if (String.IsNullOrWhiteSpace(account)) + throw new ArgumentNullException("account"); + if (String.IsNullOrWhiteSpace(shareTo)) + throw new ArgumentNullException("shareTo"); + Contract.EndContractBlock(); + + using (log4net.ThreadContext.Stacks["Share"].Push("Share Object")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { - var request = new RestRequest{Path=container}; - request.AddParameter("format", "json"); - var response = _client.Request(request); - if (response.TimedOut) - return new List(); + client.BaseAddress = GetAccountUrl(account); - ThrowIfNotStatusOK(response, "List Objects failed"); + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); - if (response.StatusCode == HttpStatusCode.NoContent) - return new List(); - + string permission = ""; + if (write) + permission = String.Format("write={0}", shareTo); + else if (read) + permission = String.Format("read={0}", shareTo); + client.Headers.Add("X-Object-Sharing", permission); + + var content = client.DownloadStringWithRetry(container, 3); + + client.AssertStatusOK("ShareObject failed"); + + //If the result is empty, return an empty list, + var infos = String.IsNullOrWhiteSpace(content) + ? new List() + //Otherwise deserialize the object list into a list of ObjectInfos + : JsonConvert.DeserializeObject>(content); + + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + } + } + + + } + + public AccountInfo GetAccountPolicies(AccountInfo accountInfo) + { + if (accountInfo==null) + throw new ArgumentNullException("accountInfo"); + Contract.EndContractBlock(); + + using (log4net.ThreadContext.Stacks["Account"].Push("GetPolicies")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(accountInfo.UserName)) + client.BaseAddress = GetAccountUrl(accountInfo.UserName); + + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + client.Head(String.Empty, 3); + + var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"]; + var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"]; + + long quota, bytes; + if (long.TryParse(quotaValue, out quota)) + accountInfo.Quota = quota; + if (long.TryParse(bytesValue, out bytes)) + accountInfo.BytesUsed = bytes; + + return accountInfo; + + } - var infos = JsonConvert.DeserializeObject>(response.Content); + } + } - return infos; + + public IList ListObjects(string account, string container, DateTime? since = null) + { + if (String.IsNullOrWhiteSpace(container)) + throw new ArgumentNullException("container"); + Contract.EndContractBlock(); + + using (log4net.ThreadContext.Stacks["Objects"].Push("List")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + client.IfModifiedSince = since; + var content = client.DownloadStringWithRetry(container, 3); + + client.AssertStatusOK("ListObjects failed"); + + //If the result is empty, return an empty list, + var infos = String.IsNullOrWhiteSpace(content) + ? new List() + //Otherwise deserialize the object list into a list of ObjectInfos + : JsonConvert.DeserializeObject>(content); + + foreach (var info in infos) + { + info.Container = container; + info.Account = account; + } + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + return infos; + } + } } - public IList ListObjects(string container,string folder) + + public IList ListObjects(string account, string container, string folder, DateTime? since = null) { if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); + throw new ArgumentNullException("container"); + if (String.IsNullOrWhiteSpace(folder)) + throw new ArgumentNullException("folder"); + Contract.EndContractBlock(); - var request = new RestRequest{Path=container}; - request.AddParameter("format", "json"); - request.AddParameter("path", folder); - var response = _client.Request(request); - if (response.TimedOut) - return new List(); + using (log4net.ThreadContext.Stacks["Objects"].Push("List")) + { + if (Log.IsDebugEnabled) Log.DebugFormat("START"); - ThrowIfNotStatusOK(response, "List Objects failed"); + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); - if (response.StatusCode == HttpStatusCode.NoContent) - return new List(); - + client.Parameters.Clear(); + client.Parameters.Add("format", "json"); + client.Parameters.Add("path", folder); + client.IfModifiedSince = since; + var content = client.DownloadStringWithRetry(container, 3); + client.AssertStatusOK("ListObjects failed"); - var infos = JsonConvert.DeserializeObject>(response.Content); + var infos = JsonConvert.DeserializeObject>(content); - return infos; + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + return infos; + } + } } - public bool ContainerExists(string container) + + public bool ContainerExists(string account, string container) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); + Contract.EndContractBlock(); - var request = new RestRequest {Path = container, Method = WebMethod.Head}; - var response = _client.Request(request); - - switch(response.StatusCode) + using (log4net.ThreadContext.Stacks["Containters"].Push("Exists")) { - case HttpStatusCode.NoContent: - return true; - case HttpStatusCode.NotFound: - return false; - default: - throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode)); + if (Log.IsDebugEnabled) Log.DebugFormat("START"); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Head(container, 3); + + bool result; + switch (client.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + result=true; + break; + case HttpStatusCode.NotFound: + result=false; + break; + default: + throw CreateWebException("ContainerExists", client.StatusCode); + } + if (Log.IsDebugEnabled) Log.DebugFormat("END"); + + return result; + } + } } - public bool ObjectExists(string container,string objectName) + public bool ObjectExists(string account, string container, string objectName) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); if (String.IsNullOrWhiteSpace(objectName)) throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + Contract.EndContractBlock(); - - var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head }; - var response = _client.Request(request); - - switch (response.StatusCode) + using (var client = new RestClient(_baseClient)) { - case HttpStatusCode.OK: - case HttpStatusCode.NoContent: - return true; - case HttpStatusCode.NotFound: - return false; - default: - throw new WebException(String.Format("ObjectExists failed with unexpected status code {0}", response.StatusCode)); + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Parameters.Clear(); + client.Head(container + "/" + objectName, 3); + + switch (client.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + return true; + case HttpStatusCode.NotFound: + return false; + default: + throw CreateWebException("ObjectExists", client.StatusCode); + } } - + } - public ObjectInfo GetObjectInfo(string container, string objectName) + public ObjectInfo GetObjectInfo(string account, string container, string objectName) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); if (String.IsNullOrWhiteSpace(objectName)) throw new ArgumentNullException("objectName", "The objectName property can't be empty"); - - - var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head }; - var response = _client.Request(request); - - if (response.TimedOut) - return ObjectInfo.Empty; - - switch (response.StatusCode) - { - case HttpStatusCode.OK: - case HttpStatusCode.NoContent: - var keys = response.Headers.AllKeys.AsQueryable(); - return new ObjectInfo - { - Name=objectName, - Bytes = long.Parse(GetHeaderValue("Content-Length", response, keys)), - Hash = GetHeaderValue("ETag", response, keys), - Content_Type = GetHeaderValue("Content-Type", response, keys) - }; - case HttpStatusCode.NotFound: - return ObjectInfo.Empty; - default: - throw new WebException(String.Format("GetObjectInfo failed with unexpected status code {0}", response.StatusCode)); + Contract.EndContractBlock(); + + using (log4net.ThreadContext.Stacks["Objects"].Push("GetObjectInfo")) + { + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + try + { + client.Parameters.Clear(); + + client.Head(container + "/" + objectName, 3); + + if (client.TimedOut) + return ObjectInfo.Empty; + + switch (client.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + var keys = client.ResponseHeaders.AllKeys.AsQueryable(); + var tags = (from key in keys + where key.StartsWith("X-Object-Meta-") + let name = key.Substring(14) + select new {Name = name, Value = client.ResponseHeaders[name]}) + .ToDictionary(t => t.Name, t => t.Value); + var extensions = (from key in keys + where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-") + select new {Name = key, Value = client.ResponseHeaders[key]}) + .ToDictionary(t => t.Name, t => t.Value); + var info = new ObjectInfo + { + Account = account, + Container = container, + Name = objectName, + Hash = client.GetHeaderValue("ETag"), + Content_Type = client.GetHeaderValue("Content-Type"), + Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length")), + Tags = tags, + Last_Modified = client.LastModified, + Extensions = extensions + }; + return info; + case HttpStatusCode.NotFound: + return ObjectInfo.Empty; + default: + throw new WebException( + String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", + objectName, client.StatusCode)); + } + + } + catch (RetryException) + { + Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed."); + return ObjectInfo.Empty; + } + catch (WebException e) + { + Log.Error( + String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", + objectName, client.StatusCode), e); + throw; + } + } } + } - public void CreateFolder(string container, string folder) + public void CreateFolder(string account, string container, string folder) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); if (String.IsNullOrWhiteSpace(folder)) throw new ArgumentNullException("folder", "The folder property can't be empty"); + Contract.EndContractBlock(); var folderUrl=String.Format("{0}/{1}",container,folder); - var request = new RestRequest { Path = folderUrl, Method = WebMethod.Put }; - request.AddHeader("Content-Type", @"application/directory"); - request.AddHeader("Content-Length", "0"); - - var response = _client.Request(request); + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); - if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted) - throw new WebException(String.Format("CreateFolder failed with unexpected status code {0}", response.StatusCode)); + client.Parameters.Clear(); + client.Headers.Add("Content-Type", @"application/directory"); + client.Headers.Add("Content-Length", "0"); + client.PutWithRetry(folderUrl, 3); + if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted) + throw CreateWebException("CreateFolder", client.StatusCode); + } } - public ContainerInfo GetContainerInfo(string container) + public ContainerInfo GetContainerInfo(string account, string container) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); + Contract.EndContractBlock(); - var request = new RestRequest {Path = container, Method = WebMethod.Head}; - var response = _client.Request(request); + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Head(container); + switch (client.StatusCode) + { + case HttpStatusCode.OK: + case HttpStatusCode.NoContent: + var containerInfo = new ContainerInfo + { + Account=account, + Name = container, + Count = + long.Parse(client.GetHeaderValue("X-Container-Object-Count")), + Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")), + BlockHash = client.GetHeaderValue("X-Container-Block-Hash"), + BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")), + Last_Modified=client.LastModified + }; + return containerInfo; + case HttpStatusCode.NotFound: + return ContainerInfo.Empty; + default: + throw CreateWebException("GetContainerInfo", client.StatusCode); + } + } + } - switch(response.StatusCode) + public void CreateContainer(string account, string container) + { + if (String.IsNullOrWhiteSpace(container)) + throw new ArgumentNullException("container", "The container property can't be empty"); + Contract.EndContractBlock(); + + using (var client = new RestClient(_baseClient)) { - case HttpStatusCode.NoContent: - var keys = response.Headers.AllKeys.AsQueryable(); - var containerInfo = new ContainerInfo - { - Name = container, - Count =long.Parse(GetHeaderValue("X-Container-Object-Count", response, keys)), - Bytes =long.Parse(GetHeaderValue("X-Container-Bytes-Used", response, keys)) - }; - return containerInfo; - case HttpStatusCode.NotFound: - return ContainerInfo.Empty; - default: - throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode)); + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.PutWithRetry(container, 3); + var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK}; + if (!expectedCodes.Contains(client.StatusCode)) + throw CreateWebException("CreateContainer", client.StatusCode); } } - public void CreateContainer(string container) + public void DeleteContainer(string account, string container) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); + Contract.EndContractBlock(); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.DeleteWithRetry(container, 3); + var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent}; + if (!expectedCodes.Contains(client.StatusCode)) + throw CreateWebException("DeleteContainer", client.StatusCode); + } - var request = new RestRequest { Path = container, Method = WebMethod.Put }; - - var response = _client.Request(request); - - if (response.StatusCode!=HttpStatusCode.Created && response.StatusCode!=HttpStatusCode.Accepted ) - throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}", response.StatusCode)); } - public void DeleteContainer(string container) + /// + /// + /// + /// + /// + /// + /// + /// + /// This method should have no timeout or a very long one + //Asynchronously download the object specified by *objectName* in a specific *container* to + // a local file + public Task GetObject(string account, string container, string objectName, string fileName) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); + if (String.IsNullOrWhiteSpace(objectName)) + throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + Contract.EndContractBlock(); - var request = new RestRequest { Path = container , Method = WebMethod.Delete }; - var response = _client.Request(request); + try + { + //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient + //object to avoid concurrency errors. + // + //Download operations take a long time therefore they have no timeout. + var client = new RestClient(_baseClient) { Timeout = 0 }; + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + //The container and objectName are relative names. They are joined with the client's + //BaseAddress to create the object's absolute address + var builder = client.GetAddressBuilder(container, objectName); + var uri = builder.Uri; + + //Download progress is reported to the Trace log + Log.InfoFormat("[GET] START {0}", objectName); + client.DownloadProgressChanged += (sender, args) => + Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}", + fileName, args.ProgressPercentage, + args.BytesReceived, + args.TotalBytesToReceive); + + + //Start downloading the object asynchronously + var downloadTask = client.DownloadFileTask(uri, fileName); + + //Once the download completes + return downloadTask.ContinueWith(download => + { + //Delete the local client object + client.Dispose(); + //And report failure or completion + if (download.IsFaulted) + { + Log.ErrorFormat("[GET] FAIL for {0} with \r{1}", objectName, + download.Exception); + } + else + { + Log.InfoFormat("[GET] END {0}", objectName); + } + }); + } + catch (Exception exc) + { + Log.ErrorFormat("[GET] END {0} with {1}", objectName, exc); + throw; + } - if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent) - return; - else - throw new WebException(String.Format("DeleteContainer failed with unexpected status code {0}", response.StatusCode)); - } + } - public Stream GetObject(string container, string objectName) + public Task> PutHashMap(string account, string container, string objectName, TreeHash hash) { if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); + throw new ArgumentNullException("container"); if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName", "The objectName property can't be empty"); + throw new ArgumentNullException("objectName"); + if (hash==null) + throw new ArgumentNullException("hash"); + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + Contract.EndContractBlock(); + + + //Don't use a timeout because putting the hashmap may be a long process + var client = new RestClient(_baseClient) { Timeout = 0 }; + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + //The container and objectName are relative names. They are joined with the client's + //BaseAddress to create the object's absolute address + var builder = client.GetAddressBuilder(container, objectName); + builder.Query = "format=json&hashmap"; + var uri = builder.Uri; + + + //Send the tree hash as Json to the server + client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream"; + var uploadTask=client.UploadStringTask(uri, "PUT", hash.ToJson()); + + + return uploadTask.ContinueWith(t => + { + + var empty = (IList)new List(); + + + //The server will respond either with 201-created if all blocks were already on the server + if (client.StatusCode == HttpStatusCode.Created) + { + //in which case we return an empty hash list + return empty; + } + //or with a 409-conflict and return the list of missing parts + //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception + if (t.IsFaulted) + { + var ex = t.Exception.InnerException; + var we = ex as WebException; + var response = we.Response as HttpWebResponse; + if (response!=null && response.StatusCode==HttpStatusCode.Conflict) + { + //In case of 409 the missing parts will be in the response content + using (var stream = response.GetResponseStream()) + using(var reader=new StreamReader(stream)) + { + //We need to cleanup the content before returning it because it contains + //error content after the list of hashes + var hashes = new List(); + string line=null; + //All lines up to the first empty line are hashes + while(!String.IsNullOrWhiteSpace(line=reader.ReadLine())) + { + hashes.Add(line); + } + + return hashes; + } + } + else + //Any other status code is unexpected and the exception should be rethrown + throw ex; + + } + //Any other status code is unexpected but there was no exception. We can probably continue processing + else + { + Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription); + } + return empty; + }); + + } + + public Task GetBlock(string account, string container, Uri relativeUrl, long start, long? end) + { + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + if (String.IsNullOrWhiteSpace(container)) + throw new ArgumentNullException("container"); + if (relativeUrl== null) + throw new ArgumentNullException("relativeUrl"); + if (end.HasValue && end<0) + throw new ArgumentOutOfRangeException("end"); + if (start<0) + throw new ArgumentOutOfRangeException("start"); + Contract.EndContractBlock(); + + + //Don't use a timeout because putting the hashmap may be a long process + var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end}; + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + var builder = client.GetAddressBuilder(container, relativeUrl.ToString()); + var uri = builder.Uri; + + return client.DownloadDataTask(uri) + .ContinueWith(t=> + { + client.Dispose(); + return t.Result; + }); + } + + + public Task PostBlock(string account, string container, byte[] block, int offset, int count) + { + if (String.IsNullOrWhiteSpace(container)) + throw new ArgumentNullException("container"); + if (block == null) + throw new ArgumentNullException("block"); + if (offset < 0 || offset >= block.Length) + throw new ArgumentOutOfRangeException("offset"); + if (count < 0 || count > block.Length) + throw new ArgumentOutOfRangeException("count"); + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + Contract.EndContractBlock(); + + + //Don't use a timeout because putting the hashmap may be a long process + var client = new RestClient(_baseClient) { Timeout = 0 }; + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + var builder = client.GetAddressBuilder(container, ""); + //We are doing an update + builder.Query = "update"; + var uri = builder.Uri; + + client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream"; + + Log.InfoFormat("[BLOCK POST] START"); + + client.UploadProgressChanged += (sender, args) => + Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}", + args.ProgressPercentage, args.BytesSent, + args.TotalBytesToSend); + client.UploadFileCompleted += (sender, args) => + Log.InfoFormat("[BLOCK POST PROGRESS] Completed "); - var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Get }; - var response = _client.Request(request); - if (response.StatusCode == HttpStatusCode.NotFound) - throw new FileNotFoundException(); - if (response.StatusCode == HttpStatusCode.OK) + //Send the block + var uploadTask = client.UploadDataTask(uri, "POST", block) + .ContinueWith(upload => + { + client.Dispose(); + + if (upload.IsFaulted) + { + var exception = upload.Exception.InnerException; + Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exception); + throw exception; + } + + Log.InfoFormat("[BLOCK POST] END"); + }); + return uploadTask; + } + + + public Task GetHashMap(string account, string container, string objectName) + { + if (String.IsNullOrWhiteSpace(container)) + throw new ArgumentNullException("container"); + if (String.IsNullOrWhiteSpace(objectName)) + throw new ArgumentNullException("objectName"); + if (String.IsNullOrWhiteSpace(Token)) + throw new InvalidOperationException("Invalid Token"); + if (StorageUrl == null) + throw new InvalidOperationException("Invalid Storage Url"); + Contract.EndContractBlock(); + + try + { + //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient + //object to avoid concurrency errors. + // + //Download operations take a long time therefore they have no timeout. + //TODO: Do we really? this is a hashmap operation, not a download + var client = new RestClient(_baseClient) { Timeout = 0 }; + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + + //The container and objectName are relative names. They are joined with the client's + //BaseAddress to create the object's absolute address + var builder = client.GetAddressBuilder(container, objectName); + builder.Query = "format=json&hashmap"; + var uri = builder.Uri; + + //Start downloading the object asynchronously + var downloadTask = client.DownloadStringTask(uri); + + //Once the download completes + return downloadTask.ContinueWith(download => + { + //Delete the local client object + client.Dispose(); + //And report failure or completion + if (download.IsFaulted) + { + Log.ErrorFormat("[GET HASH] FAIL for {0} with \r{1}", objectName, + download.Exception); + throw download.Exception; + } + + //The server will return an empty string if the file is empty + var json = download.Result; + var treeHash = TreeHash.Parse(json); + Log.InfoFormat("[GET HASH] END {0}", objectName); + return treeHash; + }); + } + catch (Exception exc) { - return response.ContentStream; + Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc); + throw; } - else - throw new WebException(String.Format("GetObject failed with unexpected status code {0}", response.StatusCode)); + + + } - public void PutObject(string container, string objectName, string fileName) + + /// + /// + /// + /// + /// + /// + /// + /// Optional hash value for the file. If no hash is provided, the method calculates a new hash + /// >This method should have no timeout or a very long one + public Task PutObject(string account, string container, string objectName, string fileName, string hash = null) { if (String.IsNullOrWhiteSpace(container)) throw new ArgumentNullException("container", "The container property can't be empty"); @@ -352,30 +1004,70 @@ namespace Pithos.Network throw new ArgumentNullException("fileName", "The fileName property can't be empty"); if (!File.Exists(fileName)) throw new FileNotFoundException("The file does not exist",fileName); + Contract.EndContractBlock(); + + try + { + var client = new RestClient(_baseClient){Timeout=0}; + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + var builder = client.GetAddressBuilder(container, objectName); + var uri = builder.Uri; + + string etag = hash ?? CalculateHash(fileName); + + client.Headers.Add("Content-Type", "application/octet-stream"); + client.Headers.Add("ETag", etag); + + + Log.InfoFormat("[PUT] START {0}", objectName); + client.UploadProgressChanged += (sender, args) => + { + using (log4net.ThreadContext.Stacks["PUT"].Push("Progress")) + { + Log.InfoFormat("{0} {1}% {2} of {3}", fileName, args.ProgressPercentage, + args.BytesSent, args.TotalBytesToSend); + } + }; + + client.UploadFileCompleted += (sender, args) => + { + using (log4net.ThreadContext.Stacks["PUT"].Push("Progress")) + { + Log.InfoFormat("Completed {0}", fileName); + } + }; + return client.UploadFileTask(uri, "PUT", fileName) + .ContinueWith(upload=> + { + client.Dispose(); + + if (upload.IsFaulted) + { + var exc = upload.Exception.InnerException; + Log.ErrorFormat("[PUT] FAIL for {0} with \r{1}",objectName,exc); + throw exc; + } + else + Log.InfoFormat("[PUT] END {0}", objectName); + }); + } + catch (Exception exc) + { + Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc); + throw; + } - string url = container + "/" + objectName; - - var request = new RestRequest {Path=url,Method=WebMethod.Put}; - - - string hash = CalculateHash(fileName); - - request.AddPostContent(File.ReadAllBytes(fileName)); - request.AddHeader("Content-Type","application/octet-stream"); - request.AddHeader("ETag",hash); - var response=_client.Request(request); - - if (response.StatusCode == HttpStatusCode.Created) - return; - if (response.StatusCode == HttpStatusCode.LengthRequired) - throw new InvalidOperationException(); - else - throw new WebException(String.Format("GetObject failed with unexpected status code {0}", response.StatusCode)); } - + + private static string CalculateHash(string fileName) { + Contract.Requires(!String.IsNullOrWhiteSpace(fileName)); + Contract.EndContractBlock(); + string hash; using (var hasher = MD5.Create()) using(var stream=File.OpenRead(fileName)) @@ -387,59 +1079,74 @@ namespace Pithos.Network } return hash; } - - public void DeleteObject(string container, string objectName) - { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); - if (String.IsNullOrWhiteSpace(objectName)) - throw new ArgumentNullException("objectName", "The objectName property can't be empty"); - - var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Delete }; - var response = _client.Request(request); - - if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent) - return; - else - throw new WebException(String.Format("DeleteObject failed with unexpected status code {0}", response.StatusCode)); - - } - - public void MoveObject(string container, string oldObjectName, string newObjectName) + + public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName) { - if (String.IsNullOrWhiteSpace(container)) - throw new ArgumentNullException("container", "The container property can't be empty"); + if (String.IsNullOrWhiteSpace(sourceContainer)) + throw new ArgumentNullException("sourceContainer", "The container property can't be empty"); if (String.IsNullOrWhiteSpace(oldObjectName)) throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty"); + if (String.IsNullOrWhiteSpace(targetContainer)) + throw new ArgumentNullException("targetContainer", "The container property can't be empty"); if (String.IsNullOrWhiteSpace(newObjectName)) throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty"); + Contract.EndContractBlock(); - var request = new RestRequest { Path = container + "/" + newObjectName, Method = WebMethod.Put }; - request.AddHeader("X-Copy-From",String.Format("/{0}/{1}",container,oldObjectName)); - request.AddPostContent(new byte[]{}); - var response = _client.Request(request); + var targetUrl = targetContainer + "/" + newObjectName; + var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName); - if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.NoContent || response.StatusCode==HttpStatusCode.Created) + using (var client = new RestClient(_baseClient)) { - this.DeleteObject(container,oldObjectName); - } - else - throw new WebException(String.Format("MoveObject failed with unexpected status code {0}", response.StatusCode)); + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Headers.Add("X-Move-From", sourceUrl); + client.PutWithRetry(targetUrl, 3); + + var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created}; + if (!expectedCodes.Contains(client.StatusCode)) + throw CreateWebException("MoveObject", client.StatusCode); + } } - private string GetHeaderValue(string headerName, RestResponse response, IQueryable keys) - { - if (keys.Any(key => key == headerName)) - return response.Headers[headerName]; - else - throw new WebException(String.Format("The {0} header is missing",headerName)); + public void DeleteObject(string account, string sourceContainer, string objectName) + { + if (String.IsNullOrWhiteSpace(sourceContainer)) + throw new ArgumentNullException("sourceContainer", "The container property can't be empty"); + if (String.IsNullOrWhiteSpace(objectName)) + throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty"); + Contract.EndContractBlock(); + + var targetUrl = FolderConstants.TrashContainer + "/" + objectName; + var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName); + + using (var client = new RestClient(_baseClient)) + { + if (!String.IsNullOrWhiteSpace(account)) + client.BaseAddress = GetAccountUrl(account); + + client.Headers.Add("X-Move-From", sourceUrl); + client.AllowedStatusCodes.Add(HttpStatusCode.NotFound); + client.PutWithRetry(targetUrl, 3); + + var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound}; + if (!expectedCodes.Contains(client.StatusCode)) + throw CreateWebException("DeleteObject", client.StatusCode); + } } - private static void ThrowIfNotStatusOK(RestResponse response, string message) + + private static WebException CreateWebException(string operation, HttpStatusCode statusCode) { - int status = (int)response.StatusCode; - if (status < 200 || status >= 300) - throw new WebException(String.Format("{0} with code {1}",message, status)); + return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode)); } + + + } + + public class ShareAccountInfo + { + public DateTime? last_modified { get; set; } + public string name { get; set; } } }