-using System;
+#region
+/* -----------------------------------------------------------------------
+ * <copyright file="CloudFilesClient.cs" company="GRNet">
+ *
+ * Copyright 2011-2012 GRNET S.A. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and
+ * documentation are those of the authors and should not be
+ * interpreted as representing official policies, either expressed
+ * or implied, of GRNET S.A.
+ * </copyright>
+ * -----------------------------------------------------------------------
+ */
+#endregion
+
+// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
+//
+// The class provides methods to upload/download files, delete files, manage containers
+
+
+using System;
using System.Collections.Generic;
+using System.Collections.Specialized;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.Diagnostics.Contracts;
using System.IO;
using System.Linq;
using System.Net;
+using System.Reflection;
using System.Security.Cryptography;
using System.Text;
+using System.Threading;
using System.Threading.Tasks;
-using Hammock;
-using Hammock.Caching;
-using Hammock.Retries;
-using Hammock.Serialization;
-using Hammock.Tasks;
-using Hammock.Web;
using Newtonsoft.Json;
using Pithos.Interfaces;
+using log4net;
namespace Pithos.Network
{
[Export(typeof(ICloudClient))]
public class CloudFilesClient:ICloudClient
{
- string _rackSpaceAuthUrl = "https://auth.api.rackspacecloud.com";
- private string _pithosAuthUrl = "http://pithos.dev.grnet.gr";
+ private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
- private RestClient _client;
- private readonly TimeSpan _shortTimeout = TimeSpan.FromSeconds(10);
- private readonly int _retries = 5;
- private RetryPolicy _retryPolicy;
- public string ApiKey { get; set; }
+ //CloudFilesClient uses *_baseClient* internally to communicate with the server
+ //RestClient provides a REST-friendly interface over the standard WebClient.
+ private RestClient _baseClient;
+
+
+ //During authentication the client provides a UserName
public string UserName { get; set; }
+
+ //and and ApiKey to the server
+ public string ApiKey { get; set; }
+
+ //And receives an authentication Token. This token must be provided in ALL other operations,
+ //in the X-Auth-Token header
+ private string _token;
+ public string Token
+ {
+ get { return _token; }
+ set
+ {
+ _token = value;
+ _baseClient.Headers["X-Auth-Token"] = value;
+ }
+ }
+
+ //The client also receives a StorageUrl after authentication. All subsequent operations must
+ //use this url
public Uri StorageUrl { get; set; }
- public string Token { get; set; }
- public Uri Proxy { get; set; }
- public double DownloadPercentLimit { get; set; }
- public double UploadPercentLimit { get; set; }
-
- public string AuthUrl
+
+ protected Uri RootAddressUri { get; set; }
+
+ /* private WebProxy _proxy;
+ public WebProxy Proxy
{
- get { return UsePithos ? _pithosAuthUrl : _rackSpaceAuthUrl; }
+ get { return _proxy; }
+ set
+ {
+ _proxy = value;
+ if (_baseClient != null)
+ _baseClient.Proxy = value;
+ }
}
+*/
+
+ /* private Uri _proxy;
+ public Uri Proxy
+ {
+ get { return _proxy; }
+ set
+ {
+ _proxy = value;
+ if (_baseClient != null)
+ _baseClient.Proxy = new WebProxy(value);
+ }
+ }*/
+
+ public double DownloadPercentLimit { get; set; }
+ public double UploadPercentLimit { get; set; }
+
+ public string AuthenticationUrl { get; set; }
+
public string VersionPath
{
public bool UsePithos { get; set; }
- public void Authenticate(string userName,string apiKey)
- {
- Trace.TraceInformation("[AUTHENTICATE] Start for {0}",userName);
- if (String.IsNullOrWhiteSpace(userName))
- throw new ArgumentNullException("userName","The userName property can't be empty");
- if (String.IsNullOrWhiteSpace(apiKey))
- throw new ArgumentNullException("apiKey", "The apiKey property can't be empty");
-
+
+ public CloudFilesClient(string userName, string apiKey)
+ {
UserName = userName;
ApiKey = apiKey;
+ }
- string authUrl = UsePithos ? String.Format("{0}/{1}/{2}", AuthUrl, VersionPath,UserName)
- : String.Format("{0}/{1}", AuthUrl, VersionPath);
-
- var proxy = Proxy != null ? Proxy.ToString():null;
-
- var authClient = new RestClient{Path=authUrl,Proxy=proxy};
-
- authClient.AddHeader("X-Auth-User", UserName);
- authClient.AddHeader("X-Auth-Key", ApiKey);
-
- var response=authClient.Request();
-
- ThrowIfNotStatusOK(response, "Authentication failed");
-
- var keys = response.Headers.AllKeys.AsQueryable();
+ public CloudFilesClient(AccountInfo accountInfo)
+ {
+ if (accountInfo==null)
+ throw new ArgumentNullException("accountInfo");
+ Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
+ Contract.Ensures(StorageUrl != null);
+ Contract.Ensures(_baseClient != null);
+ Contract.Ensures(RootAddressUri != null);
+ Contract.EndContractBlock();
+
+ _baseClient = new RestClient
+ {
+ BaseAddress = accountInfo.StorageUri.ToString(),
+ Timeout = 10000,
+ Retries = 3,
+ };
+ StorageUrl = accountInfo.StorageUri;
+ Token = accountInfo.Token;
+ UserName = accountInfo.UserName;
+
+ //Get the root address (StorageUrl without the account)
+ var storageUrl = StorageUrl.AbsoluteUri;
+ var usernameIndex = storageUrl.LastIndexOf(UserName);
+ var rootUrl = storageUrl.Substring(0, usernameIndex);
+ RootAddressUri = new Uri(rootUrl);
+ }
- string storageUrl =UsePithos?
- String.Format("{0}/{1}/{2}",AuthUrl,VersionPath,UserName)
- :GetHeaderValue("X-Storage-Url", response, keys);
-
- if (String.IsNullOrWhiteSpace(storageUrl))
- throw new InvalidOperationException("Failed to obtain storage url");
- StorageUrl = new Uri(storageUrl);
- if (!UsePithos)
- {
- var token = GetHeaderValue("X-Auth-Token", response, keys);
+ public AccountInfo Authenticate()
+ {
+ if (String.IsNullOrWhiteSpace(UserName))
+ throw new InvalidOperationException("UserName is empty");
+ if (String.IsNullOrWhiteSpace(ApiKey))
+ throw new InvalidOperationException("ApiKey is empty");
+ if (String.IsNullOrWhiteSpace(AuthenticationUrl))
+ throw new InvalidOperationException("AuthenticationUrl is empty");
+ Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
+ Contract.Ensures(StorageUrl != null);
+ Contract.Ensures(_baseClient != null);
+ Contract.Ensures(RootAddressUri != null);
+ Contract.EndContractBlock();
+
+
+ Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
+
+ var groups = new List<Group>();
+
+ using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
+ {
+ /* if (Proxy != null)
+ authClient.Proxy = Proxy;*/
+
+ Contract.Assume(authClient.Headers!=null);
+
+ authClient.Headers.Add("X-Auth-User", UserName);
+ authClient.Headers.Add("X-Auth-Key", ApiKey);
+
+ authClient.DownloadStringWithRetry(VersionPath, 3);
+
+ authClient.AssertStatusOK("Authentication failed");
+
+ var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
+ if (String.IsNullOrWhiteSpace(storageUrl))
+ throw new InvalidOperationException("Failed to obtain storage url");
+
+ _baseClient = new RestClient
+ {
+ BaseAddress = storageUrl,
+ Timeout = 10000,
+ Retries = 3,
+ //Proxy=Proxy
+ };
+
+ StorageUrl = new Uri(storageUrl);
+
+ //Get the root address (StorageUrl without the account)
+ var usernameIndex=storageUrl.LastIndexOf(UserName);
+ var rootUrl = storageUrl.Substring(0, usernameIndex);
+ RootAddressUri = new Uri(rootUrl);
+
+ var token = authClient.GetHeaderValue("X-Auth-Token");
if (String.IsNullOrWhiteSpace(token))
throw new InvalidOperationException("Failed to obtain token url");
Token = token;
+
+ /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
+ groups = (from key in keys
+ where key.StartsWith("X-Account-Group-")
+ let name = key.Substring(16)
+ select new Group(name, authClient.ResponseHeaders[key]))
+ .ToList();
+
+*/
}
- else
- Token = "0000";
- _retryPolicy = new RetryPolicy { RetryCount = _retries };
- _retryPolicy.RetryConditions.Add(new TimeoutRetryCondition());
+ Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
+ Debug.Assert(_baseClient!=null);
- _client = new RestClient { Authority = StorageUrl.AbsoluteUri, Path = UserName, Proxy = proxy };
- _client.FileProgress += OnFileProgress;
-
- _client.AddHeader("X-Auth-Token", Token);
- if (UsePithos)
+ return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};
+
+ }
+
+
+
+ public IList<ContainerInfo> ListContainers(string account)
+ {
+ using (var client = new RestClient(_baseClient))
{
- _client.AddHeader("X-Auth-User", UserName);
- _client.AddHeader("X-Auth-Key",ApiKey);
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Parameters.Clear();
+ client.Parameters.Add("format", "json");
+ var content = client.DownloadStringWithRetry("", 3);
+ client.AssertStatusOK("List Containers failed");
+
+ if (client.StatusCode == HttpStatusCode.NoContent)
+ return new List<ContainerInfo>();
+ var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
+
+ foreach (var info in infos)
+ {
+ info.Account = account;
+ }
+ return infos;
}
- Trace.TraceInformation("[AUTHENTICATE] End for {0}", userName);
}
- private void OnFileProgress(object sender, FileProgressEventArgs e)
+ private string GetAccountUrl(string account)
{
- Trace.TraceInformation("[PROGRESS] {0} {1:p} {2} of {3}",e.FileName,(double)e.BytesWritten/e.TotalBytes, e.BytesWritten,e.TotalBytes);
+ return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
}
- public IList<ContainerInfo> ListContainers()
- {
- //Workaround for Hammock quirk: Hammock always
- //appends a / unless a Path is specified.
-
- //Create a request with a complete path
- var request = new RestRequest { Path = StorageUrl.ToString(), RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
- request.AddParameter("format","json");
- //Create a client clone
- var client = new RestClient{Proxy=Proxy.ToString()};
- foreach (var header in _client.GetAllHeaders())
+ public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
+ {
+ using (ThreadContext.Stacks["Share"].Push("List Accounts"))
{
- client.AddHeader(header.Name,header.Value);
- }
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
- var response = client.Request(request);
+ using (var client = new RestClient(_baseClient))
+ {
+ client.Parameters.Clear();
+ client.Parameters.Add("format", "json");
+ client.IfModifiedSince = since;
- if (response.StatusCode == HttpStatusCode.NoContent)
- return new List<ContainerInfo>();
+ //Extract the username from the base address
+ client.BaseAddress = RootAddressUri.AbsoluteUri;
- ThrowIfNotStatusOK(response, "List Containers failed");
+ var content = client.DownloadStringWithRetry(@"", 3);
+ client.AssertStatusOK("ListSharingAccounts failed");
- var infos=JsonConvert.DeserializeObject<IList<ContainerInfo>>(response.Content);
-
- return infos;
+ //If the result is empty, return an empty list,
+ var infos = String.IsNullOrWhiteSpace(content)
+ ? new List<ShareAccountInfo>()
+ //Otherwise deserialize the account list into a list of ShareAccountInfos
+ : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
+
+ Log.DebugFormat("END");
+ return infos;
+ }
+ }
}
- public IList<ObjectInfo> ListObjects(string container)
+ //Request listing of all objects in a container modified since a specific time.
+ //If the *since* value is missing, return all objects
+ public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
{
+
+ using (ThreadContext.Stacks["Share"].Push("List Objects"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+ //'since' is not used here because we need to have ListObjects return a NoChange result
+ //for all shared accounts,containers
+ var accounts = ListSharingAccounts();
+ var items = from account in accounts
+ let containers = ListContainers(account.name)
+ from container in containers
+ select ListObjects(account.name, container.Name,since);
+ var objects=items.SelectMany(r=> r).ToList();
+/*
+ var objects = new List<ObjectInfo>();
+ foreach (var containerObjects in items)
+ {
+ objects.AddRange(containerObjects);
+ }
+*/
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ return objects;
+ }
+ }
+
+ public void SetTags(ObjectInfo target,IDictionary<string,string> tags)
+ {
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("The Token is not set");
+ if (StorageUrl == null)
+ throw new InvalidOperationException("The StorageUrl is not set");
+ if (target == null)
+ throw new ArgumentNullException("target");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Share"].Push("Share Object"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+ using (var client = new RestClient(_baseClient))
+ {
+
+ client.BaseAddress = GetAccountUrl(target.Account);
+
+ client.Parameters.Clear();
+ client.Parameters.Add("update", "");
+
+ foreach (var tag in tags)
+ {
+ var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
+ client.Headers.Add(headerTag, tag.Value);
+ }
+
+ client.DownloadStringWithRetry(target.Container, 3);
+
+
+ client.AssertStatusOK("SetTags failed");
+ //If the status is NOT ACCEPTED we have a problem
+ if (client.StatusCode != HttpStatusCode.Accepted)
+ {
+ Log.Error("Failed to set tags");
+ throw new Exception("Failed to set tags");
+ }
+
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ }
+ }
+
+
+ }
+
+ public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
+ {
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("The Token is not set");
+ if (StorageUrl==null)
+ throw new InvalidOperationException("The StorageUrl is not set");
if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
+ throw new ArgumentNullException("container");
+ if (String.IsNullOrWhiteSpace(objectName))
+ throw new ArgumentNullException("objectName");
+ if (String.IsNullOrWhiteSpace(account))
+ throw new ArgumentNullException("account");
+ if (String.IsNullOrWhiteSpace(shareTo))
+ throw new ArgumentNullException("shareTo");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Share"].Push("Share Object"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+ using (var client = new RestClient(_baseClient))
+ {
- Trace.TraceInformation("[START] ListObjects");
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Parameters.Clear();
+ client.Parameters.Add("format", "json");
+
+ string permission = "";
+ if (write)
+ permission = String.Format("write={0}", shareTo);
+ else if (read)
+ permission = String.Format("read={0}", shareTo);
+ client.Headers.Add("X-Object-Sharing", permission);
+
+ var content = client.DownloadStringWithRetry(container, 3);
+
+ client.AssertStatusOK("ShareObject failed");
+
+ //If the result is empty, return an empty list,
+ var infos = String.IsNullOrWhiteSpace(content)
+ ? new List<ObjectInfo>()
+ //Otherwise deserialize the object list into a list of ObjectInfos
+ : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
+
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ }
+ }
- var request = new RestRequest { Path = container, RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) };
- request.AddParameter("format", "json");
- var response = _client.Request(request);
-
- var infos = InfosFromContent(response);
- Trace.TraceInformation("[END] ListObjects");
- return infos;
}
+ public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
+ {
+ if (accountInfo==null)
+ throw new ArgumentNullException("accountInfo");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Account"].Push("GetPolicies"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
+ client.BaseAddress = GetAccountUrl(accountInfo.UserName);
+
+ client.Parameters.Clear();
+ client.Parameters.Add("format", "json");
+ client.Head(String.Empty, 3);
+
+ var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
+ var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
+ long quota, bytes;
+ if (long.TryParse(quotaValue, out quota))
+ accountInfo.Quota = quota;
+ if (long.TryParse(bytesValue, out bytes))
+ accountInfo.BytesUsed = bytes;
+
+ return accountInfo;
- public IList<ObjectInfo> ListObjects(string container,string folder)
+ }
+
+ }
+ }
+
+ public void UpdateMetadata(ObjectInfo objectInfo)
{
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
+ if (objectInfo == null)
+ throw new ArgumentNullException("objectInfo");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
- Trace.TraceInformation("[START] ListObjects");
- var request = new RestRequest { Path = container,RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) };
- request.AddParameter("format", "json");
- request.AddParameter("path", folder);
- var response = _client.Request(request);
-
- var infos = InfosFromContent(response);
+ using(var client=new RestClient(_baseClient))
+ {
+
+ client.BaseAddress = GetAccountUrl(objectInfo.Account);
+
+ client.Parameters.Clear();
+
+
+ //Set Tags
+ foreach (var tag in objectInfo.Tags)
+ {
+ var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
+ client.Headers.Add(headerTag, tag.Value);
+ }
+
+ //Set Permissions
+
+ var permissions=objectInfo.GetPermissionString();
+ client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);
+
+ client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
+ client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
+ client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
+ var isPublic = objectInfo.IsPublic.ToString().ToLower();
+ client.Headers.Add("X-Object-Public", isPublic);
+
+
+ /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);
+ uriBuilder.Query = "update=";
+ var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/
+ var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);
+ client.PostWithRetry(address,"application/xml");
+
+ //client.UploadValues(uri,new NameValueCollection());
+
+
+ client.AssertStatusOK("UpdateMetadata failed");
+ //If the status is NOT ACCEPTED or OK we have a problem
+ if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
+ {
+ Log.Error("Failed to update metadata");
+ throw new Exception("Failed to update metadata");
+ }
+
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ }
+ }
- Trace.TraceInformation("[END] ListObjects");
- return infos;
}
- private static IList<ObjectInfo> InfosFromContent(RestResponse response)
+ public void UpdateMetadata(ContainerInfo containerInfo)
{
- if (response.TimedOut)
- return new List<ObjectInfo>();
+ if (containerInfo == null)
+ throw new ArgumentNullException("containerInfo");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+
+ using(var client=new RestClient(_baseClient))
+ {
+
+ client.BaseAddress = GetAccountUrl(containerInfo.Account);
+
+ client.Parameters.Clear();
+
+
+ //Set Tags
+ foreach (var tag in containerInfo.Tags)
+ {
+ var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
+ client.Headers.Add(headerTag, tag.Value);
+ }
+
+
+ //Set Policies
+ foreach (var policy in containerInfo.Policies)
+ {
+ var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
+ client.Headers.Add(headerPolicy, policy.Value);
+ }
+
+
+ var uriBuilder = client.GetAddressBuilder(containerInfo.Name,"");
+ var uri = uriBuilder.Uri;
+
+ client.UploadValues(uri,new NameValueCollection());
+
+
+ client.AssertStatusOK("UpdateMetadata failed");
+ //If the status is NOT ACCEPTED or OK we have a problem
+ if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
+ {
+ Log.Error("Failed to update metadata");
+ throw new Exception("Failed to update metadata");
+ }
- if (response.StatusCode == 0)
- return new List<ObjectInfo>();
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ }
+ }
+
+ }
- if (response.StatusCode == HttpStatusCode.NoContent)
- return new List<ObjectInfo>();
+ public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
+ {
+ if (String.IsNullOrWhiteSpace(container))
+ throw new ArgumentNullException("container");
+ Contract.EndContractBlock();
- var statusCode = (int)response.StatusCode;
- if (statusCode < 200 || statusCode >= 300)
+ using (ThreadContext.Stacks["Objects"].Push("List"))
{
- Trace.TraceWarning("ListObjects failed with code {1} - {2}", response.StatusCode, response.StatusDescription);
- return new List<ObjectInfo>();
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Parameters.Clear();
+ client.Parameters.Add("format", "json");
+ client.IfModifiedSince = since;
+ var content = client.DownloadStringWithRetry(container, 3);
+
+ client.AssertStatusOK("ListObjects failed");
+
+ if (client.StatusCode==HttpStatusCode.NotModified)
+ return new[]{new NoModificationInfo(account,container)};
+ //If the result is empty, return an empty list,
+ var infos = String.IsNullOrWhiteSpace(content)
+ ? new List<ObjectInfo>()
+ //Otherwise deserialize the object list into a list of ObjectInfos
+ : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
+
+ foreach (var info in infos)
+ {
+ info.Container = container;
+ info.Account = account;
+ info.StorageUri = this.StorageUrl;
+ }
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ return infos;
+ }
}
+ }
+
+ public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
+ {
+ if (String.IsNullOrWhiteSpace(container))
+ throw new ArgumentNullException("container");
+/*
+ if (String.IsNullOrWhiteSpace(folder))
+ throw new ArgumentNullException("folder");
+*/
+ Contract.EndContractBlock();
- var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(response.Content);
- return infos;
+ using (ThreadContext.Stacks["Objects"].Push("List"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Parameters.Clear();
+ client.Parameters.Add("format", "json");
+ client.Parameters.Add("path", folder);
+ client.IfModifiedSince = since;
+ var content = client.DownloadStringWithRetry(container, 3);
+ client.AssertStatusOK("ListObjects failed");
+
+ if (client.StatusCode==HttpStatusCode.NotModified)
+ return new[]{new NoModificationInfo(account,container,folder)};
+
+ var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
+ foreach (var info in infos)
+ {
+ info.Account = account;
+ if (info.Container == null)
+ info.Container = container;
+ info.StorageUri = this.StorageUrl;
+ }
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ return infos;
+ }
+ }
}
- public bool ContainerExists(string container)
+
+ public bool ContainerExists(string account, string container)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
+ Contract.EndContractBlock();
- var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
- var response = _client.Request(request);
-
- switch(response.StatusCode)
+ using (ThreadContext.Stacks["Containters"].Push("Exists"))
{
- case HttpStatusCode.NoContent:
- return true;
- case HttpStatusCode.NotFound:
- return false;
- default:
- throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode));
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Parameters.Clear();
+ client.Head(container, 3);
+
+ bool result;
+ switch (client.StatusCode)
+ {
+ case HttpStatusCode.OK:
+ case HttpStatusCode.NoContent:
+ result=true;
+ break;
+ case HttpStatusCode.NotFound:
+ result=false;
+ break;
+ default:
+ throw CreateWebException("ContainerExists", client.StatusCode);
+ }
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+
+ return result;
+ }
+
}
}
- public bool ObjectExists(string container,string objectName)
+ public bool ObjectExists(string account, string container, string objectName)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(objectName))
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
+ Contract.EndContractBlock();
-
- var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head,RetryPolicy = _retryPolicy, Timeout = _shortTimeout };
- var response = _client.Request(request);
-
- switch (response.StatusCode)
+ using (var client = new RestClient(_baseClient))
{
- case HttpStatusCode.OK:
- case HttpStatusCode.NoContent:
- return true;
- case HttpStatusCode.NotFound:
- return false;
- default:
- throw new WebException(String.Format("ObjectExists failed with unexpected status code {0}", response.StatusCode));
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Parameters.Clear();
+ client.Head(container + "/" + objectName, 3);
+
+ switch (client.StatusCode)
+ {
+ case HttpStatusCode.OK:
+ case HttpStatusCode.NoContent:
+ return true;
+ case HttpStatusCode.NotFound:
+ return false;
+ default:
+ throw CreateWebException("ObjectExists", client.StatusCode);
+ }
}
-
+
}
- public ObjectInfo GetObjectInfo(string container, string objectName)
+ public ObjectInfo GetObjectInfo(string account, string container, string objectName)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(objectName))
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
+ Contract.EndContractBlock();
+ using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
+ {
- var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
- var response = _client.Request(request);
-
- if (response.TimedOut)
- return ObjectInfo.Empty;
-
- switch (response.StatusCode)
- {
- case HttpStatusCode.OK:
- case HttpStatusCode.NoContent:
- var keys = response.Headers.AllKeys.AsQueryable();
- return new ObjectInfo
- {
- Name = objectName,
- Bytes = long.Parse(GetHeaderValue("Content-Length", response, keys)),
- Hash = GetHeaderValue("ETag", response, keys),
- Content_Type = GetHeaderValue("Content-Type", response, keys)
- };
- case HttpStatusCode.NotFound:
- return ObjectInfo.Empty;
- default:
- if (request.RetryState.RepeatCount > 0)
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+ try
{
- Trace.TraceWarning("[RETRY FAIL] GetObjectInfo for {0} failed after {1} retries",
- objectName, request.RetryState.RepeatCount);
+ client.Parameters.Clear();
+
+ client.Head(container + "/" + objectName, 3);
+
+ if (client.TimedOut)
+ return ObjectInfo.Empty;
+
+ switch (client.StatusCode)
+ {
+ case HttpStatusCode.OK:
+ case HttpStatusCode.NoContent:
+ var keys = client.ResponseHeaders.AllKeys.AsQueryable();
+ var tags = client.GetMeta("X-Object-Meta-");
+ var extensions = (from key in keys
+ where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
+ select new {Name = key, Value = client.ResponseHeaders[key]})
+ .ToDictionary(t => t.Name, t => t.Value);
+
+ var permissions=client.GetHeaderValue("X-Object-Sharing", true);
+
+
+ var info = new ObjectInfo
+ {
+ Account = account,
+ Container = container,
+ Name = objectName,
+ ETag = client.GetHeaderValue("ETag"),
+ X_Object_Hash = client.GetHeaderValue("X-Object-Hash"),
+ Content_Type = client.GetHeaderValue("Content-Type"),
+ Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),
+ Tags = tags,
+ Last_Modified = client.LastModified,
+ Extensions = extensions,
+ ContentEncoding=client.GetHeaderValue("Content-Encoding",true),
+ ContendDisposition = client.GetHeaderValue("Content-Disposition",true),
+ Manifest=client.GetHeaderValue("X-Object-Manifest",true),
+ PublicUrl=client.GetHeaderValue("X-Object-Public",true),
+ StorageUri=this.StorageUrl,
+ };
+ info.SetPermissions(permissions);
+ return info;
+ case HttpStatusCode.NotFound:
+ return ObjectInfo.Empty;
+ default:
+ throw new WebException(
+ String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
+ objectName, client.StatusCode));
+ }
+
+ }
+ catch (RetryException)
+ {
+ Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);
return ObjectInfo.Empty;
}
- if (response.InnerException != null)
- throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode), response.InnerException);
- throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode));
+ catch (WebException e)
+ {
+ Log.Error(
+ String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
+ objectName, client.StatusCode), e);
+ throw;
+ }
+ }
}
+
}
- public void CreateFolder(string container, string folder)
+ public void CreateFolder(string account, string container, string folder)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(folder))
throw new ArgumentNullException("folder", "The folder property can't be empty");
+ Contract.EndContractBlock();
var folderUrl=String.Format("{0}/{1}",container,folder);
- var request = new RestRequest { Path = folderUrl, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
- request.AddHeader("Content-Type", @"application/directory");
- request.AddHeader("Content-Length", "0");
-
- var response = _client.Request(request);
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
- if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)
- throw new WebException(String.Format("CreateFolder failed with unexpected status code {0}", response.StatusCode));
+ client.Parameters.Clear();
+ client.Headers.Add("Content-Type", @"application/directory");
+ client.Headers.Add("Content-Length", "0");
+ client.PutWithRetry(folderUrl, 3);
+ if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
+ throw CreateWebException("CreateFolder", client.StatusCode);
+ }
}
- public ContainerInfo GetContainerInfo(string container)
+
+
+ public ContainerInfo GetContainerInfo(string account, string container)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
+ Contract.EndContractBlock();
- var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
- var response = _client.Request(request);
-
- switch(response.StatusCode)
+ using (var client = new RestClient(_baseClient))
{
- case HttpStatusCode.NoContent:
- var keys = response.Headers.AllKeys.AsQueryable();
- var containerInfo = new ContainerInfo
- {
- Name = container,
- Count =long.Parse(GetHeaderValue("X-Container-Object-Count", response, keys)),
- Bytes =long.Parse(GetHeaderValue("X-Container-Bytes-Used", response, keys))
- };
- return containerInfo;
- case HttpStatusCode.NotFound:
- return ContainerInfo.Empty;
- default:
- throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode));
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Head(container);
+ switch (client.StatusCode)
+ {
+ case HttpStatusCode.OK:
+ case HttpStatusCode.NoContent:
+ var tags = client.GetMeta("X-Container-Meta-");
+ var policies = client.GetMeta("X-Container-Policy-");
+
+ var containerInfo = new ContainerInfo
+ {
+ Account=account,
+ Name = container,
+ StorageUrl=this.StorageUrl.ToString(),
+ Count =
+ long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
+ Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
+ BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
+ BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),
+ Last_Modified=client.LastModified,
+ Tags=tags,
+ Policies=policies
+ };
+
+
+ return containerInfo;
+ case HttpStatusCode.NotFound:
+ return ContainerInfo.Empty;
+ default:
+ throw CreateWebException("GetContainerInfo", client.StatusCode);
+ }
}
}
- public void CreateContainer(string container)
+ public void CreateContainer(string account, string container)
{
+ if (String.IsNullOrWhiteSpace(account))
+ throw new ArgumentNullException("account");
if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
+ throw new ArgumentNullException("container");
+ Contract.EndContractBlock();
- var request = new RestRequest { Path = container, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
-
- var response = _client.Request(request);
-
- if (response.StatusCode!=HttpStatusCode.Created && response.StatusCode!=HttpStatusCode.Accepted )
- throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}", response.StatusCode));
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.PutWithRetry(container, 3);
+ var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
+ if (!expectedCodes.Contains(client.StatusCode))
+ throw CreateWebException("CreateContainer", client.StatusCode);
+ }
}
- public void DeleteContainer(string container)
+ public void DeleteContainer(string account, string container)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
+ Contract.EndContractBlock();
- var request = new RestRequest { Path = container, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
- var response = _client.Request(request);
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
- if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent)
- return;
- else
- throw new WebException(String.Format("DeleteContainer failed with unexpected status code {0}", response.StatusCode));
+ client.DeleteWithRetry(container, 3);
+ var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
+ if (!expectedCodes.Contains(client.StatusCode))
+ throw CreateWebException("DeleteContainer", client.StatusCode);
+ }
}
/// <summary>
///
/// </summary>
+ /// <param name="account"></param>
/// <param name="container"></param>
/// <param name="objectName"></param>
+ /// <param name="fileName"></param>
/// <returns></returns>
- /// <remarks>>This method should have no timeout or a very long one</remarks>
- public Stream GetObject(string container, string objectName)
+ /// <remarks>This method should have no timeout or a very long one</remarks>
+ //Asynchronously download the object specified by *objectName* in a specific *container* to
+ // a local file
+ public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The objectName property can't be empty");
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");
+ Contract.EndContractBlock();
- var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Get };
-/*
- if (DownloadPercentLimit > 0)
- request.TaskOptions = new TaskOptions<int> { RateLimitPercent = DownloadPercentLimit };
-*/
-
- var response = _client.Request(request);
-
- if (response.StatusCode == HttpStatusCode.NotFound)
- throw new FileNotFoundException();
- if (response.StatusCode == HttpStatusCode.OK)
+ try
+ {
+ //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
+ //object to avoid concurrency errors.
+ //
+ //Download operations take a long time therefore they have no timeout.
+ using(var client = new RestClient(_baseClient) { Timeout = 0 })
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ //The container and objectName are relative names. They are joined with the client's
+ //BaseAddress to create the object's absolute address
+ var builder = client.GetAddressBuilder(container, objectName);
+ var uri = builder.Uri;
+
+ //Download progress is reported to the Trace log
+ Log.InfoFormat("[GET] START {0}", objectName);
+ client.DownloadProgressChanged += (sender, args) =>
+ Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
+ fileName, args.ProgressPercentage,
+ args.BytesReceived,
+ args.TotalBytesToReceive);
+
+
+ //Start downloading the object asynchronously
+ await client.DownloadFileTaskAsync(uri, fileName,cancellationToken);
+
+ //Once the download completes
+ //Delete the local client object
+ }
+ //And report failure or completion
+ }
+ catch (Exception exc)
+ {
+ Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);
+ throw;
+ }
+
+ Log.InfoFormat("[GET] END {0}", objectName);
+
+
+ }
+
+ public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
+ {
+ if (String.IsNullOrWhiteSpace(container))
+ throw new ArgumentNullException("container");
+ if (String.IsNullOrWhiteSpace(objectName))
+ throw new ArgumentNullException("objectName");
+ if (hash==null)
+ throw new ArgumentNullException("hash");
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("Invalid Token");
+ if (StorageUrl == null)
+ throw new InvalidOperationException("Invalid Storage Url");
+ Contract.EndContractBlock();
+
+
+ //Don't use a timeout because putting the hashmap may be a long process
+ var client = new RestClient(_baseClient) { Timeout = 0 };
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ //The container and objectName are relative names. They are joined with the client's
+ //BaseAddress to create the object's absolute address
+ var builder = client.GetAddressBuilder(container, objectName);
+ builder.Query = "format=json&hashmap";
+ var uri = builder.Uri;
+
+
+ //Send the tree hash as Json to the server
+ client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
+ var jsonHash = hash.ToJson();
+ var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);
+ if (Log.IsDebugEnabled)
+ Log.DebugFormat("Hashes:\r\n{0}", jsonHash);
+ return uploadTask.ContinueWith(t =>
+ {
+
+ var empty = (IList<string>)new List<string>();
+
+
+ //The server will respond either with 201-created if all blocks were already on the server
+ if (client.StatusCode == HttpStatusCode.Created)
+ {
+ //in which case we return an empty hash list
+ return empty;
+ }
+ //or with a 409-conflict and return the list of missing parts
+ //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
+ if (t.IsFaulted)
+ {
+ var ex = t.Exception.InnerException;
+ var we = ex as WebException;
+ var response = we.Response as HttpWebResponse;
+ if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
+ {
+ //In case of 409 the missing parts will be in the response content
+ using (var stream = response.GetResponseStream())
+ using(var reader=stream.GetLoggedReader(Log))
+ {
+ //We used to have to cleanup the content before returning it because it contains
+ //error content after the list of hashes
+ //
+ //As of 30/1/2012, the result is a proper Json array so we don't need to read the content
+ //line by line
+
+ var serializer = new JsonSerializer();
+ serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);
+ var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));
+ return hashes;
+ }
+ }
+ //Any other status code is unexpected and the exception should be rethrown
+ Log.LogError(response);
+ throw ex;
+
+ }
+
+ //Any other status code is unexpected but there was no exception. We can probably continue processing
+ Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
+
+ return empty;
+ });
+
+ }
+
+
+ public async Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)
+ {
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("Invalid Token");
+ if (StorageUrl == null)
+ throw new InvalidOperationException("Invalid Storage Url");
+ if (String.IsNullOrWhiteSpace(container))
+ throw new ArgumentNullException("container");
+ if (relativeUrl == null)
+ throw new ArgumentNullException("relativeUrl");
+ if (end.HasValue && end < 0)
+ throw new ArgumentOutOfRangeException("end");
+ if (start < 0)
+ throw new ArgumentOutOfRangeException("start");
+ Contract.EndContractBlock();
+
+ //Don't use a timeout because putting the hashmap may be a long process
+ using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end})
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
+ var uri = builder.Uri;
+
+ client.DownloadProgressChanged += (sender, args) =>
+ Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
+ uri.Segments.Last(), args.ProgressPercentage,
+ args.BytesReceived,
+ args.TotalBytesToReceive);
+
+
+ var result = await client.DownloadDataTaskAsync(uri, cancellationToken);
+ return result;
+ }
+ }
+
+
+ public async Task PostBlock(string account, string container, byte[] block, int offset, int count)
+ {
+ if (String.IsNullOrWhiteSpace(container))
+ throw new ArgumentNullException("container");
+ if (block == null)
+ throw new ArgumentNullException("block");
+ if (offset < 0 || offset >= block.Length)
+ throw new ArgumentOutOfRangeException("offset");
+ if (count < 0 || count > block.Length)
+ throw new ArgumentOutOfRangeException("count");
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("Invalid Token");
+ if (StorageUrl == null)
+ throw new InvalidOperationException("Invalid Storage Url");
+ Contract.EndContractBlock();
+
+
+ try
+ {
+
+ //Don't use a timeout because putting the hashmap may be a long process
+ using (var client = new RestClient(_baseClient) { Timeout = 0 })
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ var builder = client.GetAddressBuilder(container, "");
+ //We are doing an update
+ builder.Query = "update";
+ var uri = builder.Uri;
+
+ client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
+
+ Log.InfoFormat("[BLOCK POST] START");
+
+ client.UploadProgressChanged += (sender, args) =>
+ Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
+ args.ProgressPercentage, args.BytesSent,
+ args.TotalBytesToSend);
+ client.UploadFileCompleted += (sender, args) =>
+ Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
+
+ var buffer = new byte[count];
+ Buffer.BlockCopy(block, offset, buffer, 0, count);
+ //Send the block
+ await client.UploadDataTaskAsync(uri, "POST", buffer);
+ Log.InfoFormat("[BLOCK POST] END");
+ }
+ }
+ catch (Exception exc)
{
- return response.ContentStream;
+ Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);
+ throw;
}
- else
- throw new WebException(String.Format("GetObject failed with unexpected status code {0}", response.StatusCode));
}
+
+ public async Task<TreeHash> GetHashMap(string account, string container, string objectName)
+ {
+ if (String.IsNullOrWhiteSpace(container))
+ throw new ArgumentNullException("container");
+ if (String.IsNullOrWhiteSpace(objectName))
+ throw new ArgumentNullException("objectName");
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("Invalid Token");
+ if (StorageUrl == null)
+ throw new InvalidOperationException("Invalid Storage Url");
+ Contract.EndContractBlock();
+
+ try
+ {
+ //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
+ //object to avoid concurrency errors.
+ //
+ //Download operations take a long time therefore they have no timeout.
+ //TODO: Do they really? this is a hashmap operation, not a download
+
+ //Start downloading the object asynchronously
+ using (var client = new RestClient(_baseClient) { Timeout = 0 })
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ //The container and objectName are relative names. They are joined with the client's
+ //BaseAddress to create the object's absolute address
+ var builder = client.GetAddressBuilder(container, objectName);
+ builder.Query = "format=json&hashmap";
+ var uri = builder.Uri;
+
+
+ var json = await client.DownloadStringTaskAsync(uri);
+ var treeHash = TreeHash.Parse(json);
+ Log.InfoFormat("[GET HASH] END {0}", objectName);
+ return treeHash;
+ }
+ }
+ catch (Exception exc)
+ {
+ Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
+ throw;
+ }
+
+ }
+
+
/// <summary>
///
/// </summary>
+ /// <param name="account"></param>
/// <param name="container"></param>
/// <param name="objectName"></param>
/// <param name="fileName"></param>
/// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
/// <remarks>>This method should have no timeout or a very long one</remarks>
- public Task PutObject(string container, string objectName, string fileName, string hash = null)
+ public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream")
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
if (String.IsNullOrWhiteSpace(fileName))
throw new ArgumentNullException("fileName", "The fileName property can't be empty");
- if (!File.Exists(fileName))
- throw new FileNotFoundException("The file does not exist",fileName);
-
-
- string url = container + "/" + objectName;
-
- var request = new RestRequest {Path=url,Method=WebMethod.Put};
-
/*
- if(UploadPercentLimit>0)
- request.TaskOptions=new TaskOptions<int>{RateLimitPercent=UploadPercentLimit};
+ if (!File.Exists(fileName) && !Directory.Exists(fileName))
+ throw new FileNotFoundException("The file or directory does not exist",fileName);
*/
- Trace.TraceInformation("[PUT] START {0}",objectName);
- string etag = hash??CalculateHash(fileName);
- request.AddFile(fileName, fileName, fileName);
- //request.AddPostContent(File.ReadAllBytes(fileName));
- request.AddHeader("Content-Type","application/octet-stream");
- request.AddHeader("ETag", etag);
- //_client.TaskOptions = new TaskOptions<int> {RateLimitPercent = 0.5};
+
try
{
- var response=_client.Request(request);
- Trace.TraceInformation("[PUT] END {0}", objectName);
+ using (var client = new RestClient(_baseClient) { Timeout = 0 })
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ var builder = client.GetAddressBuilder(container, objectName);
+ var uri = builder.Uri;
+
+ string etag = hash ?? CalculateHash(fileName);
+
+ client.Headers.Add("Content-Type", contentType);
+ client.Headers.Add("ETag", etag);
+
+
+ Log.InfoFormat("[PUT] START {0}", objectName);
+ client.UploadProgressChanged += (sender, args) =>
+ {
+ using (ThreadContext.Stacks["PUT"].Push("Progress"))
+ {
+ Log.InfoFormat("{0} {1}% {2} of {3}", fileName,
+ args.ProgressPercentage,
+ args.BytesSent, args.TotalBytesToSend);
+ }
+ };
+
+ client.UploadFileCompleted += (sender, args) =>
+ {
+ using (ThreadContext.Stacks["PUT"].Push("Progress"))
+ {
+ Log.InfoFormat("Completed {0}", fileName);
+ }
+ };
+ if (contentType=="application/directory")
+ await client.UploadDataTaskAsync(uri, "PUT", new byte[0]);
+ else
+ await client.UploadFileTaskAsync(uri, "PUT", fileName);
+ }
- if (response.StatusCode == HttpStatusCode.Created)
- return Task.Factory.StartNew(()=>{});
- if (response.StatusCode == HttpStatusCode.LengthRequired)
- throw new InvalidOperationException();
- else
- throw new WebException(String.Format("GetObject failed with unexpected status code {0}",
- response.StatusCode));
- /* return Task.Factory.FromAsync(_client.BeginRequest(request),ar=>_client.EndRequest(ar))
- .ContinueWith(t=>
- {*/
+ Log.InfoFormat("[PUT] END {0}", objectName);
}
catch (Exception exc)
{
- Trace.TraceError("[PUT] END {0} with {1}", objectName, exc);
+ Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
throw;
}
-/*
- var response = t.Result;
- if (t.IsFaulted)
- Trace.TraceError("[PUT] END {0} with {1}", objectName, t.Exception);
- else
- {
- Trace.TraceInformation("[PUT] END {0}",objectName);
- }
-*/
- /* if (response.StatusCode == HttpStatusCode.Created)
- return;
- if (response.StatusCode == HttpStatusCode.LengthRequired)
- throw new InvalidOperationException();
- else
- throw new WebException(String.Format("GetObject failed with unexpected status code {0}",
- response.StatusCode));*/
- /*});*/
}
-
+
+
private static string CalculateHash(string fileName)
{
+ Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
+ Contract.EndContractBlock();
+
string hash;
using (var hasher = MD5.Create())
using(var stream=File.OpenRead(fileName))
}
return hash;
}
-
- public void DeleteObject(string container, string objectName)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The objectName property can't be empty");
-
- var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
- var response = _client.Request(request);
-
- if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent)
- return;
- else
- throw new WebException(String.Format("DeleteObject failed with unexpected status code {0}", response.StatusCode));
-
- }
-
- public void MoveObject(string container, string oldObjectName, string newObjectName)
+
+ public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
{
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
+ if (String.IsNullOrWhiteSpace(sourceContainer))
+ throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(oldObjectName))
throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
+ if (String.IsNullOrWhiteSpace(targetContainer))
+ throw new ArgumentNullException("targetContainer", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(newObjectName))
throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
+ Contract.EndContractBlock();
- var request = new RestRequest { Path = container + "/" + newObjectName, Method = WebMethod.Put };
- request.AddHeader("X-Copy-From",String.Format("/{0}/{1}",container,oldObjectName));
- request.AddPostContent(new byte[]{});
- var response = _client.Request(request);
+ var targetUrl = targetContainer + "/" + newObjectName;
+ var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
- if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.NoContent || response.StatusCode==HttpStatusCode.Created)
+ using (var client = new RestClient(_baseClient))
{
- this.DeleteObject(container,oldObjectName);
- }
- else
- throw new WebException(String.Format("MoveObject failed with unexpected status code {0}", response.StatusCode));
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Headers.Add("X-Move-From", sourceUrl);
+ client.PutWithRetry(targetUrl, 3);
+
+ var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
+ if (!expectedCodes.Contains(client.StatusCode))
+ throw CreateWebException("MoveObject", client.StatusCode);
+ }
+ }
+
+ public void DeleteObject(string account, string sourceContainer, string objectName)
+ {
+ if (String.IsNullOrWhiteSpace(sourceContainer))
+ throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
+ if (String.IsNullOrWhiteSpace(objectName))
+ throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
+ Contract.EndContractBlock();
+
+ var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
+ var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
+
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Headers.Add("X-Move-From", sourceUrl);
+ client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
+ Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);
+ client.PutWithRetry(targetUrl, 3);
+
+ var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
+ if (!expectedCodes.Contains(client.StatusCode))
+ throw CreateWebException("DeleteObject", client.StatusCode);
+ }
+ }
+
+
+ private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
+ {
+ return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
}
- private string GetHeaderValue(string headerName, RestResponse response, IQueryable<string> keys)
+
+/*
+ public IEnumerable<ObjectInfo> ListDirectories(ContainerInfo container)
{
- if (keys.Any(key => key == headerName))
- return response.Headers[headerName];
- else
- throw new WebException(String.Format("The {0} header is missing",headerName));
+ var directories=this.ListObjects(container.Account, container.Name, "/");
}
+*/
- private static void ThrowIfNotStatusOK(RestResponse response, string message)
+ public bool CanUpload(string account, ObjectInfo cloudFile)
{
- int status = (int)response.StatusCode;
- if (status < 200 || status >= 300)
- throw new WebException(String.Format("{0} with code {1}",message, status));
+ Contract.Requires(!String.IsNullOrWhiteSpace(account));
+ Contract.Requires(cloudFile!=null);
+
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+
+ var parts = cloudFile.Name.Split('/');
+ var folder = String.Join("/", parts,0,parts.Length-1);
+
+ var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid());
+
+ client.Parameters.Clear();
+ try
+ {
+ client.PutWithRetry(fileUrl, 3, @"application/octet-stream");
+
+ var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
+ var result=(expectedCodes.Contains(client.StatusCode));
+ DeleteObject(account, cloudFile.Container, fileUrl);
+ return result;
+ }
+ catch
+ {
+ return false;
+ }
+ }
}
}
+
+ public class ShareAccountInfo
+ {
+ public DateTime? last_modified { get; set; }
+ public string name { get; set; }
+ }
}