#region
/* -----------------------------------------------------------------------
*
*
* Copyright 2011-2012 GRNET S.A. All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* 1. Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* 2. Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
*
* THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and
* documentation are those of the authors and should not be
* interpreted as representing official policies, either expressed
* or implied, of GRNET S.A.
*
* -----------------------------------------------------------------------
*/
#endregion
// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
//
// The class provides methods to upload/download files, delete files, manage containers
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.Diagnostics.Contracts;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Pithos.Interfaces;
using log4net;
namespace Pithos.Network
{
[Export(typeof(ICloudClient))]
public class CloudFilesClient:ICloudClient
{
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
//CloudFilesClient uses *_baseClient* internally to communicate with the server
//RestClient provides a REST-friendly interface over the standard WebClient.
private RestClient _baseClient;
//During authentication the client provides a UserName
public string UserName { get; set; }
//and and ApiKey to the server
public string ApiKey { get; set; }
//And receives an authentication Token. This token must be provided in ALL other operations,
//in the X-Auth-Token header
private string _token;
public string Token
{
get { return _token; }
set
{
_token = value;
_baseClient.Headers["X-Auth-Token"] = value;
}
}
//The client also receives a StorageUrl after authentication. All subsequent operations must
//use this url
public Uri StorageUrl { get; set; }
protected Uri RootAddressUri { get; set; }
/* private WebProxy _proxy;
public WebProxy Proxy
{
get { return _proxy; }
set
{
_proxy = value;
if (_baseClient != null)
_baseClient.Proxy = value;
}
}
*/
/* private Uri _proxy;
public Uri Proxy
{
get { return _proxy; }
set
{
_proxy = value;
if (_baseClient != null)
_baseClient.Proxy = new WebProxy(value);
}
}*/
public double DownloadPercentLimit { get; set; }
public double UploadPercentLimit { get; set; }
public string AuthenticationUrl { get; set; }
public string VersionPath
{
get { return UsePithos ? "v1" : "v1.0"; }
}
public bool UsePithos { get; set; }
public CloudFilesClient(string userName, string apiKey)
{
UserName = userName;
ApiKey = apiKey;
}
public CloudFilesClient(AccountInfo accountInfo)
{
if (accountInfo==null)
throw new ArgumentNullException("accountInfo");
Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
Contract.Ensures(StorageUrl != null);
Contract.Ensures(_baseClient != null);
Contract.Ensures(RootAddressUri != null);
Contract.EndContractBlock();
_baseClient = new RestClient
{
BaseAddress = accountInfo.StorageUri.ToString(),
Timeout = 10000,
Retries = 3,
};
StorageUrl = accountInfo.StorageUri;
Token = accountInfo.Token;
UserName = accountInfo.UserName;
//Get the root address (StorageUrl without the account)
var storageUrl = StorageUrl.AbsoluteUri;
var usernameIndex = storageUrl.LastIndexOf(UserName);
var rootUrl = storageUrl.Substring(0, usernameIndex);
RootAddressUri = new Uri(rootUrl);
}
public AccountInfo Authenticate()
{
if (String.IsNullOrWhiteSpace(UserName))
throw new InvalidOperationException("UserName is empty");
if (String.IsNullOrWhiteSpace(ApiKey))
throw new InvalidOperationException("ApiKey is empty");
if (String.IsNullOrWhiteSpace(AuthenticationUrl))
throw new InvalidOperationException("AuthenticationUrl is empty");
Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
Contract.Ensures(StorageUrl != null);
Contract.Ensures(_baseClient != null);
Contract.Ensures(RootAddressUri != null);
Contract.EndContractBlock();
Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
var groups = new List();
using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
{
/* if (Proxy != null)
authClient.Proxy = Proxy;*/
Contract.Assume(authClient.Headers!=null);
authClient.Headers.Add("X-Auth-User", UserName);
authClient.Headers.Add("X-Auth-Key", ApiKey);
authClient.DownloadStringWithRetry(VersionPath, 3);
authClient.AssertStatusOK("Authentication failed");
var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
if (String.IsNullOrWhiteSpace(storageUrl))
throw new InvalidOperationException("Failed to obtain storage url");
_baseClient = new RestClient
{
BaseAddress = storageUrl,
Timeout = 10000,
Retries = 3,
//Proxy=Proxy
};
StorageUrl = new Uri(storageUrl);
//Get the root address (StorageUrl without the account)
var usernameIndex=storageUrl.LastIndexOf(UserName);
var rootUrl = storageUrl.Substring(0, usernameIndex);
RootAddressUri = new Uri(rootUrl);
var token = authClient.GetHeaderValue("X-Auth-Token");
if (String.IsNullOrWhiteSpace(token))
throw new InvalidOperationException("Failed to obtain token url");
Token = token;
/* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
groups = (from key in keys
where key.StartsWith("X-Account-Group-")
let name = key.Substring(16)
select new Group(name, authClient.ResponseHeaders[key]))
.ToList();
*/
}
Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
Debug.Assert(_baseClient!=null);
return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};
}
public IList ListContainers(string account)
{
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.Parameters.Clear();
client.Parameters.Add("format", "json");
var content = client.DownloadStringWithRetry("", 3);
client.AssertStatusOK("List Containers failed");
if (client.StatusCode == HttpStatusCode.NoContent)
return new List();
var infos = JsonConvert.DeserializeObject>(content);
foreach (var info in infos)
{
info.Account = account;
}
return infos;
}
}
private string GetAccountUrl(string account)
{
return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
}
public IList ListSharingAccounts(DateTime? since=null)
{
using (ThreadContext.Stacks["Share"].Push("List Accounts"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using (var client = new RestClient(_baseClient))
{
client.Parameters.Clear();
client.Parameters.Add("format", "json");
client.IfModifiedSince = since;
//Extract the username from the base address
client.BaseAddress = RootAddressUri.AbsoluteUri;
var content = client.DownloadStringWithRetry(@"", 3);
client.AssertStatusOK("ListSharingAccounts failed");
//If the result is empty, return an empty list,
var infos = String.IsNullOrWhiteSpace(content)
? new List()
//Otherwise deserialize the account list into a list of ShareAccountInfos
: JsonConvert.DeserializeObject>(content);
Log.DebugFormat("END");
return infos;
}
}
}
//Request listing of all objects in a container modified since a specific time.
//If the *since* value is missing, return all objects
public IList ListSharedObjects(DateTime? since = null)
{
using (ThreadContext.Stacks["Share"].Push("List Objects"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
//'since' is not used here because we need to have ListObjects return a NoChange result
//for all shared accounts,containers
var accounts = ListSharingAccounts();
var items = from account in accounts
let containers = ListContainers(account.name)
from container in containers
select ListObjects(account.name, container.Name,since);
var objects=items.SelectMany(r=> r).ToList();
/*
var objects = new List();
foreach (var containerObjects in items)
{
objects.AddRange(containerObjects);
}
*/
if (Log.IsDebugEnabled) Log.DebugFormat("END");
return objects;
}
}
public void SetTags(ObjectInfo target,IDictionary tags)
{
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("The Token is not set");
if (StorageUrl == null)
throw new InvalidOperationException("The StorageUrl is not set");
if (target == null)
throw new ArgumentNullException("target");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Share"].Push("Share Object"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using (var client = new RestClient(_baseClient))
{
client.BaseAddress = GetAccountUrl(target.Account);
client.Parameters.Clear();
client.Parameters.Add("update", "");
foreach (var tag in tags)
{
var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
client.Headers.Add(headerTag, tag.Value);
}
client.DownloadStringWithRetry(target.Container, 3);
client.AssertStatusOK("SetTags failed");
//If the status is NOT ACCEPTED we have a problem
if (client.StatusCode != HttpStatusCode.Accepted)
{
Log.Error("Failed to set tags");
throw new Exception("Failed to set tags");
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
}
}
}
public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
{
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("The Token is not set");
if (StorageUrl==null)
throw new InvalidOperationException("The StorageUrl is not set");
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container");
if (String.IsNullOrWhiteSpace(objectName))
throw new ArgumentNullException("objectName");
if (String.IsNullOrWhiteSpace(account))
throw new ArgumentNullException("account");
if (String.IsNullOrWhiteSpace(shareTo))
throw new ArgumentNullException("shareTo");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Share"].Push("Share Object"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using (var client = new RestClient(_baseClient))
{
client.BaseAddress = GetAccountUrl(account);
client.Parameters.Clear();
client.Parameters.Add("format", "json");
string permission = "";
if (write)
permission = String.Format("write={0}", shareTo);
else if (read)
permission = String.Format("read={0}", shareTo);
client.Headers.Add("X-Object-Sharing", permission);
var content = client.DownloadStringWithRetry(container, 3);
client.AssertStatusOK("ShareObject failed");
//If the result is empty, return an empty list,
var infos = String.IsNullOrWhiteSpace(content)
? new List()
//Otherwise deserialize the object list into a list of ObjectInfos
: JsonConvert.DeserializeObject>(content);
if (Log.IsDebugEnabled) Log.DebugFormat("END");
}
}
}
public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
{
if (accountInfo==null)
throw new ArgumentNullException("accountInfo");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Account"].Push("GetPolicies"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
client.BaseAddress = GetAccountUrl(accountInfo.UserName);
client.Parameters.Clear();
client.Parameters.Add("format", "json");
client.Head(String.Empty, 3);
var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
long quota, bytes;
if (long.TryParse(quotaValue, out quota))
accountInfo.Quota = quota;
if (long.TryParse(bytesValue, out bytes))
accountInfo.BytesUsed = bytes;
return accountInfo;
}
}
}
public void UpdateMetadata(ObjectInfo objectInfo)
{
if (objectInfo == null)
throw new ArgumentNullException("objectInfo");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using(var client=new RestClient(_baseClient))
{
client.BaseAddress = GetAccountUrl(objectInfo.Account);
client.Parameters.Clear();
//Set Tags
foreach (var tag in objectInfo.Tags)
{
var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
client.Headers.Add(headerTag, tag.Value);
}
//Set Permissions
var permissions=objectInfo.GetPermissionString();
client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);
client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
var isPublic = objectInfo.IsPublic.ToString().ToLower();
client.Headers.Add("X-Object-Public", isPublic);
/*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);
uriBuilder.Query = "update=";
var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/
var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);
client.PostWithRetry(address,"application/xml");
//client.UploadValues(uri,new NameValueCollection());
client.AssertStatusOK("UpdateMetadata failed");
//If the status is NOT ACCEPTED or OK we have a problem
if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
{
Log.Error("Failed to update metadata");
throw new Exception("Failed to update metadata");
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
}
}
}
public void UpdateMetadata(ContainerInfo containerInfo)
{
if (containerInfo == null)
throw new ArgumentNullException("containerInfo");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using(var client=new RestClient(_baseClient))
{
client.BaseAddress = GetAccountUrl(containerInfo.Account);
client.Parameters.Clear();
//Set Tags
foreach (var tag in containerInfo.Tags)
{
var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
client.Headers.Add(headerTag, tag.Value);
}
//Set Policies
foreach (var policy in containerInfo.Policies)
{
var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
client.Headers.Add(headerPolicy, policy.Value);
}
var uriBuilder = client.GetAddressBuilder(containerInfo.Name,"");
var uri = uriBuilder.Uri;
client.UploadValues(uri,new NameValueCollection());
client.AssertStatusOK("UpdateMetadata failed");
//If the status is NOT ACCEPTED or OK we have a problem
if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
{
Log.Error("Failed to update metadata");
throw new Exception("Failed to update metadata");
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
}
}
}
public IList ListObjects(string account, string container, DateTime? since = null)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Objects"].Push("List"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.Parameters.Clear();
client.Parameters.Add("format", "json");
client.IfModifiedSince = since;
var content = client.DownloadStringWithRetry(container, 3);
client.AssertStatusOK("ListObjects failed");
if (client.StatusCode==HttpStatusCode.NotModified)
return new[]{new NoModificationInfo(account,container)};
//If the result is empty, return an empty list,
var infos = String.IsNullOrWhiteSpace(content)
? new List()
//Otherwise deserialize the object list into a list of ObjectInfos
: JsonConvert.DeserializeObject>(content);
foreach (var info in infos)
{
info.Container = container;
info.Account = account;
info.StorageUri = this.StorageUrl;
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
return infos;
}
}
}
public IList ListObjects(string account, string container, string folder, DateTime? since = null)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container");
/*
if (String.IsNullOrWhiteSpace(folder))
throw new ArgumentNullException("folder");
*/
Contract.EndContractBlock();
using (ThreadContext.Stacks["Objects"].Push("List"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.Parameters.Clear();
client.Parameters.Add("format", "json");
client.Parameters.Add("path", folder);
client.IfModifiedSince = since;
var content = client.DownloadStringWithRetry(container, 3);
client.AssertStatusOK("ListObjects failed");
if (client.StatusCode==HttpStatusCode.NotModified)
return new[]{new NoModificationInfo(account,container,folder)};
var infos = JsonConvert.DeserializeObject>(content);
foreach (var info in infos)
{
info.Account = account;
if (info.Container == null)
info.Container = container;
info.StorageUri = this.StorageUrl;
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
return infos;
}
}
}
public bool ContainerExists(string account, string container)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Containters"].Push("Exists"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.Parameters.Clear();
client.Head(container, 3);
bool result;
switch (client.StatusCode)
{
case HttpStatusCode.OK:
case HttpStatusCode.NoContent:
result=true;
break;
case HttpStatusCode.NotFound:
result=false;
break;
default:
throw CreateWebException("ContainerExists", client.StatusCode);
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
return result;
}
}
}
public bool ObjectExists(string account, string container, string objectName)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(objectName))
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
Contract.EndContractBlock();
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.Parameters.Clear();
client.Head(container + "/" + objectName, 3);
switch (client.StatusCode)
{
case HttpStatusCode.OK:
case HttpStatusCode.NoContent:
return true;
case HttpStatusCode.NotFound:
return false;
default:
throw CreateWebException("ObjectExists", client.StatusCode);
}
}
}
public ObjectInfo GetObjectInfo(string account, string container, string objectName)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(objectName))
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
{
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
try
{
client.Parameters.Clear();
client.Head(container + "/" + objectName, 3);
if (client.TimedOut)
return ObjectInfo.Empty;
switch (client.StatusCode)
{
case HttpStatusCode.OK:
case HttpStatusCode.NoContent:
var keys = client.ResponseHeaders.AllKeys.AsQueryable();
var tags = client.GetMeta("X-Object-Meta-");
var extensions = (from key in keys
where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
select new {Name = key, Value = client.ResponseHeaders[key]})
.ToDictionary(t => t.Name, t => t.Value);
var permissions=client.GetHeaderValue("X-Object-Sharing", true);
var info = new ObjectInfo
{
Account = account,
Container = container,
Name = objectName,
ETag = client.GetHeaderValue("ETag"),
X_Object_Hash = client.GetHeaderValue("X-Object-Hash"),
Content_Type = client.GetHeaderValue("Content-Type"),
Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),
Tags = tags,
Last_Modified = client.LastModified,
Extensions = extensions,
ContentEncoding=client.GetHeaderValue("Content-Encoding",true),
ContendDisposition = client.GetHeaderValue("Content-Disposition",true),
Manifest=client.GetHeaderValue("X-Object-Manifest",true),
PublicUrl=client.GetHeaderValue("X-Object-Public",true),
StorageUri=this.StorageUrl,
};
info.SetPermissions(permissions);
return info;
case HttpStatusCode.NotFound:
return ObjectInfo.Empty;
default:
throw new WebException(
String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
objectName, client.StatusCode));
}
}
catch (RetryException)
{
Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);
return ObjectInfo.Empty;
}
catch (WebException e)
{
Log.Error(
String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
objectName, client.StatusCode), e);
throw;
}
}
}
}
public void CreateFolder(string account, string container, string folder)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(folder))
throw new ArgumentNullException("folder", "The folder property can't be empty");
Contract.EndContractBlock();
var folderUrl=String.Format("{0}/{1}",container,folder);
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.Parameters.Clear();
client.Headers.Add("Content-Type", @"application/directory");
client.Headers.Add("Content-Length", "0");
client.PutWithRetry(folderUrl, 3);
if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
throw CreateWebException("CreateFolder", client.StatusCode);
}
}
public ContainerInfo GetContainerInfo(string account, string container)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
Contract.EndContractBlock();
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.Head(container);
switch (client.StatusCode)
{
case HttpStatusCode.OK:
case HttpStatusCode.NoContent:
var tags = client.GetMeta("X-Container-Meta-");
var policies = client.GetMeta("X-Container-Policy-");
var containerInfo = new ContainerInfo
{
Account=account,
Name = container,
StorageUrl=this.StorageUrl.ToString(),
Count =
long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),
Last_Modified=client.LastModified,
Tags=tags,
Policies=policies
};
return containerInfo;
case HttpStatusCode.NotFound:
return ContainerInfo.Empty;
default:
throw CreateWebException("GetContainerInfo", client.StatusCode);
}
}
}
public void CreateContainer(string account, string container)
{
if (String.IsNullOrWhiteSpace(account))
throw new ArgumentNullException("account");
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container");
Contract.EndContractBlock();
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.PutWithRetry(container, 3);
var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
if (!expectedCodes.Contains(client.StatusCode))
throw CreateWebException("CreateContainer", client.StatusCode);
}
}
public void DeleteContainer(string account, string container)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
Contract.EndContractBlock();
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.DeleteWithRetry(container, 3);
var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
if (!expectedCodes.Contains(client.StatusCode))
throw CreateWebException("DeleteContainer", client.StatusCode);
}
}
///
///
///
///
///
///
///
///
/// This method should have no timeout or a very long one
//Asynchronously download the object specified by *objectName* in a specific *container* to
// a local file
public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(objectName))
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
Contract.EndContractBlock();
try
{
//WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
//object to avoid concurrency errors.
//
//Download operations take a long time therefore they have no timeout.
using(var client = new RestClient(_baseClient) { Timeout = 0 })
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
//The container and objectName are relative names. They are joined with the client's
//BaseAddress to create the object's absolute address
var builder = client.GetAddressBuilder(container, objectName);
var uri = builder.Uri;
//Download progress is reported to the Trace log
Log.InfoFormat("[GET] START {0}", objectName);
client.DownloadProgressChanged += (sender, args) =>
Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
fileName, args.ProgressPercentage,
args.BytesReceived,
args.TotalBytesToReceive);
//Start downloading the object asynchronously
await client.DownloadFileTaskAsync(uri, fileName,cancellationToken);
//Once the download completes
//Delete the local client object
}
//And report failure or completion
}
catch (Exception exc)
{
Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);
throw;
}
Log.InfoFormat("[GET] END {0}", objectName);
}
public Task> PutHashMap(string account, string container, string objectName, TreeHash hash)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container");
if (String.IsNullOrWhiteSpace(objectName))
throw new ArgumentNullException("objectName");
if (hash==null)
throw new ArgumentNullException("hash");
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("Invalid Token");
if (StorageUrl == null)
throw new InvalidOperationException("Invalid Storage Url");
Contract.EndContractBlock();
//Don't use a timeout because putting the hashmap may be a long process
var client = new RestClient(_baseClient) { Timeout = 0 };
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
//The container and objectName are relative names. They are joined with the client's
//BaseAddress to create the object's absolute address
var builder = client.GetAddressBuilder(container, objectName);
builder.Query = "format=json&hashmap";
var uri = builder.Uri;
//Send the tree hash as Json to the server
client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
var jsonHash = hash.ToJson();
var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);
if (Log.IsDebugEnabled)
Log.DebugFormat("Hashes:\r\n{0}", jsonHash);
return uploadTask.ContinueWith(t =>
{
var empty = (IList)new List();
//The server will respond either with 201-created if all blocks were already on the server
if (client.StatusCode == HttpStatusCode.Created)
{
//in which case we return an empty hash list
return empty;
}
//or with a 409-conflict and return the list of missing parts
//A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
if (t.IsFaulted)
{
var ex = t.Exception.InnerException;
var we = ex as WebException;
var response = we.Response as HttpWebResponse;
if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
{
//In case of 409 the missing parts will be in the response content
using (var stream = response.GetResponseStream())
using(var reader=stream.GetLoggedReader(Log))
{
//We used to have to cleanup the content before returning it because it contains
//error content after the list of hashes
//
//As of 30/1/2012, the result is a proper Json array so we don't need to read the content
//line by line
var serializer = new JsonSerializer();
serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);
var hashes = (List)serializer.Deserialize(reader, typeof(List));
return hashes;
}
}
//Any other status code is unexpected and the exception should be rethrown
Log.LogError(response);
throw ex;
}
//Any other status code is unexpected but there was no exception. We can probably continue processing
Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
return empty;
});
}
public async Task GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)
{
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("Invalid Token");
if (StorageUrl == null)
throw new InvalidOperationException("Invalid Storage Url");
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container");
if (relativeUrl == null)
throw new ArgumentNullException("relativeUrl");
if (end.HasValue && end < 0)
throw new ArgumentOutOfRangeException("end");
if (start < 0)
throw new ArgumentOutOfRangeException("start");
Contract.EndContractBlock();
//Don't use a timeout because putting the hashmap may be a long process
using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end})
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
var uri = builder.Uri;
client.DownloadProgressChanged += (sender, args) =>
Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
uri.Segments.Last(), args.ProgressPercentage,
args.BytesReceived,
args.TotalBytesToReceive);
var result = await client.DownloadDataTaskAsync(uri, cancellationToken);
return result;
}
}
public async Task PostBlock(string account, string container, byte[] block, int offset, int count,CancellationToken token)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container");
if (block == null)
throw new ArgumentNullException("block");
if (offset < 0 || offset >= block.Length)
throw new ArgumentOutOfRangeException("offset");
if (count < 0 || count > block.Length)
throw new ArgumentOutOfRangeException("count");
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("Invalid Token");
if (StorageUrl == null)
throw new InvalidOperationException("Invalid Storage Url");
Contract.EndContractBlock();
try
{
//Don't use a timeout because putting the hashmap may be a long process
using (var client = new RestClient(_baseClient) { Timeout = 0 })
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
token.Register(client.CancelAsync);
var builder = client.GetAddressBuilder(container, "");
//We are doing an update
builder.Query = "update";
var uri = builder.Uri;
client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
Log.InfoFormat("[BLOCK POST] START");
client.UploadProgressChanged += (sender, args) =>
Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
args.ProgressPercentage, args.BytesSent,
args.TotalBytesToSend);
client.UploadFileCompleted += (sender, args) =>
Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
var buffer = new byte[count];
Buffer.BlockCopy(block, offset, buffer, 0, count);
//Send the block
await client.UploadDataTaskAsync(uri, "POST", buffer);
Log.InfoFormat("[BLOCK POST] END");
}
}
catch (TaskCanceledException )
{
Log.Info("Aborting block");
throw;
}
catch (Exception exc)
{
Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);
throw;
}
}
public async Task GetHashMap(string account, string container, string objectName)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container");
if (String.IsNullOrWhiteSpace(objectName))
throw new ArgumentNullException("objectName");
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("Invalid Token");
if (StorageUrl == null)
throw new InvalidOperationException("Invalid Storage Url");
Contract.EndContractBlock();
try
{
//WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
//object to avoid concurrency errors.
//
//Download operations take a long time therefore they have no timeout.
//TODO: Do they really? this is a hashmap operation, not a download
//Start downloading the object asynchronously
using (var client = new RestClient(_baseClient) { Timeout = 0 })
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
//The container and objectName are relative names. They are joined with the client's
//BaseAddress to create the object's absolute address
var builder = client.GetAddressBuilder(container, objectName);
builder.Query = "format=json&hashmap";
var uri = builder.Uri;
var json = await client.DownloadStringTaskAsync(uri);
var treeHash = TreeHash.Parse(json);
Log.InfoFormat("[GET HASH] END {0}", objectName);
return treeHash;
}
}
catch (Exception exc)
{
Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
throw;
}
}
///
///
///
///
///
///
///
/// Optional hash value for the file. If no hash is provided, the method calculates a new hash
/// >This method should have no timeout or a very long one
public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream")
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(objectName))
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
if (String.IsNullOrWhiteSpace(fileName))
throw new ArgumentNullException("fileName", "The fileName property can't be empty");
/*
if (!File.Exists(fileName) && !Directory.Exists(fileName))
throw new FileNotFoundException("The file or directory does not exist",fileName);
*/
try
{
using (var client = new RestClient(_baseClient) { Timeout = 0 })
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
var builder = client.GetAddressBuilder(container, objectName);
var uri = builder.Uri;
string etag = hash ?? CalculateHash(fileName);
client.Headers.Add("Content-Type", contentType);
client.Headers.Add("ETag", etag);
Log.InfoFormat("[PUT] START {0}", objectName);
client.UploadProgressChanged += (sender, args) =>
{
using (ThreadContext.Stacks["PUT"].Push("Progress"))
{
Log.InfoFormat("{0} {1}% {2} of {3}", fileName,
args.ProgressPercentage,
args.BytesSent, args.TotalBytesToSend);
}
};
client.UploadFileCompleted += (sender, args) =>
{
using (ThreadContext.Stacks["PUT"].Push("Progress"))
{
Log.InfoFormat("Completed {0}", fileName);
}
};
if (contentType=="application/directory")
await client.UploadDataTaskAsync(uri, "PUT", new byte[0]);
else
await client.UploadFileTaskAsync(uri, "PUT", fileName);
}
Log.InfoFormat("[PUT] END {0}", objectName);
}
catch (Exception exc)
{
Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
throw;
}
}
private static string CalculateHash(string fileName)
{
Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
Contract.EndContractBlock();
string hash;
using (var hasher = MD5.Create())
using(var stream=File.OpenRead(fileName))
{
var hashBuilder=new StringBuilder();
foreach (byte b in hasher.ComputeHash(stream))
hashBuilder.Append(b.ToString("x2").ToLower());
hash = hashBuilder.ToString();
}
return hash;
}
public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
{
if (String.IsNullOrWhiteSpace(sourceContainer))
throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(oldObjectName))
throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
if (String.IsNullOrWhiteSpace(targetContainer))
throw new ArgumentNullException("targetContainer", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(newObjectName))
throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
Contract.EndContractBlock();
var targetUrl = targetContainer + "/" + newObjectName;
var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.Headers.Add("X-Move-From", sourceUrl);
client.PutWithRetry(targetUrl, 3);
var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
if (!expectedCodes.Contains(client.StatusCode))
throw CreateWebException("MoveObject", client.StatusCode);
}
}
public void DeleteObject(string account, string sourceContainer, string objectName)
{
if (String.IsNullOrWhiteSpace(sourceContainer))
throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
if (String.IsNullOrWhiteSpace(objectName))
throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
Contract.EndContractBlock();
var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.Headers.Add("X-Move-From", sourceUrl);
client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);
client.PutWithRetry(targetUrl, 3);
var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
if (!expectedCodes.Contains(client.StatusCode))
throw CreateWebException("DeleteObject", client.StatusCode);
}
}
private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
{
return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
}
/*
public IEnumerable ListDirectories(ContainerInfo container)
{
var directories=this.ListObjects(container.Account, container.Name, "/");
}
*/
public bool CanUpload(string account, ObjectInfo cloudFile)
{
Contract.Requires(!String.IsNullOrWhiteSpace(account));
Contract.Requires(cloudFile!=null);
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
var parts = cloudFile.Name.Split('/');
var folder = String.Join("/", parts,0,parts.Length-1);
var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid());
client.Parameters.Clear();
try
{
client.PutWithRetry(fileUrl, 3, @"application/octet-stream");
var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
var result=(expectedCodes.Contains(client.StatusCode));
DeleteObject(account, cloudFile.Container, fileUrl);
return result;
}
catch
{
return false;
}
}
}
}
public class ShareAccountInfo
{
public DateTime? last_modified { get; set; }
public string name { get; set; }
}
}