#region
/* -----------------------------------------------------------------------
*
*
* Copyright 2011-2012 GRNET S.A. All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* 1. Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* 2. Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
*
* THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and
* documentation are those of the authors and should not be
* interpreted as representing official policies, either expressed
* or implied, of GRNET S.A.
*
* -----------------------------------------------------------------------
*/
#endregion
// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
//
// The class provides methods to upload/download files, delete files, manage containers
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.Diagnostics.Contracts;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Reflection;
using System.ServiceModel.Channels;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Pithos.Interfaces;
using Pithos.Network;
using log4net;
namespace Pithos.Network
{
[Export(typeof(ICloudClient))]
public class CloudFilesClient:ICloudClient,IDisposable
{
private const string TOKEN_HEADER = "X-Auth-Token";
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
//CloudFilesClient uses *_baseClient* internally to communicate with the server
//RestClient provides a REST-friendly interface over the standard WebClient.
private RestClient _baseClient;
private HttpClient _baseHttpClient;
private HttpClient _baseHttpClientNoTimeout;
//During authentication the client provides a UserName
public string UserName { get; set; }
//and and ApiKey to the server
public string ApiKey { get; set; }
//And receives an authentication Token. This token must be provided in ALL other operations,
//in the X-Auth-Token header
private string _token;
private readonly string _emptyGuid = Guid.Empty.ToString();
private readonly Uri _emptyUri = new Uri("",UriKind.Relative);
private HttpClientHandler _httpClientHandler = new HttpClientHandler
{
AllowAutoRedirect = true,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
UseCookies = true,
};
public string Token
{
get { return _token; }
set
{
_token = value;
_baseClient.Headers[TOKEN_HEADER] = value;
}
}
//The client also receives a StorageUrl after authentication. All subsequent operations must
//use this url
public Uri StorageUrl { get; set; }
public Uri RootAddressUri { get; set; }
public double DownloadPercentLimit { get; set; }
public double UploadPercentLimit { get; set; }
public string AuthenticationUrl { get; set; }
public string VersionPath
{
get { return UsePithos ? "v1" : "v1.0"; }
}
public bool UsePithos { get; set; }
BufferManager _bufferManager=BufferManager.CreateBufferManager(TreeHash.DEFAULT_BLOCK_SIZE*4,(int)TreeHash.DEFAULT_BLOCK_SIZE);
private string _userCatalogUrl;
public CloudFilesClient(string userName, string apiKey)
{
UserName = userName;
ApiKey = apiKey;
_userCatalogUrl = "https://pithos.okeanos.io/user_catalogs";
}
public CloudFilesClient(AccountInfo accountInfo)
{
Contract.Requires(accountInfo!=null,"accountInfo is null");
Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
Contract.Ensures(StorageUrl != null);
Contract.Ensures(_baseClient != null);
Contract.Ensures(RootAddressUri != null);
Contract.EndContractBlock();
_baseClient = new RestClient
{
BaseAddress = accountInfo.StorageUri.ToString(),
Timeout = 30000,
Retries = 3,
};
StorageUrl = accountInfo.StorageUri;
Token = accountInfo.Token;
UserName = accountInfo.UserName;
_userCatalogUrl = "https://pithos.okeanos.io/user_catalogs";
//Get the root address (StorageUrl without the account)
var storageUrl = StorageUrl.AbsoluteUri;
var usernameIndex = storageUrl.LastIndexOf(UserName);
var rootUrl = storageUrl.Substring(0, usernameIndex);
RootAddressUri = new Uri(rootUrl);
var httpClientHandler = new HttpClientHandler
{
AllowAutoRedirect = true,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
UseCookies = true,
};
_baseHttpClient = new HttpClient(httpClientHandler)
{
BaseAddress = StorageUrl,
Timeout = TimeSpan.FromSeconds(30)
};
_baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);
_baseHttpClientNoTimeout = new HttpClient(httpClientHandler)
{
BaseAddress = StorageUrl,
Timeout = TimeSpan.FromMilliseconds(-1)
};
_baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);
}
private static void AssertStatusOK(HttpResponseMessage response, string message)
{
var statusCode = response.StatusCode;
if (statusCode >= HttpStatusCode.BadRequest)
throw new WebException(String.Format("{0} with code {1} - {2}", message, statusCode, response.ReasonPhrase));
}
public async Task Authenticate()
{
if (String.IsNullOrWhiteSpace(UserName))
throw new InvalidOperationException("UserName is empty");
if (String.IsNullOrWhiteSpace(ApiKey))
throw new InvalidOperationException("ApiKey is empty");
if (String.IsNullOrWhiteSpace(AuthenticationUrl))
throw new InvalidOperationException("AuthenticationUrl is empty");
Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
Contract.Ensures(StorageUrl != null);
Contract.Ensures(_baseClient != null);
Contract.Ensures(RootAddressUri != null);
Contract.EndContractBlock();
Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
var groups = new List();
using (var authClient = new HttpClient(_httpClientHandler,false){ BaseAddress = new Uri(AuthenticationUrl),Timeout=TimeSpan.FromSeconds(30) })
{
authClient.DefaultRequestHeaders.Add("X-Auth-User", UserName);
authClient.DefaultRequestHeaders.Add("X-Auth-Key", ApiKey);
string storageUrl;
string token;
using (var response = await authClient.GetAsyncWithRetries(new Uri(VersionPath, UriKind.Relative),3).ConfigureAwait(false)) // .DownloadStringWithRetryRelative(new Uri(VersionPath, UriKind.Relative), 3);
{
AssertStatusOK(response,"Authentication failed");
storageUrl = response.Headers.GetFirstValue("X-Storage-Url");
if (String.IsNullOrWhiteSpace(storageUrl))
throw new InvalidOperationException("Failed to obtain storage url");
token = response.Headers.GetFirstValue(TOKEN_HEADER);
if (String.IsNullOrWhiteSpace(token))
throw new InvalidOperationException("Failed to obtain token url");
}
_baseClient = new RestClient
{
BaseAddress = storageUrl,
Timeout = 30000,
Retries = 3,
};
StorageUrl = new Uri(storageUrl);
Token = token;
//Get the root address (StorageUrl without the account)
var usernameIndex=storageUrl.LastIndexOf(UserName);
var rootUrl = storageUrl.Substring(0, usernameIndex);
RootAddressUri = new Uri(rootUrl);
_baseHttpClient = new HttpClient(_httpClientHandler,false)
{
BaseAddress = StorageUrl,
Timeout = TimeSpan.FromSeconds(30)
};
_baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, token);
_baseHttpClientNoTimeout = new HttpClient(_httpClientHandler,false)
{
BaseAddress = StorageUrl,
Timeout = TimeSpan.FromMilliseconds(-1)
};
_baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, token);
/* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
groups = (from key in keys
where key.StartsWith("X-Account-Group-")
let name = key.Substring(16)
select new Group(name, authClient.ResponseHeaders[key]))
.ToList();
*/
}
Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
Debug.Assert(_baseClient!=null);
var displayName = UserName;
Guid uuid;
if (Guid.TryParse(UserName, out uuid))
{
displayName = await ResolveName(uuid);
}
return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,DisplayName=displayName,Groups=groups};
}
private static void TraceStart(string method, Uri actualAddress)
{
Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
}
private async Task GetStringAsync(Uri targetUri, string errorMessage,DateTimeOffset? since=null)
{
TraceStart("GET",targetUri);
var request = new HttpRequestMessage(HttpMethod.Get, targetUri);
//request.Headers.Add("User-Agent", "Pithos+ Custom Header");
if (since.HasValue)
{
request.Headers.IfModifiedSince = since.Value;
}
using (var response = await _baseHttpClient.SendAsyncWithRetries(request,3).ConfigureAwait(false))
{
AssertStatusOK(response, errorMessage);
if (response.StatusCode == HttpStatusCode.NoContent)
return String.Empty;
var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
return content;
}
}
public async Task> ListContainers(string account)
{
var targetUrl = GetTargetUrl(account);
var targetUri = new Uri(String.Format("{0}?format=json", targetUrl));
var result = await GetStringAsync(targetUri, "List Containers failed").ConfigureAwait(false);
if (String.IsNullOrWhiteSpace(result))
return new List();
var infos = JsonConvert.DeserializeObject>(result);
foreach (var info in infos)
{
info.Account = account;
}
return infos;
}
private string GetAccountUrl(string account)
{
return RootAddressUri.Combine(account).AbsoluteUri;
}
public IList ListSharingAccounts(DateTime? since=null)
{
using (ThreadContext.Stacks["Share"].Push("List Accounts"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
var targetUri = new Uri(String.Format("{0}?format=json", RootAddressUri), UriKind.Absolute);
var content=TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListSharingAccounts failed", since).ConfigureAwait(false)).Result;
//If the result is empty, return an empty list,
var infos = String.IsNullOrWhiteSpace(content)
? new List()
//Otherwise deserialize the account list into a list of ShareAccountInfos
: JsonConvert.DeserializeObject>(content);
Log.DebugFormat("END");
return infos;
}
}
///
/// Request listing of all objects in a container modified since a specific time.
/// If the *since* value is missing, return all objects
///
/// Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new
/// and should be polled anyway
///
///
///
public IList ListSharedObjects(HashSet knownContainers, DateTimeOffset? since)
{
using (ThreadContext.Stacks["Share"].Push("List Objects"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
//'since' is not used here because we need to have ListObjects return a NoChange result
//for all shared accounts,containers
Func getKey = c => String.Format("{0}\\{1}", c.Account, c.Name);
var containers = (from account in ListSharingAccounts()
let conts = TaskEx.Run(async ()=>await ListContainers(account.name).ConfigureAwait(false)).Result
from container in conts
select container).ToList();
var items = from container in containers
let actualSince=knownContainers.Contains(getKey(container))?since:null
select ListObjects(container.Account , container.Name, actualSince);
var objects=items.SelectMany(r=> r).ToList();
//For each object
//Check parents recursively up to (but not including) the container.
//If parents are missing, add them to the list
//Need function to calculate all parent URLs
objects = AddMissingParents(objects);
//Store any new containers
foreach (var container in containers)
{
knownContainers.Add(getKey(container));
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
return objects;
}
}
private List AddMissingParents(List objects)
{
//TODO: Remove short-circuit when we decide to use Missing Parents functionality
//return objects;
var existingUris = objects.ToDictionary(o => o.Uri, o => o);
foreach (var objectInfo in objects)
{
//Can be null when retrieving objects to show in selective sync
if (objectInfo.Name == null)
continue;
//No need to unescape here, the parts will be used to create new ObjectInfos
var parts = objectInfo.Name.ToString().Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries);
//If there is no parent, skip
if (parts.Length == 1)
continue;
var baseParts = new[]
{
objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container.ToString()
};
for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++)
{
var nameparts = parts.Range(0, partIdx).ToArray();
var parentName= String.Join("/", nameparts);
var parentParts = baseParts.Concat(nameparts);
var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts);
var parentUri = new Uri(parentUrl, UriKind.Absolute);
ObjectInfo existingInfo;
if (!existingUris.TryGetValue(parentUri,out existingInfo))
{
var h = parentUrl.GetHashCode();
var reverse = new string(parentUrl.Reverse().ToArray());
var rh = reverse.GetHashCode();
var b1 = BitConverter.GetBytes(h);
var b2 = BitConverter.GetBytes(rh);
var g = new Guid(0,0,0,b1.Concat(b2).ToArray());
existingUris[parentUri] = new ObjectInfo
{
Account = objectInfo.Account,
Container = objectInfo.Container,
Content_Type = ObjectInfo.CONTENT_TYPE_DIRECTORY,
ETag = Signature.MERKLE_EMPTY,
X_Object_Hash = Signature.MERKLE_EMPTY,
Name=new Uri(parentName,UriKind.Relative),
StorageUri=objectInfo.StorageUri,
Bytes = 0,
UUID=g.ToString(),
};
}
}
}
return existingUris.Values.ToList();
}
public void SetTags(ObjectInfo target,IDictionary tags)
{
Contract.Requires(!String.IsNullOrWhiteSpace(Token),"The Token is not set");
Contract.Requires(StorageUrl != null,"The StorageUrl is not set");
Contract.Requires(target != null,"target is null");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Share"].Push("Share Object"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using (var client = new RestClient(_baseClient))
{
client.BaseAddress = GetAccountUrl(target.Account);
client.Parameters.Clear();
client.Parameters.Add("update", "");
foreach (var tag in tags)
{
var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
client.Headers.Add(headerTag, tag.Value);
}
client.DownloadStringWithRetryRelative(target.Container, 3);
client.AssertStatusOK("SetTags failed");
//If the status is NOT ACCEPTED we have a problem
if (client.StatusCode != HttpStatusCode.Accepted)
{
Log.Error("Failed to set tags");
throw new Exception("Failed to set tags");
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
}
}
}
public void ShareObject(string account, Uri container, Uri objectName, string shareTo, bool read, bool write)
{
Contract.Requires(!String.IsNullOrWhiteSpace(Token), "The Token is not set");
Contract.Requires(StorageUrl != null, "The StorageUrl is not set");
Contract.Requires(container != null, "container is null");
Contract.Requires(!container.IsAbsoluteUri, "container is absolute");
Contract.Requires(objectName != null, "objectName is null");
Contract.Requires(!objectName.IsAbsoluteUri, "objectName is absolute");
Contract.Requires(!String.IsNullOrWhiteSpace(account), "account is not set");
Contract.Requires(!String.IsNullOrWhiteSpace(shareTo), "shareTo is not set");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Share"].Push("Share Object"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using (var client = new RestClient(_baseClient))
{
client.BaseAddress = GetAccountUrl(account);
client.Parameters.Clear();
client.Parameters.Add("format", "json");
string permission = "";
if (write)
permission = String.Format("write={0}", shareTo);
else if (read)
permission = String.Format("read={0}", shareTo);
client.Headers.Add("X-Object-Sharing", permission);
var content = client.DownloadStringWithRetryRelative(container, 3);
client.AssertStatusOK("ShareObject failed");
//If the result is empty, return an empty list,
var infos = String.IsNullOrWhiteSpace(content)
? new List()
//Otherwise deserialize the object list into a list of ObjectInfos
: JsonConvert.DeserializeObject>(content);
if (Log.IsDebugEnabled) Log.DebugFormat("END");
}
}
}
public async Task GetAccountPolicies(AccountInfo accountInfo)
{
if (accountInfo==null)
throw new ArgumentNullException("accountInfo");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Account"].Push("GetPolicies"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
/*
if (_baseClient == null)
{
_baseClient = new RestClient
{
BaseAddress = accountInfo.StorageUri.ToString(),
Timeout = 30000,
Retries = 3,
};
}
*/
var containerUri = GetTargetUri(accountInfo.UserName);
var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);
using(var response=await _baseHttpClient.HeadAsyncWithRetries(targetUri,3).ConfigureAwait(false))
{
var quotaValue=response.Headers.GetFirstValue("X-Account-Policy-Quota");
var bytesValue = response.Headers.GetFirstValue("X-Account-Bytes-Used");
long quota, bytes;
if (long.TryParse(quotaValue, out quota))
accountInfo.Quota = quota;
if (long.TryParse(bytesValue, out bytes))
accountInfo.BytesUsed = bytes;
return accountInfo;
}
//using (var client = new RestClient(_baseClient))
//{
// if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
// client.BaseAddress = GetAccountUrl(accountInfo.UserName);
// client.Parameters.Clear();
// client.Parameters.Add("format", "json");
// client.Head(_emptyUri, 3);
// var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
// var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
// long quota, bytes;
// if (long.TryParse(quotaValue, out quota))
// accountInfo.Quota = quota;
// if (long.TryParse(bytesValue, out bytes))
// accountInfo.BytesUsed = bytes;
// return accountInfo;
//}
}
}
public void UpdateMetadata(ObjectInfo objectInfo)
{
Contract.Requires(objectInfo != null,"objectInfo is null");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using(var client=new RestClient(_baseClient))
{
client.BaseAddress = GetAccountUrl(objectInfo.Account);
client.Parameters.Clear();
//Set Tags
foreach (var tag in objectInfo.Tags)
{
var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
client.Headers.Add(headerTag, tag.Value);
}
//Set Permissions
var permissions=objectInfo.GetPermissionString();
client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);
client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
var isPublic = objectInfo.IsPublic.ToString().ToLower();
client.Headers.Add("X-Object-Public", isPublic);
var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);
client.PostWithRetry(new Uri(address,UriKind.Relative),"application/xml");
client.AssertStatusOK("UpdateMetadata failed");
//If the status is NOT ACCEPTED or OK we have a problem
if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
{
Log.Error("Failed to update metadata");
throw new Exception("Failed to update metadata");
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
}
}
}
public void UpdateMetadata(ContainerInfo containerInfo)
{
if (containerInfo == null)
throw new ArgumentNullException("containerInfo");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
using(var client=new RestClient(_baseClient))
{
client.BaseAddress = GetAccountUrl(containerInfo.Account);
client.Parameters.Clear();
//Set Tags
foreach (var tag in containerInfo.Tags)
{
var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
client.Headers.Add(headerTag, tag.Value);
}
//Set Policies
foreach (var policy in containerInfo.Policies)
{
var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
client.Headers.Add(headerPolicy, policy.Value);
}
var uriBuilder = client.GetAddressBuilder(containerInfo.Name,_emptyUri);
var uri = uriBuilder.Uri;
client.UploadValues(uri,new NameValueCollection());
client.AssertStatusOK("UpdateMetadata failed");
//If the status is NOT ACCEPTED or OK we have a problem
if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
{
Log.Error("Failed to update metadata");
throw new Exception("Failed to update metadata");
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
}
}
}
public IList ListObjects(string account, Uri container, DateTimeOffset? since = null)
{
/*
if (container==null)
throw new ArgumentNullException("container");
if (container.IsAbsoluteUri)
throw new ArgumentException("container");
Contract.EndContractBlock();
*/
using (ThreadContext.Stacks["Objects"].Push("List"))
{
var containerUri = GetTargetUri(account).Combine(container);
var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);
var content =TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result;
//304 will result in an empty string. Empty containers return an empty json array
if (String.IsNullOrWhiteSpace(content))
return new[] {new NoModificationInfo(account, container)};
//If the result is empty, return an empty list,
var infos = String.IsNullOrWhiteSpace(content)
? new List()
//Otherwise deserialize the object list into a list of ObjectInfos
: JsonConvert.DeserializeObject>(content);
foreach (var info in infos)
{
info.Container = container;
info.Account = account;
info.StorageUri = StorageUrl;
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
return infos;
}
}
public IList ListObjects(string account, Uri container, Uri folder, DateTimeOffset? since = null)
{
/* if (container==null)
throw new ArgumentNullException("container");
if (container.IsAbsoluteUri)
throw new ArgumentException("container");
Contract.EndContractBlock();*/
using (ThreadContext.Stacks["Objects"].Push("List"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
var containerUri = GetTargetUri(account).Combine(container);
var targetUri = new Uri(String.Format("{0}?format=json&path={1}", containerUri,folder), UriKind.Absolute);
var content = TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result;
//304 will result in an empty string. Empty containers return an empty json array
if (String.IsNullOrWhiteSpace(content))
return new[] { new NoModificationInfo(account, container) };
var infos = JsonConvert.DeserializeObject>(content);
foreach (var info in infos)
{
info.Account = account;
if (info.Container == null)
info.Container = container;
info.StorageUri = StorageUrl;
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
return infos;
/*
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.Parameters.Clear();
client.Parameters.Add("format", "json");
client.Parameters.Add("path", folder.ToString());
client.IfModifiedSince = since;
var content = client.DownloadStringWithRetryRelative(container, 3);
client.AssertStatusOK("ListObjects failed");
if (client.StatusCode==HttpStatusCode.NotModified)
return new[]{new NoModificationInfo(account,container,folder)};
var infos = JsonConvert.DeserializeObject>(content);
foreach (var info in infos)
{
info.Account = account;
if (info.Container == null)
info.Container = container;
info.StorageUri = StorageUrl;
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
return infos;
}
*/
}
}
public async Task ContainerExists(string account, Uri container)
{
if (container==null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException( "The container must be relative","container");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Containters"].Push("Exists"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
var targetUri = GetTargetUri(account).Combine(container);
using (var response =await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3))
{
bool result;
switch (response.StatusCode)
{
case HttpStatusCode.OK:
case HttpStatusCode.NoContent:
result = true;
break;
case HttpStatusCode.NotFound:
result = false;
break;
default:
throw CreateWebException("ContainerExists", response.StatusCode);
}
if (Log.IsDebugEnabled) Log.DebugFormat("END");
return result;
}
}
}
private Uri GetTargetUri(string account)
{
return new Uri(GetTargetUrl(account),UriKind.Absolute);
}
private string GetTargetUrl(string account)
{
return String.IsNullOrWhiteSpace(account)
? _baseHttpClient.BaseAddress.ToString()
: GetAccountUrl(account);
}
public async Task ObjectExists(string account, Uri container, Uri objectName)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative","container");
if (objectName == null)
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
if (objectName.IsAbsoluteUri)
throw new ArgumentException("The objectName must be relative","objectName");
Contract.EndContractBlock();
var targetUri=GetTargetUri(account).Combine(container).Combine(objectName);
using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).ConfigureAwait(false))
{
switch (response.StatusCode)
{
case HttpStatusCode.OK:
case HttpStatusCode.NoContent:
return true;
case HttpStatusCode.NotFound:
return false;
default:
throw CreateWebException("ObjectExists", response.StatusCode);
}
}
}
public async Task GetObjectInfo(string account, Uri container, Uri objectName)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative", "container");
if (objectName == null)
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
if (objectName.IsAbsoluteUri)
throw new ArgumentException("The objectName must be relative", "objectName");
Contract.EndContractBlock();
using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
{
var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);
try
{
using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true))
{
switch (response.StatusCode)
{
case HttpStatusCode.OK:
case HttpStatusCode.NoContent:
var tags = response.Headers.GetMeta("X-Object-Meta-");
var extensions = (from header in response.Headers
where
header.Key.StartsWith("X-Object-") &&
!header.Key.StartsWith("X-Object-Meta-")
select new {Name = header.Key, Value = header.Value.FirstOrDefault()})
.ToDictionary(t => t.Name, t => t.Value);
var permissions = response.Headers.GetFirstValue("X-Object-Sharing");
var info = new ObjectInfo
{
Account = account,
Container = container,
Name = objectName,
ETag = response.Headers.ETag.NullSafe(e=>e.Tag),
UUID = response.Headers.GetFirstValue("X-Object-UUID"),
X_Object_Hash = response.Headers.GetFirstValue("X-Object-Hash"),
Content_Type = response.Headers.GetFirstValue("Content-Type"),
Bytes = Convert.ToInt64(response.Content.Headers.ContentLength),
Tags = tags,
Last_Modified = response.Content.Headers.LastModified,
Extensions = extensions,
ContentEncoding =
response.Content.Headers.ContentEncoding.FirstOrDefault(),
ContendDisposition =
response.Content.Headers.ContentDisposition.NullSafe(c=>c.ToString()),
Manifest = response.Headers.GetFirstValue("X-Object-Manifest"),
PublicUrl = response.Headers.GetFirstValue("X-Object-Public"),
StorageUri = StorageUrl,
};
info.SetPermissions(permissions);
return info;
case HttpStatusCode.NotFound:
return ObjectInfo.Empty;
default:
throw new WebException(
String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
objectName, response.StatusCode));
}
}
}
catch (RetryException)
{
Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.", objectName);
return ObjectInfo.Empty;
}
catch (WebException e)
{
Log.Error(
String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status {1}",
objectName, e.Status), e);
throw;
}
}
}
public async Task CreateFolder(string account, Uri container, Uri folder)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative","container");
if (folder == null)
throw new ArgumentNullException("folder", "The objectName property can't be empty");
if (folder.IsAbsoluteUri)
throw new ArgumentException("The objectName must be relative","folder");
Contract.EndContractBlock();
var folderUri=container.Combine(folder);
var targetUri = GetTargetUri(account).Combine(folderUri);
var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
message.Content=new StringContent("");
message.Content.Headers.ContentType = new MediaTypeHeaderValue(ObjectInfo.CONTENT_TYPE_DIRECTORY);
//message.Headers.Add("Content-Length", "0");
using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
{
if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)
throw CreateWebException("CreateFolder", response.StatusCode);
}
}
private Dictionary GetMeta(HttpResponseMessage response,string metaPrefix)
{
Contract.Requires(!String.IsNullOrWhiteSpace(metaPrefix), "metaPrefix is empty");
Contract.EndContractBlock();
var dict = (from header in response.Headers
where header.Key.StartsWith(metaPrefix)
select new { Name = header.Key, Value = String.Join(",", header.Value) })
.ToDictionary(t => t.Name, t => t.Value);
return dict;
}
public async Task GetContainerInfo(string account, Uri container)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative","container");
Contract.EndContractBlock();
var targetUri = GetTargetUri(account).Combine(container);
using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true).ConfigureAwait(false))
{
if (Log.IsDebugEnabled)
Log.DebugFormat("ContainerInfo data: {0}\n{1}",response,await response.Content.ReadAsStringAsync().ConfigureAwait(false));
switch (response.StatusCode)
{
case HttpStatusCode.OK:
case HttpStatusCode.NoContent:
var tags = GetMeta(response,"X-Container-Meta-");
var policies = GetMeta(response,"X-Container-Policy-");
var containerInfo = new ContainerInfo
{
Account = account,
Name = container,
StorageUrl = StorageUrl.ToString(),
Count =long.Parse(response.Headers.GetFirstValue("X-Container-Object-Count")),
Bytes = long.Parse(response.Headers.GetFirstValue("X-Container-Bytes-Used")),
BlockHash = response.Headers.GetFirstValue("X-Container-Block-Hash"),
BlockSize =
int.Parse(response.Headers.GetFirstValue("X-Container-Block-Size")),
Last_Modified = response.Content.Headers.LastModified,
Tags = tags,
Policies = policies
};
return containerInfo;
case HttpStatusCode.NotFound:
return ContainerInfo.Empty;
default:
throw CreateWebException("GetContainerInfo", response.StatusCode);
}
}
}
public async Task CreateContainer(string account, Uri container)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative","container");
Contract.EndContractBlock();
var targetUri=GetTargetUri(account).Combine(container);
var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
//message.Content.Headers.ContentLength = 0;
using (var response =await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
{
var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
if (!expectedCodes.Contains(response.StatusCode))
throw CreateWebException("CreateContainer", response.StatusCode);
}
}
public async Task WipeContainer(string account, Uri container)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative", "container");
Contract.EndContractBlock();
await DeleteContainer(account, new Uri(String.Format("{0}?delimiter=/", container), UriKind.Relative)).ConfigureAwait(false);
}
public async Task DeleteContainer(string account, Uri container)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative","container");
Contract.EndContractBlock();
var targetUri = GetTargetUri(account).Combine(container);
var message = new HttpRequestMessage(HttpMethod.Delete, targetUri);
using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
{
var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent };
if (!expectedCodes.Contains(response.StatusCode))
throw CreateWebException("DeleteContainer", response.StatusCode);
}
}
///
///
///
///
///
///
///
///
///
/// This method should have no timeout or a very long one
//Asynchronously download the object specified by *objectName* in a specific *container* to
// a local file
public async Task GetObject(string account, Uri container, Uri objectName, string fileName,CancellationToken cancellationToken)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative","container");
if (objectName == null)
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
if (objectName.IsAbsoluteUri)
throw new ArgumentException("The objectName must be relative","objectName");
Contract.EndContractBlock();
try
{
//WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
//object to avoid concurrency errors.
//
//Download operations take a long time therefore they have no timeout.
using(var client = new RestClient(_baseClient) { Timeout = 0 })
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
//The container and objectName are relative names. They are joined with the client's
//BaseAddress to create the object's absolute address
var builder = client.GetAddressBuilder(container, objectName);
var uri = builder.Uri;
//Download progress is reported to the Trace log
Log.InfoFormat("[GET] START {0}", objectName);
/*client.DownloadProgressChanged += (sender, args) =>
Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
fileName, args.ProgressPercentage,
args.BytesReceived,
args.TotalBytesToReceive);*/
var progress = new Progress(args =>
{
Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
fileName, args.ProgressPercentage,
args.BytesReceived,
args.TotalBytesToReceive);
if (DownloadProgressChanged!=null)
DownloadProgressChanged(this, new DownloadArgs(args));
});
//Start downloading the object asynchronously
await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false);
//Once the download completes
//Delete the local client object
}
//And report failure or completion
}
catch (Exception exc)
{
Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);
throw;
}
Log.InfoFormat("[GET] END {0}", objectName);
}
public async Task> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative","container");
if (objectName == null)
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
if (objectName.IsAbsoluteUri)
throw new ArgumentException("The objectName must be relative","objectName");
if (hash == null)
throw new ArgumentNullException("hash");
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("Invalid Token");
if (StorageUrl == null)
throw new InvalidOperationException("Invalid Storage Url");
Contract.EndContractBlock();
//The container and objectName are relative names. They are joined with the client's
//BaseAddress to create the object's absolute address
var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);
var uri = new Uri(String.Format("{0}?format=json&hashmap",targetUri),UriKind.Absolute);
//Send the tree hash as Json to the server
var jsonHash = hash.ToJson();
if (Log.IsDebugEnabled)
Log.DebugFormat("Hashes:\r\n{0}", jsonHash);
var mimeType = objectName.GetMimeType();
var message = new HttpRequestMessage(HttpMethod.Put, uri)
{
Content = new StringContent(jsonHash)
};
message.Content.Headers.ContentType = mimeType;
message.Headers.Add("ETag",hash.TopHash.ToHashString());
//Don't use a timeout because putting the hashmap may be a long process
using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
{
var empty = (IList)new List();
switch (response.StatusCode)
{
case HttpStatusCode.Created:
//The server will respond either with 201-created if all blocks were already on the server
return empty;
case HttpStatusCode.Conflict:
//or with a 409-conflict and return the list of missing parts
using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
using(var reader=stream.GetLoggedReader(Log))
{
var serializer = new JsonSerializer();
serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);
var hashes = (List)serializer.Deserialize(reader, typeof(List));
return hashes;
}
default:
//All other cases are unexpected
//Ensure that failure codes raise exceptions
response.EnsureSuccessStatusCode();
//And log any other codes as warngings, but continute processing
Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",response.StatusCode,response.ReasonPhrase);
return empty;
}
}
}
public async Task GetBlock(string account, Uri container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)
{
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("Invalid Token");
if (StorageUrl == null)
throw new InvalidOperationException("Invalid Storage Url");
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative","container");
if (relativeUrl == null)
throw new ArgumentNullException("relativeUrl");
if (end.HasValue && end < 0)
throw new ArgumentOutOfRangeException("end");
if (start < 0)
throw new ArgumentOutOfRangeException("start");
Contract.EndContractBlock();
var targetUri = GetTargetUri(account).Combine(container).Combine(relativeUrl);
var message = new HttpRequestMessage(HttpMethod.Get, targetUri);
//Don't add a range if start=0, end=null (empty files)
if (start!=0 || end!=null)
message.Headers.Range=new RangeHeaderValue(start,end);
//Don't use a timeout because putting the hashmap may be a long process
IProgress progress = new Progress(args =>
{
Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
targetUri.Segments.Last(), args.ProgressPercentage,
args.BytesReceived,
args.TotalBytesToReceive);
if (DownloadProgressChanged!=null)
DownloadProgressChanged(this, args);
});
using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false,HttpCompletionOption.ResponseHeadersRead,
cancellationToken).ConfigureAwait(false))
using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
{
long totalSize = response.Content.Headers.ContentLength ?? 0;
byte[] buffer,streambuf;
lock (_bufferManager)
{
buffer = _bufferManager.TakeBuffer(65536);
streambuf = _bufferManager.TakeBuffer((int)totalSize);
}
using (var targetStream = new MemoryStream(streambuf))
{
long total = 0;
try
{
int read;
while ((read = await stream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) != 0)
{
total += read;
progress.Report(new DownloadArgs(total, totalSize));
await targetStream.WriteAsync(buffer, 0, read).ConfigureAwait(false);
}
}
finally
{
lock (_bufferManager)
{
_bufferManager.ReturnBuffer(buffer);
_bufferManager.ReturnBuffer(streambuf);
}
}
var result = targetStream.ToArray();
return result;
}
}
}
public event EventHandler UploadProgressChanged;
public event EventHandler DownloadProgressChanged;
public async Task PostBlock(string account, Uri container, byte[] block, int offset, int count,string blockHash,CancellationToken token)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative","container");
if (block == null)
throw new ArgumentNullException("block");
if (offset < 0 || offset >= block.Length)
throw new ArgumentOutOfRangeException("offset");
if (count < 0 || count > block.Length)
throw new ArgumentOutOfRangeException("count");
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("Invalid Token");
if (StorageUrl == null)
throw new InvalidOperationException("Invalid Storage Url");
Contract.EndContractBlock();
try
{
var containerUri = GetTargetUri(account).Combine(container);
var targetUri = new Uri(String.Format("{0}?update", containerUri));
//Don't use a timeout because putting the hashmap may be a long process
Log.InfoFormat("[BLOCK POST] START");
var progress = new Progress(args =>
{
Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
args.ProgressPercentage,
args.BytesSent,
args.TotalBytesToSend);
if (UploadProgressChanged != null)
UploadProgressChanged(this,args);
});
var message = new HttpRequestMessage(HttpMethod.Post, targetUri)
{
Content = new ByteArrayContentWithProgress(block, offset, count,progress)
};
message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream");
//Send the block
using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3,false,HttpCompletionOption.ResponseContentRead,token).ConfigureAwait(false))
{
Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
response.EnsureSuccessStatusCode();
var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
var cleanHash = responseHash.TrimEnd();
Debug.Assert(blockHash==cleanHash);
if (!blockHash.Equals(cleanHash, StringComparison.OrdinalIgnoreCase))
Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]", account, container, blockHash, cleanHash);
}
Log.InfoFormat("[BLOCK POST] END");
}
catch (TaskCanceledException )
{
Log.Info("Aborting block");
throw;
}
catch (Exception exc)
{
Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);
throw;
}
}
public async Task PostBlock(string account, Uri container, string filePath, long offset, int count, string blockHash, CancellationToken token)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative", "container");
if (String.IsNullOrWhiteSpace(filePath))
throw new ArgumentNullException("filePath");
if (!File.Exists(filePath))
throw new FileNotFoundException("Missing file","filePath");
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("Invalid Token");
if (StorageUrl == null)
throw new InvalidOperationException("Invalid Storage Url");
Contract.EndContractBlock();
try
{
var containerUri = GetTargetUri(account).Combine(container);
var targetUri = new Uri(String.Format("{0}?update", containerUri));
//Don't use a timeout because putting the hashmap may be a long process
Log.InfoFormat("[BLOCK POST] START");
var progress = new Progress(args =>
{
Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2} at {3:###,} Kbps ",
args.ProgressPercentage,
args.BytesSent,
args.TotalBytesToSend,args.Speed);
if (UploadProgressChanged != null)
UploadProgressChanged(this, args);
});
var message = new HttpRequestMessage(HttpMethod.Post, targetUri)
{
Content = new FileBlockContent(filePath, offset, count, progress)
};
message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream");
//Send the block
using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false, HttpCompletionOption.ResponseContentRead, token).ConfigureAwait(false))
{
Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
response.EnsureSuccessStatusCode();
var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
var cleanHash = responseHash.TrimEnd();
Debug.Assert(blockHash == cleanHash);
if (!blockHash.Equals(cleanHash, StringComparison.OrdinalIgnoreCase))
Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]", account, container, blockHash, cleanHash);
}
Log.InfoFormat("[BLOCK POST] END");
}
catch (TaskCanceledException)
{
Log.Info("Aborting block");
throw;
}
catch (Exception exc)
{
Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);
throw;
}
}
public async Task GetHashMap(string account, Uri container, Uri objectName)
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative","container");
if (objectName == null)
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
if (objectName.IsAbsoluteUri)
throw new ArgumentException("The objectName must be relative","objectName");
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("Invalid Token");
if (StorageUrl == null)
throw new InvalidOperationException("Invalid Storage Url");
Contract.EndContractBlock();
try
{
var objectUri = GetTargetUri(account).Combine(container).Combine(objectName);
var targetUri = new Uri(String.Format("{0}?format=json&hashmap", objectUri));
//Start downloading the object asynchronously
var json = await GetStringAsync(targetUri, "").ConfigureAwait(false);
var treeHash = TreeHash.Parse(json);
Log.InfoFormat("[GET HASH] END {0}", objectName);
return treeHash;
}
catch (Exception exc)
{
Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
throw;
}
}
///
///
///
///
///
///
///
/// Optional hash value for the file. If no hash is provided, the method calculates a new hash
///
/// >This method should have no timeout or a very long one
public async Task PutObject(string account, Uri container, Uri objectName, string fileName, string hash = Signature.MERKLE_EMPTY, string contentType = "application/octet-stream")
{
if (container == null)
throw new ArgumentNullException("container", "The container property can't be empty");
if (container.IsAbsoluteUri)
throw new ArgumentException("The container must be relative","container");
if (objectName == null)
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
if (objectName.IsAbsoluteUri)
throw new ArgumentException("The objectName must be relative","objectName");
if (String.IsNullOrWhiteSpace(fileName))
throw new ArgumentNullException("fileName", "The fileName property can't be empty");
try
{
using (var client = new RestClient(_baseClient) { Timeout = 0 })
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
var builder = client.GetAddressBuilder(container, objectName);
var uri = builder.Uri;
string etag = hash ;
client.Headers.Add("Content-Type", contentType);
if (contentType!=ObjectInfo.CONTENT_TYPE_DIRECTORY)
client.Headers.Add("ETag", etag);
Log.InfoFormat("[PUT] START {0}", objectName);
client.UploadProgressChanged += (sender, args) =>
{
using (ThreadContext.Stacks["PUT"].Push("Progress"))
{
Log.InfoFormat("{0} {1}% {2} of {3}", fileName,
args.ProgressPercentage,
args.BytesSent, args.TotalBytesToSend);
}
};
client.UploadFileCompleted += (sender, args) =>
{
using (ThreadContext.Stacks["PUT"].Push("Progress"))
{
Log.InfoFormat("Completed {0}", fileName);
}
};
if (contentType==ObjectInfo.CONTENT_TYPE_DIRECTORY)
await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false);
else
await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false);
}
Log.InfoFormat("[PUT] END {0}", objectName);
}
catch (Exception exc)
{
Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
throw;
}
}
public async Task MoveObject(string account, Uri sourceContainer, Uri oldObjectName, Uri targetContainer, Uri newObjectName)
{
if (sourceContainer == null)
throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");
if (sourceContainer.IsAbsoluteUri)
throw new ArgumentException("The sourceContainer must be relative","sourceContainer");
if (oldObjectName == null)
throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
if (oldObjectName.IsAbsoluteUri)
throw new ArgumentException("The oldObjectName must be relative","oldObjectName");
if (targetContainer == null)
throw new ArgumentNullException("targetContainer", "The targetContainer property can't be empty");
if (targetContainer.IsAbsoluteUri)
throw new ArgumentException("The targetContainer must be relative","targetContainer");
if (newObjectName == null)
throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
if (newObjectName.IsAbsoluteUri)
throw new ArgumentException("The newObjectName must be relative","newObjectName");
Contract.EndContractBlock();
var baseUri = GetTargetUri(account);
var targetUri = baseUri.Combine(targetContainer).Combine(newObjectName);
var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, oldObjectName),UriKind.Relative);
var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
message.Headers.Add("X-Move-From", sourceUri.ToString());
using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
{
var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
if (!expectedCodes.Contains(response.StatusCode))
throw CreateWebException("MoveObject", response.StatusCode);
}
}
public async Task DeleteObject(string account, Uri sourceContainer, Uri objectName, bool isDirectory)
{
if (sourceContainer == null)
throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");
if (sourceContainer.IsAbsoluteUri)
throw new ArgumentException("The sourceContainer must be relative","sourceContainer");
if (objectName == null)
throw new ArgumentNullException("objectName", "The objectName property can't be empty");
if (objectName.IsAbsoluteUri)
throw new ArgumentException("The objectName must be relative","objectName");
Contract.EndContractBlock();
var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, objectName),UriKind.Relative);
if (objectName.OriginalString.EndsWith(".ignore"))
using(var response = await _baseHttpClient.DeleteAsync(sourceUri)){}
else
{
var relativeUri = new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName),
UriKind.Relative);
/*
var relativeUri = isDirectory
? new Uri(
String.Format("{0}/{1}?delimiter=/", FolderConstants.TrashContainer,
objectName), UriKind.Relative)
: new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName),
UriKind.Relative);
*/
var targetUri = GetTargetUri(account).Combine(relativeUri);
var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
message.Headers.Add("X-Move-From", sourceUri.ToString());
Log.InfoFormat("[TRASH] [{0}] to [{1}]", sourceUri, targetUri);
using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3))
{
var expectedCodes = new[]
{
HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,
HttpStatusCode.NotFound
};
if (!expectedCodes.Contains(response.StatusCode))
throw CreateWebException("DeleteObject", response.StatusCode);
}
}
/*
var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
/*
if (isDirectory)
targetUrl = targetUrl + "?delimiter=/";
#1#
var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
using (var client = new RestClient(_baseClient))
{
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
client.Headers.Add("X-Move-From", sourceUrl);
client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);
client.PutWithRetry(new Uri(targetUrl,UriKind.Relative), 3);
var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
if (!expectedCodes.Contains(client.StatusCode))
throw CreateWebException("DeleteObject", client.StatusCode);
}
*/
}
private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
{
return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
}
public async Task CanUpload(string account, ObjectInfo cloudFile)
{
Contract.Requires(!String.IsNullOrWhiteSpace(account));
Contract.Requires(cloudFile!=null);
var parts = cloudFile.Name.ToString().Split('/');
var folder = String.Join("/", parts,0,parts.Length-1);
var fileName = String.Format("{0}/{1}.pithos.ignore", folder, Guid.NewGuid());
var fileUri=fileName.ToEscapedUri();
try
{
var relativeUri = cloudFile.Container.Combine(fileUri);
var targetUri = GetTargetUri(account).Combine(relativeUri);
var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
message.Content.Headers.ContentType =new MediaTypeHeaderValue("application/octet-stream");
var response=await _baseHttpClient.SendAsyncWithRetries(message, 3);
var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
var result=(expectedCodes.Contains(response.StatusCode));
await DeleteObject(account, cloudFile.Container, fileUri, cloudFile.IsDirectory);
return result;
}
catch
{
return false;
}
}
~CloudFilesClient()
{
Dispose(false);
}
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
if (_httpClientHandler!=null)
_httpClientHandler.Dispose();
if (_baseClient!=null)
_baseClient.Dispose();
if(_baseHttpClient!=null)
_baseHttpClient.Dispose();
if (_baseHttpClientNoTimeout!=null)
_baseHttpClientNoTimeout.Dispose();
}
_httpClientHandler = null;
_baseClient = null;
_baseHttpClient = null;
_baseHttpClientNoTimeout = null;
}
public async Task ResolveName(Guid accountToken)
{
string format = string.Format("{{\"uuids\":[\"{0}\"]}}", accountToken);
var content = new StringContent(format,Encoding.UTF8);
//content.Headers.ContentType=new MediaTypeHeaderValue("text/html; charset=utf-8");
string catalogEntry;
var catalogUrl = new Uri(_baseHttpClient.BaseAddress.Scheme + "://" +_baseHttpClient.BaseAddress.Host,UriKind.Absolute).Combine("user_catalogs");
using (var response = await _baseHttpClient.PostAsync(catalogUrl, content).ConfigureAwait(false))
{
catalogEntry=await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
var entry = (JContainer)JsonConvert.DeserializeObject(catalogEntry);
string key = accountToken.ToString();
return (string)entry["uuid_catalog"][key];
}
public async Task ResolveToken(string displayName)
{
string format = string.Format("{{\"displaynames\":[\"{0}\"]}}", displayName);
var content = new StringContent(format,Encoding.UTF8);
//content.Headers.ContentType=new MediaTypeHeaderValue("text/html; charset=utf-8");
string catalogEntry;
var catalogUrl = new Uri(_baseHttpClient.BaseAddress.Scheme + "://" +_baseHttpClient.BaseAddress.Host,UriKind.Absolute).Combine("user_catalogs");
using (var response = await _baseHttpClient.PostAsync(catalogUrl, content).ConfigureAwait(false))
{
catalogEntry=await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
var entry = (JContainer)JsonConvert.DeserializeObject(catalogEntry);
return new Guid((string)entry["displayname_catalog"][displayName]);
}
}
public class ShareAccountInfo
{
public DateTime? last_modified { get; set; }
public string name { get; set; }
}
}