-// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
-//
-// The class provides methods to upload/download files, delete files, manage containers
-
-
-using System;
-using System.Collections.Generic;
-using System.ComponentModel.Composition;
-using System.Diagnostics.Contracts;
-using System.IO;
-using System.Linq;
-using System.Net;
-using System.Security.Cryptography;
-using System.Text;
-using System.Threading.Tasks;
-using Newtonsoft.Json;
-using Pithos.Interfaces;
-using log4net;
-
-namespace Pithos.Network
-{
- [Export(typeof(ICloudClient))]
- public class CloudFilesClient:ICloudClient
- {
- //CloudFilesClient uses *_baseClient* internally to communicate with the server
- //RestClient provides a REST-friendly interface over the standard WebClient.
- private RestClient _baseClient;
-
-
- //During authentication the client provides a UserName
- public string UserName { get; set; }
-
- //and and ApiKey to the server
- public string ApiKey { get; set; }
-
- //And receives an authentication Token. This token must be provided in ALL other operations,
- //in the X-Auth-Token header
- private string _token;
- public string Token
- {
- get { return _token; }
- set
- {
- _token = value;
- _baseClient.Headers["X-Auth-Token"] = value;
- }
- }
-
- //The client also receives a StorageUrl after authentication. All subsequent operations must
- //use this url
- public Uri StorageUrl { get; set; }
-
-
- protected Uri RootAddressUri { get; set; }
-
- private Uri _proxy;
- public Uri Proxy
- {
- get { return _proxy; }
- set
- {
- _proxy = value;
- if (_baseClient != null)
- _baseClient.Proxy = new WebProxy(value);
- }
- }
-
- public double DownloadPercentLimit { get; set; }
- public double UploadPercentLimit { get; set; }
-
- public string AuthenticationUrl { get; set; }
-
-
- public string VersionPath
- {
- get { return UsePithos ? "v1" : "v1.0"; }
- }
-
- public bool UsePithos { get; set; }
-
-
- private static readonly ILog Log = LogManager.GetLogger("CloudFilesClient");
-
- public CloudFilesClient(string userName, string apiKey)
- {
- UserName = userName;
- ApiKey = apiKey;
- }
-
- public CloudFilesClient(AccountInfo accountInfo)
- {
- if (accountInfo==null)
- throw new ArgumentNullException("accountInfo");
- Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
- Contract.Ensures(StorageUrl != null);
- Contract.Ensures(_baseClient != null);
- Contract.Ensures(RootAddressUri != null);
- Contract.EndContractBlock();
-
- _baseClient = new RestClient
- {
- BaseAddress = accountInfo.StorageUri.ToString(),
- Timeout = 10000,
- Retries = 3
- };
- StorageUrl = accountInfo.StorageUri;
- Token = accountInfo.Token;
- UserName = accountInfo.UserName;
-
- //Get the root address (StorageUrl without the account)
- var storageUrl = StorageUrl.AbsoluteUri;
- var usernameIndex = storageUrl.LastIndexOf(UserName);
- var rootUrl = storageUrl.Substring(0, usernameIndex);
- RootAddressUri = new Uri(rootUrl);
- }
-
-
- public AccountInfo Authenticate()
- {
- if (String.IsNullOrWhiteSpace(UserName))
- throw new InvalidOperationException("UserName is empty");
- if (String.IsNullOrWhiteSpace(ApiKey))
- throw new InvalidOperationException("ApiKey is empty");
- if (String.IsNullOrWhiteSpace(AuthenticationUrl))
- throw new InvalidOperationException("AuthenticationUrl is empty");
- Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
- Contract.Ensures(StorageUrl != null);
- Contract.Ensures(_baseClient != null);
- Contract.Ensures(RootAddressUri != null);
- Contract.EndContractBlock();
-
-
- Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
-
- using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
- {
- if (Proxy != null)
- authClient.Proxy = new WebProxy(Proxy);
-
- Contract.Assume(authClient.Headers!=null);
-
- authClient.Headers.Add("X-Auth-User", UserName);
- authClient.Headers.Add("X-Auth-Key", ApiKey);
-
- authClient.DownloadStringWithRetry(VersionPath, 3);
-
- authClient.AssertStatusOK("Authentication failed");
-
- var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
- if (String.IsNullOrWhiteSpace(storageUrl))
- throw new InvalidOperationException("Failed to obtain storage url");
-
- _baseClient = new RestClient
- {
- BaseAddress = storageUrl,
- Timeout = 10000,
- Retries = 3
- };
-
- StorageUrl = new Uri(storageUrl);
-
- //Get the root address (StorageUrl without the account)
- var usernameIndex=storageUrl.LastIndexOf(UserName);
- var rootUrl = storageUrl.Substring(0, usernameIndex);
- RootAddressUri = new Uri(rootUrl);
-
- var token = authClient.GetHeaderValue("X-Auth-Token");
- if (String.IsNullOrWhiteSpace(token))
- throw new InvalidOperationException("Failed to obtain token url");
- Token = token;
-
- }
-
- Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
-
-
- return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName};
-
- }
-
-
-
- public IList<ContainerInfo> ListContainers(string account)
- {
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
- var content = client.DownloadStringWithRetry("", 3);
- client.AssertStatusOK("List Containers failed");
-
- if (client.StatusCode == HttpStatusCode.NoContent)
- return new List<ContainerInfo>();
- var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
-
- foreach (var info in infos)
- {
- info.Account = account;
- }
- return infos;
- }
-
- }
-
- private string GetAccountUrl(string account)
- {
- return new Uri(this.RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
- }
-
- public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
- {
- using (log4net.ThreadContext.Stacks["Share"].Push("List Accounts"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
- client.IfModifiedSince = since;
-
- //Extract the username from the base address
- client.BaseAddress = RootAddressUri.AbsoluteUri;
-
- var content = client.DownloadStringWithRetry(@"", 3);
-
- client.AssertStatusOK("ListSharingAccounts failed");
-
- //If the result is empty, return an empty list,
- var infos = String.IsNullOrWhiteSpace(content)
- ? new List<ShareAccountInfo>()
- //Otherwise deserialize the account list into a list of ShareAccountInfos
- : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
-
- Log.DebugFormat("END");
- return infos;
- }
- }
- }
-
- //Request listing of all objects in a container modified since a specific time.
- //If the *since* value is missing, return all objects
- public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
- {
-
- using (log4net.ThreadContext.Stacks["Share"].Push("List Objects"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- var objects = new List<ObjectInfo>();
- var accounts = ListSharingAccounts(since);
- foreach (var account in accounts)
- {
- var containers = ListContainers(account.name);
- foreach (var container in containers)
- {
- var containerObjects = ListObjects(account.name, container.Name, null);
- objects.AddRange(containerObjects);
- }
- }
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
- return objects;
- }
- }
-
- public void SetTags(ObjectInfo target,IDictionary<string,string> tags)
- {
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("The Token is not set");
- if (StorageUrl == null)
- throw new InvalidOperationException("The StorageUrl is not set");
- if (target == null)
- throw new ArgumentNullException("target");
- Contract.EndContractBlock();
-
- using (log4net.ThreadContext.Stacks["Share"].Push("Share Object"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
-
- client.BaseAddress = GetAccountUrl(target.Account);
-
- client.Parameters.Clear();
- client.Parameters.Add("update", "");
-
- foreach (var tag in tags)
- {
- var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
- client.Headers.Add(headerTag, tag.Value);
- }
-
- var content = client.DownloadStringWithRetry(target.Container, 3);
-
-
- client.AssertStatusOK("SetTags failed");
- //If the status is NOT ACCEPTED we have a problem
- if (client.StatusCode != HttpStatusCode.Accepted)
- {
- Log.Error("Failed to set tags");
- throw new Exception("Failed to set tags");
- }
-
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
- }
- }
-
-
- }
-
- public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
- {
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("The Token is not set");
- if (StorageUrl==null)
- throw new InvalidOperationException("The StorageUrl is not set");
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName");
- if (String.IsNullOrWhiteSpace(account))
- throw new ArgumentNullException("account");
- if (String.IsNullOrWhiteSpace(shareTo))
- throw new ArgumentNullException("shareTo");
- Contract.EndContractBlock();
-
- using (log4net.ThreadContext.Stacks["Share"].Push("Share Object"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
-
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
-
- string permission = "";
- if (write)
- permission = String.Format("write={0}", shareTo);
- else if (read)
- permission = String.Format("read={0}", shareTo);
- client.Headers.Add("X-Object-Sharing", permission);
-
- var content = client.DownloadStringWithRetry(container, 3);
-
- client.AssertStatusOK("ShareObject failed");
-
- //If the result is empty, return an empty list,
- var infos = String.IsNullOrWhiteSpace(content)
- ? new List<ObjectInfo>()
- //Otherwise deserialize the object list into a list of ObjectInfos
- : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
-
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
- }
- }
-
-
- }
-
- public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
- {
- if (accountInfo==null)
- throw new ArgumentNullException("accountInfo");
- Contract.EndContractBlock();
-
- using (log4net.ThreadContext.Stacks["Account"].Push("GetPolicies"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
- client.BaseAddress = GetAccountUrl(accountInfo.UserName);
-
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
- client.Head(String.Empty, 3);
-
- var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
- var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
-
- long quota, bytes;
- if (long.TryParse(quotaValue, out quota))
- accountInfo.Quota = quota;
- if (long.TryParse(bytesValue, out bytes))
- accountInfo.BytesUsed = bytes;
-
- return accountInfo;
-
- }
-
- }
- }
-
-
- public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- Contract.EndContractBlock();
-
- using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
- client.IfModifiedSince = since;
- var content = client.DownloadStringWithRetry(container, 3);
-
- client.AssertStatusOK("ListObjects failed");
-
- //If the result is empty, return an empty list,
- var infos = String.IsNullOrWhiteSpace(content)
- ? new List<ObjectInfo>()
- //Otherwise deserialize the object list into a list of ObjectInfos
- : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
-
- foreach (var info in infos)
- {
- info.Container = container;
- info.Account = account;
- }
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
- return infos;
- }
- }
- }
-
-
-
- public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- if (String.IsNullOrWhiteSpace(folder))
- throw new ArgumentNullException("folder");
- Contract.EndContractBlock();
-
- using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
- client.Parameters.Add("path", folder);
- client.IfModifiedSince = since;
- var content = client.DownloadStringWithRetry(container, 3);
- client.AssertStatusOK("ListObjects failed");
-
- var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
-
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
- return infos;
- }
- }
- }
-
-
- public bool ContainerExists(string account, string container)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- Contract.EndContractBlock();
-
- using (log4net.ThreadContext.Stacks["Containters"].Push("Exists"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Head(container, 3);
-
- bool result;
- switch (client.StatusCode)
- {
- case HttpStatusCode.OK:
- case HttpStatusCode.NoContent:
- result=true;
- break;
- case HttpStatusCode.NotFound:
- result=false;
- break;
- default:
- throw CreateWebException("ContainerExists", client.StatusCode);
- }
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
-
- return result;
- }
-
- }
- }
-
- public bool ObjectExists(string account, string container, string objectName)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The objectName property can't be empty");
- Contract.EndContractBlock();
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Head(container + "/" + objectName, 3);
-
- switch (client.StatusCode)
- {
- case HttpStatusCode.OK:
- case HttpStatusCode.NoContent:
- return true;
- case HttpStatusCode.NotFound:
- return false;
- default:
- throw CreateWebException("ObjectExists", client.StatusCode);
- }
- }
-
- }
-
- public ObjectInfo GetObjectInfo(string account, string container, string objectName)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The objectName property can't be empty");
- Contract.EndContractBlock();
-
- using (log4net.ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
- {
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
- try
- {
- client.Parameters.Clear();
-
- client.Head(container + "/" + objectName, 3);
-
- if (client.TimedOut)
- return ObjectInfo.Empty;
-
- switch (client.StatusCode)
- {
- case HttpStatusCode.OK:
- case HttpStatusCode.NoContent:
- var keys = client.ResponseHeaders.AllKeys.AsQueryable();
- var tags = (from key in keys
- where key.StartsWith("X-Object-Meta-")
- let name = key.Substring(14)
- select new {Name = name, Value = client.ResponseHeaders[name]})
- .ToDictionary(t => t.Name, t => t.Value);
- var extensions = (from key in keys
- where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
- select new {Name = key, Value = client.ResponseHeaders[key]})
- .ToDictionary(t => t.Name, t => t.Value);
- var info = new ObjectInfo
- {
- Account = account,
- Container = container,
- Name = objectName,
- Hash = client.GetHeaderValue("ETag"),
- Content_Type = client.GetHeaderValue("Content-Type"),
- Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length")),
- Tags = tags,
- Last_Modified = client.LastModified,
- Extensions = extensions
- };
- return info;
- case HttpStatusCode.NotFound:
- return ObjectInfo.Empty;
- default:
- throw new WebException(
- String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
- objectName, client.StatusCode));
- }
-
- }
- catch (RetryException)
- {
- Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.");
- return ObjectInfo.Empty;
- }
- catch (WebException e)
- {
- Log.Error(
- String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
- objectName, client.StatusCode), e);
- throw;
- }
- }
- }
-
- }
-
- public void CreateFolder(string account, string container, string folder)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(folder))
- throw new ArgumentNullException("folder", "The folder property can't be empty");
- Contract.EndContractBlock();
-
- var folderUrl=String.Format("{0}/{1}",container,folder);
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Headers.Add("Content-Type", @"application/directory");
- client.Headers.Add("Content-Length", "0");
- client.PutWithRetry(folderUrl, 3);
-
- if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
- throw CreateWebException("CreateFolder", client.StatusCode);
- }
- }
-
- public ContainerInfo GetContainerInfo(string account, string container)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- Contract.EndContractBlock();
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Head(container);
- switch (client.StatusCode)
- {
- case HttpStatusCode.OK:
- case HttpStatusCode.NoContent:
- var keys = client.ResponseHeaders.AllKeys.AsQueryable();
- var tags = (from key in keys
- where key.StartsWith("X-Container-Meta-")
- let name = key.Substring(14)
- select new { Name = name, Value = client.ResponseHeaders[name] })
- .ToDictionary(t => t.Name, t => t.Value);
- var policies= (from key in keys
- where key.StartsWith("X-Container-Policy-")
- let name = key.Substring(14)
- select new { Name = name, Value = client.ResponseHeaders[name] })
- .ToDictionary(t => t.Name, t => t.Value);
-
- var containerInfo = new ContainerInfo
- {
- Account=account,
- Name = container,
- Count =
- long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
- Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
- BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
- BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),
- Last_Modified=client.LastModified,
- Tags=tags,
- Policies=policies
- };
-
-
- return containerInfo;
- case HttpStatusCode.NotFound:
- return ContainerInfo.Empty;
- default:
- throw CreateWebException("GetContainerInfo", client.StatusCode);
- }
- }
- }
-
- public void CreateContainer(string account, string container)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- Contract.EndContractBlock();
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.PutWithRetry(container, 3);
- var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
- if (!expectedCodes.Contains(client.StatusCode))
- throw CreateWebException("CreateContainer", client.StatusCode);
- }
- }
-
- public void DeleteContainer(string account, string container)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- Contract.EndContractBlock();
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.DeleteWithRetry(container, 3);
- var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
- if (!expectedCodes.Contains(client.StatusCode))
- throw CreateWebException("DeleteContainer", client.StatusCode);
- }
-
- }
-
- /// <summary>
- ///
- /// </summary>
- /// <param name="account"></param>
- /// <param name="container"></param>
- /// <param name="objectName"></param>
- /// <param name="fileName"></param>
- /// <returns></returns>
- /// <remarks>This method should have no timeout or a very long one</remarks>
- //Asynchronously download the object specified by *objectName* in a specific *container* to
- // a local file
- public Task GetObject(string account, string container, string objectName, string fileName)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The objectName property can't be empty");
- Contract.EndContractBlock();
-
- try
- {
- //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
- //object to avoid concurrency errors.
- //
- //Download operations take a long time therefore they have no timeout.
- var client = new RestClient(_baseClient) { Timeout = 0 };
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- //The container and objectName are relative names. They are joined with the client's
- //BaseAddress to create the object's absolute address
- var builder = client.GetAddressBuilder(container, objectName);
- var uri = builder.Uri;
-
- //Download progress is reported to the Trace log
- Log.InfoFormat("[GET] START {0}", objectName);
- client.DownloadProgressChanged += (sender, args) =>
- Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
- fileName, args.ProgressPercentage,
- args.BytesReceived,
- args.TotalBytesToReceive);
-
-
- //Start downloading the object asynchronously
- var downloadTask = client.DownloadFileTask(uri, fileName);
-
- //Once the download completes
- return downloadTask.ContinueWith(download =>
- {
- //Delete the local client object
- client.Dispose();
- //And report failure or completion
- if (download.IsFaulted)
- {
- Log.ErrorFormat("[GET] FAIL for {0} with \r{1}", objectName,
- download.Exception);
- }
- else
- {
- Log.InfoFormat("[GET] END {0}", objectName);
- }
- });
- }
- catch (Exception exc)
- {
- Log.ErrorFormat("[GET] END {0} with {1}", objectName, exc);
- throw;
- }
-
-
-
- }
-
- public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName");
- if (hash==null)
- throw new ArgumentNullException("hash");
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("Invalid Token");
- if (StorageUrl == null)
- throw new InvalidOperationException("Invalid Storage Url");
- Contract.EndContractBlock();
-
-
- //Don't use a timeout because putting the hashmap may be a long process
- var client = new RestClient(_baseClient) { Timeout = 0 };
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- //The container and objectName are relative names. They are joined with the client's
- //BaseAddress to create the object's absolute address
- var builder = client.GetAddressBuilder(container, objectName);
- builder.Query = "format=json&hashmap";
- var uri = builder.Uri;
-
-
- //Send the tree hash as Json to the server
- client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
- var uploadTask=client.UploadStringTask(uri, "PUT", hash.ToJson());
-
-
- return uploadTask.ContinueWith(t =>
- {
-
- var empty = (IList<string>)new List<string>();
-
-
- //The server will respond either with 201-created if all blocks were already on the server
- if (client.StatusCode == HttpStatusCode.Created)
- {
- //in which case we return an empty hash list
- return empty;
- }
- //or with a 409-conflict and return the list of missing parts
- //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
- if (t.IsFaulted)
- {
- var ex = t.Exception.InnerException;
- var we = ex as WebException;
- var response = we.Response as HttpWebResponse;
- if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
- {
- //In case of 409 the missing parts will be in the response content
- using (var stream = response.GetResponseStream())
- using(var reader=new StreamReader(stream))
- {
- //We need to cleanup the content before returning it because it contains
- //error content after the list of hashes
- var hashes = new List<string>();
- string line=null;
- //All lines up to the first empty line are hashes
- while(!String.IsNullOrWhiteSpace(line=reader.ReadLine()))
- {
- hashes.Add(line);
- }
-
- return hashes;
- }
- }
- else
- //Any other status code is unexpected and the exception should be rethrown
- throw ex;
-
- }
- //Any other status code is unexpected but there was no exception. We can probably continue processing
- else
- {
- Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
- }
- return empty;
- });
-
- }
-
- public Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end)
- {
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("Invalid Token");
- if (StorageUrl == null)
- throw new InvalidOperationException("Invalid Storage Url");
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- if (relativeUrl== null)
- throw new ArgumentNullException("relativeUrl");
- if (end.HasValue && end<0)
- throw new ArgumentOutOfRangeException("end");
- if (start<0)
- throw new ArgumentOutOfRangeException("start");
- Contract.EndContractBlock();
-
-
- //Don't use a timeout because putting the hashmap may be a long process
- var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
- var uri = builder.Uri;
-
- return client.DownloadDataTask(uri)
- .ContinueWith(t=>
- {
- client.Dispose();
- return t.Result;
- });
- }
-
-
- public Task PostBlock(string account, string container, byte[] block, int offset, int count)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- if (block == null)
- throw new ArgumentNullException("block");
- if (offset < 0 || offset >= block.Length)
- throw new ArgumentOutOfRangeException("offset");
- if (count < 0 || count > block.Length)
- throw new ArgumentOutOfRangeException("count");
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("Invalid Token");
- if (StorageUrl == null)
- throw new InvalidOperationException("Invalid Storage Url");
- Contract.EndContractBlock();
-
-
- //Don't use a timeout because putting the hashmap may be a long process
- var client = new RestClient(_baseClient) { Timeout = 0 };
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- var builder = client.GetAddressBuilder(container, "");
- //We are doing an update
- builder.Query = "update";
- var uri = builder.Uri;
-
- client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
-
- Log.InfoFormat("[BLOCK POST] START");
-
- client.UploadProgressChanged += (sender, args) =>
- Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
- args.ProgressPercentage, args.BytesSent,
- args.TotalBytesToSend);
- client.UploadFileCompleted += (sender, args) =>
- Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
-
-
- //Send the block
- var uploadTask = client.UploadDataTask(uri, "POST", block)
- .ContinueWith(upload =>
- {
- client.Dispose();
-
- if (upload.IsFaulted)
- {
- var exception = upload.Exception.InnerException;
- Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exception);
- throw exception;
- }
-
- Log.InfoFormat("[BLOCK POST] END");
- });
- return uploadTask;
- }
-
-
- public Task<TreeHash> GetHashMap(string account, string container, string objectName)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName");
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("Invalid Token");
- if (StorageUrl == null)
- throw new InvalidOperationException("Invalid Storage Url");
- Contract.EndContractBlock();
-
- try
- {
- //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
- //object to avoid concurrency errors.
- //
- //Download operations take a long time therefore they have no timeout.
- //TODO: Do we really? this is a hashmap operation, not a download
- var client = new RestClient(_baseClient) { Timeout = 0 };
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
-
- //The container and objectName are relative names. They are joined with the client's
- //BaseAddress to create the object's absolute address
- var builder = client.GetAddressBuilder(container, objectName);
- builder.Query = "format=json&hashmap";
- var uri = builder.Uri;
-
- //Start downloading the object asynchronously
- var downloadTask = client.DownloadStringTask(uri);
-
- //Once the download completes
- return downloadTask.ContinueWith(download =>
- {
- //Delete the local client object
- client.Dispose();
- //And report failure or completion
- if (download.IsFaulted)
- {
- Log.ErrorFormat("[GET HASH] FAIL for {0} with \r{1}", objectName,
- download.Exception);
- throw download.Exception;
- }
-
- //The server will return an empty string if the file is empty
- var json = download.Result;
- var treeHash = TreeHash.Parse(json);
- Log.InfoFormat("[GET HASH] END {0}", objectName);
- return treeHash;
- });
- }
- catch (Exception exc)
- {
- Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
- throw;
- }
-
-
-
- }
-
-
- /// <summary>
- ///
- /// </summary>
- /// <param name="account"></param>
- /// <param name="container"></param>
- /// <param name="objectName"></param>
- /// <param name="fileName"></param>
- /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
- /// <remarks>>This method should have no timeout or a very long one</remarks>
- public Task PutObject(string account, string container, string objectName, string fileName, string hash = null)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The objectName property can't be empty");
- if (String.IsNullOrWhiteSpace(fileName))
- throw new ArgumentNullException("fileName", "The fileName property can't be empty");
- if (!File.Exists(fileName))
- throw new FileNotFoundException("The file does not exist",fileName);
- Contract.EndContractBlock();
-
- try
- {
-
- var client = new RestClient(_baseClient){Timeout=0};
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- var builder = client.GetAddressBuilder(container, objectName);
- var uri = builder.Uri;
-
- string etag = hash ?? CalculateHash(fileName);
-
- client.Headers.Add("Content-Type", "application/octet-stream");
- client.Headers.Add("ETag", etag);
-
-
- Log.InfoFormat("[PUT] START {0}", objectName);
- client.UploadProgressChanged += (sender, args) =>
- {
- using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
- {
- Log.InfoFormat("{0} {1}% {2} of {3}", fileName, args.ProgressPercentage,
- args.BytesSent, args.TotalBytesToSend);
- }
- };
-
- client.UploadFileCompleted += (sender, args) =>
- {
- using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
- {
- Log.InfoFormat("Completed {0}", fileName);
- }
- };
- return client.UploadFileTask(uri, "PUT", fileName)
- .ContinueWith(upload=>
- {
- client.Dispose();
-
- if (upload.IsFaulted)
- {
- var exc = upload.Exception.InnerException;
- Log.ErrorFormat("[PUT] FAIL for {0} with \r{1}",objectName,exc);
- throw exc;
- }
- else
- Log.InfoFormat("[PUT] END {0}", objectName);
- });
- }
- catch (Exception exc)
- {
- Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
- throw;
- }
-
- }
-
-
- private static string CalculateHash(string fileName)
- {
- Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
- Contract.EndContractBlock();
-
- string hash;
- using (var hasher = MD5.Create())
- using(var stream=File.OpenRead(fileName))
- {
- var hashBuilder=new StringBuilder();
- foreach (byte b in hasher.ComputeHash(stream))
- hashBuilder.Append(b.ToString("x2").ToLower());
- hash = hashBuilder.ToString();
- }
- return hash;
- }
-
- public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
- {
- if (String.IsNullOrWhiteSpace(sourceContainer))
- throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(oldObjectName))
- throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
- if (String.IsNullOrWhiteSpace(targetContainer))
- throw new ArgumentNullException("targetContainer", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(newObjectName))
- throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
- Contract.EndContractBlock();
-
- var targetUrl = targetContainer + "/" + newObjectName;
- var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Headers.Add("X-Move-From", sourceUrl);
- client.PutWithRetry(targetUrl, 3);
-
- var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
- if (!expectedCodes.Contains(client.StatusCode))
- throw CreateWebException("MoveObject", client.StatusCode);
- }
- }
-
- public void DeleteObject(string account, string sourceContainer, string objectName)
- {
- if (String.IsNullOrWhiteSpace(sourceContainer))
- throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
- Contract.EndContractBlock();
-
- var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
- var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Headers.Add("X-Move-From", sourceUrl);
- client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
- client.PutWithRetry(targetUrl, 3);
-
- var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
- if (!expectedCodes.Contains(client.StatusCode))
- throw CreateWebException("DeleteObject", client.StatusCode);
- }
- }
-
-
- private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
- {
- return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
- }
-
-
- }
-
- public class ShareAccountInfo
- {
- public DateTime? last_modified { get; set; }
- public string name { get; set; }
- }
-}
+#region\r
+/* -----------------------------------------------------------------------\r
+ * <copyright file="CloudFilesClient.cs" company="GRNet">\r
+ * \r
+ * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * 1. Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * 2. Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ *\r
+ * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
+ * POSSIBILITY OF SUCH DAMAGE.\r
+ *\r
+ * The views and conclusions contained in the software and\r
+ * documentation are those of the authors and should not be\r
+ * interpreted as representing official policies, either expressed\r
+ * or implied, of GRNET S.A.\r
+ * </copyright>\r
+ * -----------------------------------------------------------------------\r
+ */\r
+#endregion\r
+\r
+// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos\r
+//\r
+// The class provides methods to upload/download files, delete files, manage containers\r
+\r
+\r
+using System;\r
+using System.Collections.Generic;\r
+using System.Collections.Specialized;\r
+using System.ComponentModel.Composition;\r
+using System.Diagnostics;\r
+using System.Diagnostics.Contracts;\r
+using System.IO;\r
+using System.Linq;\r
+using System.Net;\r
+using System.Net.Http;\r
+using System.Net.Http.Headers;\r
+using System.Reflection;\r
+using System.Threading;\r
+using System.Threading.Tasks;\r
+using Newtonsoft.Json;\r
+using Pithos.Interfaces;\r
+using Pithos.Network;\r
+using log4net;\r
+\r
+namespace Pithos.Network\r
+{\r
+\r
+ [Export(typeof(ICloudClient))]\r
+ public class CloudFilesClient:ICloudClient,IDisposable\r
+ {\r
+ private const string TOKEN_HEADER = "X-Auth-Token";\r
+ private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
+\r
+ //CloudFilesClient uses *_baseClient* internally to communicate with the server\r
+ //RestClient provides a REST-friendly interface over the standard WebClient.\r
+ private RestClient _baseClient;\r
+\r
+ private HttpClient _baseHttpClient;\r
+ private HttpClient _baseHttpClientNoTimeout;\r
+ \r
+\r
+ //During authentication the client provides a UserName \r
+ public string UserName { get; set; }\r
+ \r
+ //and and ApiKey to the server\r
+ public string ApiKey { get; set; }\r
+ \r
+ //And receives an authentication Token. This token must be provided in ALL other operations,\r
+ //in the X-Auth-Token header\r
+ private string _token;\r
+ private readonly string _emptyGuid = Guid.Empty.ToString();\r
+ private readonly Uri _emptyUri = new Uri("",UriKind.Relative);\r
+\r
+ private HttpClientHandler _httpClientHandler = new HttpClientHandler\r
+ {\r
+ AllowAutoRedirect = true,\r
+ AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,\r
+ UseCookies = true, \r
+ };\r
+\r
+\r
+ public string Token\r
+ {\r
+ get { return _token; }\r
+ set\r
+ {\r
+ _token = value;\r
+ _baseClient.Headers[TOKEN_HEADER] = value;\r
+ }\r
+ }\r
+\r
+ //The client also receives a StorageUrl after authentication. All subsequent operations must\r
+ //use this url\r
+ public Uri StorageUrl { get; set; }\r
+\r
+\r
+ public Uri RootAddressUri { get; set; }\r
+\r
+\r
+ public double DownloadPercentLimit { get; set; }\r
+ public double UploadPercentLimit { get; set; }\r
+\r
+ public string AuthenticationUrl { get; set; }\r
+\r
+ \r
+ public string VersionPath\r
+ {\r
+ get { return UsePithos ? "v1" : "v1.0"; }\r
+ }\r
+\r
+ public bool UsePithos { get; set; }\r
+\r
+\r
+\r
+ public CloudFilesClient(string userName, string apiKey)\r
+ {\r
+ UserName = userName;\r
+ ApiKey = apiKey;\r
+ }\r
+\r
+ public CloudFilesClient(AccountInfo accountInfo)\r
+ {\r
+ if (accountInfo==null)\r
+ throw new ArgumentNullException("accountInfo");\r
+ Contract.Ensures(!String.IsNullOrWhiteSpace(Token));\r
+ Contract.Ensures(StorageUrl != null);\r
+ Contract.Ensures(_baseClient != null);\r
+ Contract.Ensures(RootAddressUri != null);\r
+ Contract.EndContractBlock(); \r
+\r
+ _baseClient = new RestClient\r
+ {\r
+ BaseAddress = accountInfo.StorageUri.ToString(),\r
+ Timeout = 30000,\r
+ Retries = 3,\r
+ };\r
+ StorageUrl = accountInfo.StorageUri;\r
+ Token = accountInfo.Token;\r
+ UserName = accountInfo.UserName;\r
+\r
+ //Get the root address (StorageUrl without the account)\r
+ var storageUrl = StorageUrl.AbsoluteUri;\r
+ var usernameIndex = storageUrl.LastIndexOf(UserName);\r
+ var rootUrl = storageUrl.Substring(0, usernameIndex);\r
+ RootAddressUri = new Uri(rootUrl);\r
+\r
+ var httpClientHandler = new HttpClientHandler\r
+ {\r
+ AllowAutoRedirect = true,\r
+ AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,\r
+ UseCookies = true,\r
+ };\r
+\r
+\r
+ _baseHttpClient = new HttpClient(httpClientHandler)\r
+ {\r
+ BaseAddress = StorageUrl,\r
+ Timeout = TimeSpan.FromSeconds(30)\r
+ };\r
+ _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);\r
+\r
+ _baseHttpClientNoTimeout = new HttpClient(httpClientHandler)\r
+ {\r
+ BaseAddress = StorageUrl,\r
+ Timeout = TimeSpan.FromMilliseconds(-1)\r
+ };\r
+ _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);\r
+\r
+\r
+ }\r
+\r
+\r
+ private static void AssertStatusOK(HttpResponseMessage response, string message)\r
+ {\r
+ var statusCode = response.StatusCode;\r
+ if (statusCode >= HttpStatusCode.BadRequest)\r
+ throw new WebException(String.Format("{0} with code {1} - {2}", message, statusCode, response.ReasonPhrase));\r
+ }\r
+\r
+ public async Task<AccountInfo> Authenticate()\r
+ {\r
+ if (String.IsNullOrWhiteSpace(UserName))\r
+ throw new InvalidOperationException("UserName is empty");\r
+ if (String.IsNullOrWhiteSpace(ApiKey))\r
+ throw new InvalidOperationException("ApiKey is empty");\r
+ if (String.IsNullOrWhiteSpace(AuthenticationUrl))\r
+ throw new InvalidOperationException("AuthenticationUrl is empty");\r
+ Contract.Ensures(!String.IsNullOrWhiteSpace(Token));\r
+ Contract.Ensures(StorageUrl != null);\r
+ Contract.Ensures(_baseClient != null);\r
+ Contract.Ensures(RootAddressUri != null);\r
+ Contract.EndContractBlock();\r
+\r
+\r
+ Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);\r
+\r
+ var groups = new List<Group>();\r
+\r
+ using (var authClient = new HttpClient(_httpClientHandler,false){ BaseAddress = new Uri(AuthenticationUrl),Timeout=TimeSpan.FromSeconds(30) })\r
+ { \r
+\r
+ authClient.DefaultRequestHeaders.Add("X-Auth-User", UserName);\r
+ authClient.DefaultRequestHeaders.Add("X-Auth-Key", ApiKey);\r
+\r
+ string storageUrl;\r
+ string token;\r
+ \r
+ using (var response = await authClient.GetAsyncWithRetries(new Uri(VersionPath, UriKind.Relative),3).ConfigureAwait(false)) // .DownloadStringWithRetryRelative(new Uri(VersionPath, UriKind.Relative), 3); \r
+ {\r
+ AssertStatusOK(response,"Authentication failed");\r
+ \r
+ storageUrl = response.Headers.GetFirstValue("X-Storage-Url");\r
+ if (String.IsNullOrWhiteSpace(storageUrl))\r
+ throw new InvalidOperationException("Failed to obtain storage url");\r
+\r
+ token = response.Headers.GetFirstValue(TOKEN_HEADER);\r
+ if (String.IsNullOrWhiteSpace(token))\r
+ throw new InvalidOperationException("Failed to obtain token url");\r
+\r
+ }\r
+\r
+\r
+ _baseClient = new RestClient\r
+ {\r
+ BaseAddress = storageUrl,\r
+ Timeout = 30000,\r
+ Retries = 3, \r
+ };\r
+\r
+ StorageUrl = new Uri(storageUrl);\r
+ Token = token;\r
+\r
+ \r
+\r
+ \r
+ //Get the root address (StorageUrl without the account)\r
+ var usernameIndex=storageUrl.LastIndexOf(UserName);\r
+ var rootUrl = storageUrl.Substring(0, usernameIndex);\r
+ RootAddressUri = new Uri(rootUrl);\r
+ \r
+\r
+ _baseHttpClient = new HttpClient(_httpClientHandler,false)\r
+ {\r
+ BaseAddress = StorageUrl,\r
+ Timeout = TimeSpan.FromSeconds(30)\r
+ };\r
+ _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, token);\r
+\r
+ _baseHttpClientNoTimeout = new HttpClient(_httpClientHandler,false)\r
+ {\r
+ BaseAddress = StorageUrl,\r
+ Timeout = TimeSpan.FromMilliseconds(-1)\r
+ };\r
+ _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, token);\r
+\r
+ /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();\r
+ groups = (from key in keys\r
+ where key.StartsWith("X-Account-Group-")\r
+ let name = key.Substring(16)\r
+ select new Group(name, authClient.ResponseHeaders[key]))\r
+ .ToList();\r
+ \r
+*/\r
+ }\r
+\r
+ Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);\r
+ Debug.Assert(_baseClient!=null);\r
+\r
+ return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups}; \r
+\r
+ }\r
+\r
+ private static void TraceStart(string method, Uri actualAddress)\r
+ {\r
+ Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);\r
+ }\r
+\r
+ private async Task<string> GetStringAsync(Uri targetUri, string errorMessage,DateTimeOffset? since=null)\r
+ {\r
+ TraceStart("GET",targetUri);\r
+ var request = new HttpRequestMessage(HttpMethod.Get, targetUri); \r
+ if (since.HasValue)\r
+ {\r
+ request.Headers.IfModifiedSince = since.Value;\r
+ }\r
+ using (var response = await _baseHttpClient.SendAsyncWithRetries(request,3).ConfigureAwait(false))\r
+ {\r
+ AssertStatusOK(response, errorMessage);\r
+\r
+ if (response.StatusCode == HttpStatusCode.NoContent)\r
+ return String.Empty;\r
+\r
+ var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);\r
+ return content;\r
+ }\r
+ }\r
+\r
+ public async Task<IList<ContainerInfo>> ListContainers(string account)\r
+ {\r
+\r
+ var targetUrl = GetTargetUrl(account);\r
+ var targetUri = new Uri(String.Format("{0}?format=json", targetUrl));\r
+ var result = await GetStringAsync(targetUri, "List Containers failed").ConfigureAwait(false);\r
+ if (String.IsNullOrWhiteSpace(result))\r
+ return new List<ContainerInfo>();\r
+ var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(result);\r
+ foreach (var info in infos)\r
+ {\r
+ info.Account = account;\r
+ }\r
+ return infos;\r
+ }\r
+\r
+ \r
+ private string GetAccountUrl(string account)\r
+ {\r
+ return RootAddressUri.Combine(account).AbsoluteUri;\r
+ }\r
+\r
+ public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)\r
+ {\r
+ using (ThreadContext.Stacks["Share"].Push("List Accounts"))\r
+ {\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
+\r
+ var targetUri = new Uri(String.Format("{0}?format=json", RootAddressUri), UriKind.Absolute);\r
+ var content=TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListSharingAccounts failed", since).ConfigureAwait(false)).Result;\r
+\r
+ //If the result is empty, return an empty list,\r
+ var infos = String.IsNullOrWhiteSpace(content)\r
+ ? new List<ShareAccountInfo>()\r
+ //Otherwise deserialize the account list into a list of ShareAccountInfos\r
+ : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);\r
+\r
+ Log.DebugFormat("END");\r
+ return infos;\r
+ }\r
+ }\r
+\r
+\r
+ /// <summary>\r
+ /// Request listing of all objects in a container modified since a specific time.\r
+ /// If the *since* value is missing, return all objects\r
+ /// </summary>\r
+ /// <param name="knownContainers">Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new\r
+ /// and should be polled anyway\r
+ /// </param>\r
+ /// <param name="since"></param>\r
+ /// <returns></returns>\r
+ public IList<ObjectInfo> ListSharedObjects(HashSet<string> knownContainers, DateTimeOffset? since)\r
+ {\r
+\r
+ using (ThreadContext.Stacks["Share"].Push("List Objects"))\r
+ {\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
+ //'since' is not used here because we need to have ListObjects return a NoChange result\r
+ //for all shared accounts,containers\r
+\r
+ Func<ContainerInfo, string> getKey = c => String.Format("{0}\\{1}", c.Account, c.Name);\r
+\r
+ var containers = (from account in ListSharingAccounts()\r
+ let conts = ListContainers(account.name).Result\r
+ from container in conts\r
+ select container).ToList(); \r
+ var items = from container in containers \r
+ let actualSince=knownContainers.Contains(getKey(container))?since:null\r
+ select ListObjects(container.Account , container.Name, actualSince);\r
+ var objects=items.SelectMany(r=> r).ToList();\r
+\r
+ //For each object\r
+ //Check parents recursively up to (but not including) the container.\r
+ //If parents are missing, add them to the list\r
+ //Need function to calculate all parent URLs\r
+ objects = AddMissingParents(objects);\r
+ \r
+ //Store any new containers\r
+ foreach (var container in containers)\r
+ {\r
+ knownContainers.Add(getKey(container));\r
+ }\r
+\r
+\r
+\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
+ return objects;\r
+ }\r
+ }\r
+\r
+ private List<ObjectInfo> AddMissingParents(List<ObjectInfo> objects)\r
+ {\r
+ //TODO: Remove short-circuit when we decide to use Missing Parents functionality\r
+ //return objects;\r
+\r
+ var existingUris = objects.ToDictionary(o => o.Uri, o => o);\r
+ foreach (var objectInfo in objects)\r
+ {\r
+ //Can be null when retrieving objects to show in selective sync\r
+ if (objectInfo.Name == null)\r
+ continue;\r
+\r
+ //No need to unescape here, the parts will be used to create new ObjectInfos\r
+ var parts = objectInfo.Name.ToString().Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries);\r
+ //If there is no parent, skip\r
+ if (parts.Length == 1)\r
+ continue;\r
+ var baseParts = new[]\r
+ {\r
+ objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container.ToString()\r
+ };\r
+ for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++)\r
+ {\r
+ var nameparts = parts.Range(0, partIdx).ToArray();\r
+ var parentName= String.Join("/", nameparts);\r
+\r
+ var parentParts = baseParts.Concat(nameparts);\r
+ var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts);\r
+ \r
+ var parentUri = new Uri(parentUrl, UriKind.Absolute);\r
+\r
+ ObjectInfo existingInfo;\r
+ if (!existingUris.TryGetValue(parentUri,out existingInfo))\r
+ {\r
+ var h = parentUrl.GetHashCode();\r
+ var reverse = new string(parentUrl.Reverse().ToArray());\r
+ var rh = reverse.GetHashCode();\r
+ var b1 = BitConverter.GetBytes(h);\r
+ var b2 = BitConverter.GetBytes(rh);\r
+ var g = new Guid(0,0,0,b1.Concat(b2).ToArray());\r
+ \r
+\r
+ existingUris[parentUri] = new ObjectInfo\r
+ {\r
+ Account = objectInfo.Account,\r
+ Container = objectInfo.Container,\r
+ Content_Type = ObjectInfo.CONTENT_TYPE_DIRECTORY,\r
+ ETag = Signature.MERKLE_EMPTY,\r
+ X_Object_Hash = Signature.MERKLE_EMPTY,\r
+ Name=new Uri(parentName,UriKind.Relative),\r
+ StorageUri=objectInfo.StorageUri,\r
+ Bytes = 0,\r
+ UUID=g.ToString(), \r
+ };\r
+ }\r
+ }\r
+ }\r
+ return existingUris.Values.ToList();\r
+ }\r
+\r
+ public void SetTags(ObjectInfo target,IDictionary<string,string> tags)\r
+ {\r
+ if (String.IsNullOrWhiteSpace(Token))\r
+ throw new InvalidOperationException("The Token is not set");\r
+ if (StorageUrl == null)\r
+ throw new InvalidOperationException("The StorageUrl is not set");\r
+ if (target == null)\r
+ throw new ArgumentNullException("target");\r
+ Contract.EndContractBlock();\r
+\r
+ using (ThreadContext.Stacks["Share"].Push("Share Object"))\r
+ {\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
+\r
+ using (var client = new RestClient(_baseClient))\r
+ {\r
+\r
+ client.BaseAddress = GetAccountUrl(target.Account);\r
+\r
+ client.Parameters.Clear();\r
+ client.Parameters.Add("update", "");\r
+\r
+ foreach (var tag in tags)\r
+ {\r
+ var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);\r
+ client.Headers.Add(headerTag, tag.Value);\r
+ }\r
+ \r
+ client.DownloadStringWithRetryRelative(target.Container, 3);\r
+\r
+ \r
+ client.AssertStatusOK("SetTags failed");\r
+ //If the status is NOT ACCEPTED we have a problem\r
+ if (client.StatusCode != HttpStatusCode.Accepted)\r
+ {\r
+ Log.Error("Failed to set tags");\r
+ throw new Exception("Failed to set tags");\r
+ }\r
+\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
+ }\r
+ }\r
+\r
+\r
+ }\r
+\r
+ public void ShareObject(string account, Uri container, Uri objectName, string shareTo, bool read, bool write)\r
+ {\r
+ if (String.IsNullOrWhiteSpace(Token))\r
+ throw new InvalidOperationException("The Token is not set");\r
+ if (StorageUrl==null)\r
+ throw new InvalidOperationException("The StorageUrl is not set");\r
+ if (container==null)\r
+ throw new ArgumentNullException("container");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("container");\r
+ if (objectName==null)\r
+ throw new ArgumentNullException("objectName");\r
+ if (objectName.IsAbsoluteUri)\r
+ throw new ArgumentException("objectName");\r
+ if (String.IsNullOrWhiteSpace(account))\r
+ throw new ArgumentNullException("account");\r
+ if (String.IsNullOrWhiteSpace(shareTo))\r
+ throw new ArgumentNullException("shareTo");\r
+ Contract.EndContractBlock();\r
+\r
+ using (ThreadContext.Stacks["Share"].Push("Share Object"))\r
+ {\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
+ \r
+ using (var client = new RestClient(_baseClient))\r
+ {\r
+\r
+ client.BaseAddress = GetAccountUrl(account);\r
+\r
+ client.Parameters.Clear();\r
+ client.Parameters.Add("format", "json");\r
+\r
+ string permission = "";\r
+ if (write)\r
+ permission = String.Format("write={0}", shareTo);\r
+ else if (read)\r
+ permission = String.Format("read={0}", shareTo);\r
+ client.Headers.Add("X-Object-Sharing", permission);\r
+\r
+ var content = client.DownloadStringWithRetryRelative(container, 3);\r
+\r
+ client.AssertStatusOK("ShareObject failed");\r
+\r
+ //If the result is empty, return an empty list,\r
+ var infos = String.IsNullOrWhiteSpace(content)\r
+ ? new List<ObjectInfo>()\r
+ //Otherwise deserialize the object list into a list of ObjectInfos\r
+ : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
+\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
+ }\r
+ }\r
+\r
+\r
+ }\r
+\r
+ public async Task<AccountInfo> GetAccountPolicies(AccountInfo accountInfo)\r
+ {\r
+ if (accountInfo==null)\r
+ throw new ArgumentNullException("accountInfo");\r
+ Contract.EndContractBlock();\r
+\r
+ using (ThreadContext.Stacks["Account"].Push("GetPolicies"))\r
+ {\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
+\r
+/*\r
+ if (_baseClient == null)\r
+ {\r
+ _baseClient = new RestClient\r
+ {\r
+ BaseAddress = accountInfo.StorageUri.ToString(),\r
+ Timeout = 30000,\r
+ Retries = 3,\r
+ };\r
+ }\r
+\r
+*/ \r
+ var containerUri = GetTargetUri(accountInfo.UserName);\r
+ var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);\r
+ using(var response=await _baseHttpClient.HeadAsyncWithRetries(targetUri,3).ConfigureAwait(false))\r
+ {\r
+ \r
+ var quotaValue=response.Headers.GetFirstValue("X-Account-Policy-Quota");\r
+ var bytesValue = response.Headers.GetFirstValue("X-Account-Bytes-Used");\r
+ long quota, bytes;\r
+ if (long.TryParse(quotaValue, out quota))\r
+ accountInfo.Quota = quota;\r
+ if (long.TryParse(bytesValue, out bytes))\r
+ accountInfo.BytesUsed = bytes;\r
+\r
+ return accountInfo; \r
+ }\r
+\r
+\r
+ //using (var client = new RestClient(_baseClient))\r
+ //{\r
+ // if (!String.IsNullOrWhiteSpace(accountInfo.UserName))\r
+ // client.BaseAddress = GetAccountUrl(accountInfo.UserName);\r
+\r
+ // client.Parameters.Clear();\r
+ // client.Parameters.Add("format", "json"); \r
+ // client.Head(_emptyUri, 3);\r
+\r
+ // var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];\r
+ // var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];\r
+\r
+ // long quota, bytes;\r
+ // if (long.TryParse(quotaValue, out quota))\r
+ // accountInfo.Quota = quota;\r
+ // if (long.TryParse(bytesValue, out bytes))\r
+ // accountInfo.BytesUsed = bytes;\r
+ \r
+ // return accountInfo;\r
+\r
+ //}\r
+\r
+ }\r
+ }\r
+\r
+ public void UpdateMetadata(ObjectInfo objectInfo)\r
+ {\r
+ if (objectInfo == null)\r
+ throw new ArgumentNullException("objectInfo");\r
+ Contract.EndContractBlock();\r
+\r
+ using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))\r
+ {\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
+\r
+\r
+ using(var client=new RestClient(_baseClient))\r
+ {\r
+\r
+ client.BaseAddress = GetAccountUrl(objectInfo.Account);\r
+ \r
+ client.Parameters.Clear();\r
+ \r
+\r
+ //Set Tags\r
+ foreach (var tag in objectInfo.Tags)\r
+ {\r
+ var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);\r
+ client.Headers.Add(headerTag, tag.Value);\r
+ }\r
+\r
+ //Set Permissions\r
+\r
+ var permissions=objectInfo.GetPermissionString();\r
+ client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);\r
+\r
+ client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);\r
+ client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);\r
+ client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);\r
+ var isPublic = objectInfo.IsPublic.ToString().ToLower();\r
+ client.Headers.Add("X-Object-Public", isPublic);\r
+\r
+\r
+ var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);\r
+ client.PostWithRetry(new Uri(address,UriKind.Relative),"application/xml");\r
+ \r
+ client.AssertStatusOK("UpdateMetadata failed");\r
+ //If the status is NOT ACCEPTED or OK we have a problem\r
+ if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))\r
+ {\r
+ Log.Error("Failed to update metadata");\r
+ throw new Exception("Failed to update metadata");\r
+ }\r
+\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
+ }\r
+ }\r
+\r
+ }\r
+\r
+ public void UpdateMetadata(ContainerInfo containerInfo)\r
+ {\r
+ if (containerInfo == null)\r
+ throw new ArgumentNullException("containerInfo");\r
+ Contract.EndContractBlock();\r
+\r
+ using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))\r
+ {\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
+\r
+\r
+ using(var client=new RestClient(_baseClient))\r
+ {\r
+\r
+ client.BaseAddress = GetAccountUrl(containerInfo.Account);\r
+ \r
+ client.Parameters.Clear();\r
+ \r
+\r
+ //Set Tags\r
+ foreach (var tag in containerInfo.Tags)\r
+ {\r
+ var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);\r
+ client.Headers.Add(headerTag, tag.Value);\r
+ }\r
+\r
+ \r
+ //Set Policies\r
+ foreach (var policy in containerInfo.Policies)\r
+ {\r
+ var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);\r
+ client.Headers.Add(headerPolicy, policy.Value);\r
+ }\r
+\r
+\r
+ var uriBuilder = client.GetAddressBuilder(containerInfo.Name,_emptyUri);\r
+ var uri = uriBuilder.Uri;\r
+\r
+ client.UploadValues(uri,new NameValueCollection());\r
+\r
+\r
+ client.AssertStatusOK("UpdateMetadata failed");\r
+ //If the status is NOT ACCEPTED or OK we have a problem\r
+ if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))\r
+ {\r
+ Log.Error("Failed to update metadata");\r
+ throw new Exception("Failed to update metadata");\r
+ }\r
+\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
+ }\r
+ }\r
+\r
+ }\r
+\r
+ \r
+\r
+\r
+ public IList<ObjectInfo> ListObjects(string account, Uri container, DateTimeOffset? since = null)\r
+ {\r
+ if (container==null)\r
+ throw new ArgumentNullException("container");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("container");\r
+ Contract.EndContractBlock();\r
+\r
+ using (ThreadContext.Stacks["Objects"].Push("List"))\r
+ {\r
+\r
+ var containerUri = GetTargetUri(account).Combine(container);\r
+ var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);\r
+\r
+ var content =TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result;\r
+\r
+ //304 will result in an empty string. Empty containers return an empty json array\r
+ if (String.IsNullOrWhiteSpace(content))\r
+ return new[] {new NoModificationInfo(account, container)};\r
+\r
+ //If the result is empty, return an empty list,\r
+ var infos = String.IsNullOrWhiteSpace(content)\r
+ ? new List<ObjectInfo>()\r
+ //Otherwise deserialize the object list into a list of ObjectInfos\r
+ : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
+\r
+ foreach (var info in infos)\r
+ {\r
+ info.Container = container;\r
+ info.Account = account;\r
+ info.StorageUri = StorageUrl;\r
+ }\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
+ return infos;\r
+ }\r
+ }\r
+\r
+ public IList<ObjectInfo> ListObjects(string account, Uri container, Uri folder, DateTimeOffset? since = null)\r
+ {\r
+ if (container==null)\r
+ throw new ArgumentNullException("container");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("container");\r
+ Contract.EndContractBlock();\r
+\r
+ using (ThreadContext.Stacks["Objects"].Push("List"))\r
+ {\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
+\r
+ var containerUri = GetTargetUri(account).Combine(container);\r
+ var targetUri = new Uri(String.Format("{0}?format=json&path={1}", containerUri,folder), UriKind.Absolute);\r
+ var content = TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result; \r
+\r
+ //304 will result in an empty string. Empty containers return an empty json array\r
+ if (String.IsNullOrWhiteSpace(content))\r
+ return new[] { new NoModificationInfo(account, container) };\r
+\r
+\r
+ var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
+ foreach (var info in infos)\r
+ {\r
+ info.Account = account;\r
+ if (info.Container == null)\r
+ info.Container = container;\r
+ info.StorageUri = StorageUrl;\r
+ }\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
+ return infos;\r
+/*\r
+ using (var client = new RestClient(_baseClient))\r
+ {\r
+ if (!String.IsNullOrWhiteSpace(account))\r
+ client.BaseAddress = GetAccountUrl(account);\r
+\r
+ client.Parameters.Clear();\r
+ client.Parameters.Add("format", "json");\r
+ client.Parameters.Add("path", folder.ToString());\r
+ client.IfModifiedSince = since;\r
+ var content = client.DownloadStringWithRetryRelative(container, 3);\r
+ client.AssertStatusOK("ListObjects failed");\r
+\r
+ if (client.StatusCode==HttpStatusCode.NotModified)\r
+ return new[]{new NoModificationInfo(account,container,folder)};\r
+\r
+ var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);\r
+ foreach (var info in infos)\r
+ {\r
+ info.Account = account;\r
+ if (info.Container == null)\r
+ info.Container = container;\r
+ info.StorageUri = StorageUrl;\r
+ }\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
+ return infos;\r
+ }\r
+*/\r
+ }\r
+ }\r
+\r
+ \r
+ public bool ContainerExists(string account, Uri container)\r
+ {\r
+ if (container==null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException( "The container must be relative","container");\r
+ Contract.EndContractBlock();\r
+\r
+ using (ThreadContext.Stacks["Containters"].Push("Exists"))\r
+ {\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");\r
+\r
+ var targetUri = GetTargetUri(account).Combine(container);\r
+\r
+ using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result)\r
+ {\r
+\r
+ bool result;\r
+ switch (response.StatusCode)\r
+ {\r
+ case HttpStatusCode.OK:\r
+ case HttpStatusCode.NoContent:\r
+ result = true;\r
+ break;\r
+ case HttpStatusCode.NotFound:\r
+ result = false;\r
+ break;\r
+ default:\r
+ throw CreateWebException("ContainerExists", response.StatusCode);\r
+ }\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
+\r
+ return result;\r
+ }\r
+/*\r
+ using (var client = new RestClient(_baseClient))\r
+ {\r
+ if (!String.IsNullOrWhiteSpace(account))\r
+ client.BaseAddress = GetAccountUrl(account);\r
+\r
+ client.Parameters.Clear();\r
+ client.Head(container, 3);\r
+ \r
+ bool result;\r
+ switch (client.StatusCode)\r
+ {\r
+ case HttpStatusCode.OK:\r
+ case HttpStatusCode.NoContent:\r
+ result=true;\r
+ break;\r
+ case HttpStatusCode.NotFound:\r
+ result=false;\r
+ break;\r
+ default:\r
+ throw CreateWebException("ContainerExists", client.StatusCode);\r
+ }\r
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");\r
+\r
+ return result;\r
+ }\r
+*/\r
+ \r
+ }\r
+ }\r
+\r
+ private Uri GetTargetUri(string account)\r
+ {\r
+ return new Uri(GetTargetUrl(account),UriKind.Absolute);\r
+ }\r
+\r
+ private string GetTargetUrl(string account)\r
+ {\r
+ return String.IsNullOrWhiteSpace(account)\r
+ ? _baseHttpClient.BaseAddress.ToString()\r
+ : GetAccountUrl(account);\r
+ }\r
+\r
+ public bool ObjectExists(string account, Uri container, Uri objectName)\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative","container");\r
+ if (objectName == null)\r
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
+ if (objectName.IsAbsoluteUri)\r
+ throw new ArgumentException("The objectName must be relative","objectName");\r
+ Contract.EndContractBlock();\r
+\r
+ var targetUri=GetTargetUri(account).Combine(container).Combine(objectName);\r
+\r
+ using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result)\r
+ {\r
+ switch (response.StatusCode)\r
+ {\r
+ case HttpStatusCode.OK:\r
+ case HttpStatusCode.NoContent:\r
+ return true;\r
+ case HttpStatusCode.NotFound:\r
+ return false;\r
+ default:\r
+ throw CreateWebException("ObjectExists", response.StatusCode);\r
+ }\r
+ }\r
+\r
+/*\r
+ using (var client = new RestClient(_baseClient))\r
+ {\r
+ if (!String.IsNullOrWhiteSpace(account))\r
+ client.BaseAddress = GetAccountUrl(account);\r
+\r
+ client.Parameters.Clear();\r
+ client.Head(container.Combine(objectName), 3);\r
+\r
+ switch (client.StatusCode)\r
+ {\r
+ case HttpStatusCode.OK:\r
+ case HttpStatusCode.NoContent:\r
+ return true;\r
+ case HttpStatusCode.NotFound:\r
+ return false;\r
+ default:\r
+ throw CreateWebException("ObjectExists", client.StatusCode);\r
+ }\r
+ }\r
+*/\r
+\r
+ }\r
+\r
+ public async Task<ObjectInfo> GetObjectInfo(string account, Uri container, Uri objectName)\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative", "container");\r
+ if (objectName == null)\r
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
+ if (objectName.IsAbsoluteUri)\r
+ throw new ArgumentException("The objectName must be relative", "objectName");\r
+ Contract.EndContractBlock();\r
+\r
+ using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))\r
+ {\r
+\r
+ var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);\r
+ try\r
+ {\r
+ using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true))\r
+ {\r
+ switch (response.StatusCode)\r
+ {\r
+ case HttpStatusCode.OK:\r
+ case HttpStatusCode.NoContent:\r
+ var tags = response.Headers.GetMeta("X-Object-Meta-");\r
+ var extensions = (from header in response.Headers\r
+ where\r
+ header.Key.StartsWith("X-Object-") &&\r
+ !header.Key.StartsWith("X-Object-Meta-")\r
+ select new {Name = header.Key, Value = header.Value.FirstOrDefault()})\r
+ .ToDictionary(t => t.Name, t => t.Value);\r
+\r
+ var permissions = response.Headers.GetFirstValue("X-Object-Sharing");\r
+\r
+\r
+ var info = new ObjectInfo\r
+ {\r
+ Account = account,\r
+ Container = container,\r
+ Name = objectName,\r
+ ETag = response.Headers.ETag.NullSafe(e=>e.Tag),\r
+ UUID = response.Headers.GetFirstValue("X-Object-UUID"),\r
+ X_Object_Hash = response.Headers.GetFirstValue("X-Object-Hash"),\r
+ Content_Type = response.Headers.GetFirstValue("Content-Type"),\r
+ Bytes = Convert.ToInt64(response.Content.Headers.ContentLength),\r
+ Tags = tags,\r
+ Last_Modified = response.Content.Headers.LastModified,\r
+ Extensions = extensions,\r
+ ContentEncoding =\r
+ response.Content.Headers.ContentEncoding.FirstOrDefault(),\r
+ ContendDisposition =\r
+ response.Content.Headers.ContentDisposition.NullSafe(c=>c.ToString()),\r
+ Manifest = response.Headers.GetFirstValue("X-Object-Manifest"),\r
+ PublicUrl = response.Headers.GetFirstValue("X-Object-Public"),\r
+ StorageUri = StorageUrl,\r
+ };\r
+ info.SetPermissions(permissions);\r
+ return info;\r
+ case HttpStatusCode.NotFound:\r
+ return ObjectInfo.Empty;\r
+ default:\r
+ throw new WebException(\r
+ String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",\r
+ objectName, response.StatusCode));\r
+ }\r
+ }\r
+ }\r
+ catch (RetryException)\r
+ {\r
+ Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.", objectName);\r
+ return ObjectInfo.Empty;\r
+ }\r
+ catch (WebException e)\r
+ {\r
+ Log.Error(\r
+ String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status {1}",\r
+ objectName, e.Status), e);\r
+ throw;\r
+ }\r
+ } /* using (var client = new RestClient(_baseClient))\r
+ {\r
+ if (!String.IsNullOrWhiteSpace(account))\r
+ client.BaseAddress = GetAccountUrl(account);\r
+ try\r
+ {\r
+ client.Parameters.Clear();\r
+\r
+ client.Head(container.Combine(objectName), 3);\r
+\r
+ if (client.TimedOut)\r
+ return ObjectInfo.Empty;\r
+\r
+ switch (client.StatusCode)\r
+ {\r
+ case HttpStatusCode.OK:\r
+ case HttpStatusCode.NoContent:\r
+ var keys = client.ResponseHeaders.AllKeys.AsQueryable();\r
+ var tags = client.GetMeta("X-Object-Meta-");\r
+ var extensions = (from key in keys\r
+ where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")\r
+ select new {Name = key, Value = client.ResponseHeaders[key]})\r
+ .ToDictionary(t => t.Name, t => t.Value);\r
+\r
+ var permissions=client.GetHeaderValue("X-Object-Sharing", true);\r
+ \r
+ \r
+ var info = new ObjectInfo\r
+ {\r
+ Account = account,\r
+ Container = container,\r
+ Name = objectName,\r
+ ETag = client.GetHeaderValue("ETag"),\r
+ UUID=client.GetHeaderValue("X-Object-UUID"),\r
+ X_Object_Hash = client.GetHeaderValue("X-Object-Hash"),\r
+ Content_Type = client.GetHeaderValue("Content-Type"),\r
+ Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),\r
+ Tags = tags,\r
+ Last_Modified = client.LastModified,\r
+ Extensions = extensions,\r
+ ContentEncoding=client.GetHeaderValue("Content-Encoding",true),\r
+ ContendDisposition = client.GetHeaderValue("Content-Disposition",true),\r
+ Manifest=client.GetHeaderValue("X-Object-Manifest",true),\r
+ PublicUrl=client.GetHeaderValue("X-Object-Public",true), \r
+ StorageUri=StorageUrl,\r
+ };\r
+ info.SetPermissions(permissions);\r
+ return info;\r
+ case HttpStatusCode.NotFound:\r
+ return ObjectInfo.Empty;\r
+ default:\r
+ throw new WebException(\r
+ String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",\r
+ objectName, client.StatusCode));\r
+ }\r
+\r
+ }\r
+ catch (RetryException)\r
+ {\r
+ Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);\r
+ return ObjectInfo.Empty;\r
+ }\r
+ catch (WebException e)\r
+ {\r
+ Log.Error(\r
+ String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",\r
+ objectName, client.StatusCode), e);\r
+ throw;\r
+ }\r
+ } */\r
+\r
+\r
+ }\r
+\r
+\r
+\r
+ public void CreateFolder(string account, Uri container, Uri folder)\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative","container");\r
+ if (folder == null)\r
+ throw new ArgumentNullException("folder", "The objectName property can't be empty");\r
+ if (folder.IsAbsoluteUri)\r
+ throw new ArgumentException("The objectName must be relative","folder");\r
+ Contract.EndContractBlock();\r
+\r
+ var folderUri=container.Combine(folder); \r
+ var targetUri = GetTargetUri(account).Combine(folderUri);\r
+ var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
+ \r
+ message.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY);\r
+ message.Headers.Add("Content-Length", "0");\r
+ using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)\r
+ {\r
+ if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)\r
+ throw CreateWebException("CreateFolder", response.StatusCode);\r
+ }\r
+/*\r
+ using (var client = new RestClient(_baseClient))\r
+ {\r
+ if (!String.IsNullOrWhiteSpace(account))\r
+ client.BaseAddress = GetAccountUrl(account);\r
+\r
+ client.Parameters.Clear();\r
+ client.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY);\r
+ client.Headers.Add("Content-Length", "0");\r
+ client.PutWithRetry(folderUri, 3);\r
+\r
+ if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)\r
+ throw CreateWebException("CreateFolder", client.StatusCode);\r
+ }\r
+*/\r
+ }\r
+\r
+ private Dictionary<string, string> GetMeta(HttpResponseMessage response,string metaPrefix)\r
+ {\r
+ if (String.IsNullOrWhiteSpace(metaPrefix))\r
+ throw new ArgumentNullException("metaPrefix");\r
+ Contract.EndContractBlock();\r
+\r
+ var dict = (from header in response.Headers\r
+ where header.Key.StartsWith(metaPrefix)\r
+ select new { Name = header.Key, Value = String.Join(",", header.Value) })\r
+ .ToDictionary(t => t.Name, t => t.Value);\r
+\r
+ \r
+ return dict;\r
+ }\r
+\r
+\r
+ public ContainerInfo GetContainerInfo(string account, Uri container)\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative","container");\r
+ Contract.EndContractBlock();\r
+\r
+ var targetUri = GetTargetUri(account).Combine(container); \r
+ using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result)\r
+ {\r
+ if (Log.IsDebugEnabled)\r
+ Log.DebugFormat("ContainerInfo data: {0}\n{1}",response,response.Content.ReadAsStringAsync().Result);\r
+ switch (response.StatusCode)\r
+ {\r
+ case HttpStatusCode.OK:\r
+ case HttpStatusCode.NoContent:\r
+ var tags = GetMeta(response,"X-Container-Meta-");\r
+ var policies = GetMeta(response,"X-Container-Policy-");\r
+\r
+ var containerInfo = new ContainerInfo\r
+ {\r
+ Account = account,\r
+ Name = container,\r
+ StorageUrl = StorageUrl.ToString(),\r
+ Count =long.Parse(response.Headers.GetFirstValue("X-Container-Object-Count")),\r
+ Bytes = long.Parse(response.Headers.GetFirstValue("X-Container-Bytes-Used")),\r
+ BlockHash = response.Headers.GetFirstValue("X-Container-Block-Hash"),\r
+ BlockSize =\r
+ int.Parse(response.Headers.GetFirstValue("X-Container-Block-Size")),\r
+ Last_Modified = response.Content.Headers.LastModified,\r
+ Tags = tags,\r
+ Policies = policies\r
+ };\r
+\r
+\r
+ return containerInfo;\r
+ case HttpStatusCode.NotFound:\r
+ return ContainerInfo.Empty;\r
+ default:\r
+ throw CreateWebException("GetContainerInfo", response.StatusCode);\r
+ }\r
+ } \r
+ }\r
+\r
+ public void CreateContainer(string account, Uri container)\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative","container");\r
+ Contract.EndContractBlock();\r
+\r
+ var targetUri=GetTargetUri(account).Combine(container);\r
+ var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
+ message.Headers.Add("Content-Length", "0");\r
+ using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)\r
+ { \r
+ var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};\r
+ if (!expectedCodes.Contains(response.StatusCode))\r
+ throw CreateWebException("CreateContainer", response.StatusCode);\r
+ }\r
+/*\r
+ using (var client = new RestClient(_baseClient))\r
+ {\r
+ if (!String.IsNullOrWhiteSpace(account))\r
+ client.BaseAddress = GetAccountUrl(account);\r
+\r
+ client.PutWithRetry(container, 3);\r
+ var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};\r
+ if (!expectedCodes.Contains(client.StatusCode))\r
+ throw CreateWebException("CreateContainer", client.StatusCode);\r
+ }\r
+*/\r
+ }\r
+\r
+ public async Task WipeContainer(string account, Uri container)\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative", "container");\r
+ Contract.EndContractBlock();\r
+\r
+ await DeleteContainer(account, new Uri(String.Format("{0}?delimiter=/", container), UriKind.Relative)).ConfigureAwait(false);\r
+ }\r
+\r
+\r
+ public async Task DeleteContainer(string account, Uri container)\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative","container");\r
+ Contract.EndContractBlock();\r
+\r
+ var targetUri = GetTargetUri(account).Combine(container);\r
+ var message = new HttpRequestMessage(HttpMethod.Delete, targetUri);\r
+ using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))\r
+ {\r
+ var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent };\r
+ if (!expectedCodes.Contains(response.StatusCode))\r
+ throw CreateWebException("DeleteContainer", response.StatusCode);\r
+ }\r
+\r
+ }\r
+\r
+ /// <summary>\r
+ /// \r
+ /// </summary>\r
+ /// <param name="account"></param>\r
+ /// <param name="container"></param>\r
+ /// <param name="objectName"></param>\r
+ /// <param name="fileName"></param>\r
+ /// <param name="cancellationToken"> </param>\r
+ /// <returns></returns>\r
+ /// <remarks>This method should have no timeout or a very long one</remarks>\r
+ //Asynchronously download the object specified by *objectName* in a specific *container* to \r
+ // a local file\r
+ public async Task GetObject(string account, Uri container, Uri objectName, string fileName,CancellationToken cancellationToken)\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative","container");\r
+ if (objectName == null)\r
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
+ if (objectName.IsAbsoluteUri)\r
+ throw new ArgumentException("The objectName must be relative","objectName");\r
+ Contract.EndContractBlock();\r
+ \r
+\r
+ try\r
+ {\r
+ //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient\r
+ //object to avoid concurrency errors.\r
+ //\r
+ //Download operations take a long time therefore they have no timeout.\r
+ using(var client = new RestClient(_baseClient) { Timeout = 0 })\r
+ {\r
+ if (!String.IsNullOrWhiteSpace(account))\r
+ client.BaseAddress = GetAccountUrl(account);\r
+\r
+ //The container and objectName are relative names. They are joined with the client's\r
+ //BaseAddress to create the object's absolute address\r
+ var builder = client.GetAddressBuilder(container, objectName);\r
+ var uri = builder.Uri;\r
+\r
+ //Download progress is reported to the Trace log\r
+ Log.InfoFormat("[GET] START {0}", objectName);\r
+ /*client.DownloadProgressChanged += (sender, args) =>\r
+ Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
+ fileName, args.ProgressPercentage,\r
+ args.BytesReceived,\r
+ args.TotalBytesToReceive);*/\r
+ var progress = new Progress<DownloadProgressChangedEventArgs>(args =>\r
+ {\r
+ Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
+ fileName, args.ProgressPercentage,\r
+ args.BytesReceived,\r
+ args.TotalBytesToReceive);\r
+ if (DownloadProgressChanged!=null)\r
+ DownloadProgressChanged(this, new DownloadArgs(args));\r
+ });\r
+ \r
+ //Start downloading the object asynchronously \r
+ await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false);\r
+\r
+ //Once the download completes\r
+ //Delete the local client object\r
+ }\r
+ //And report failure or completion\r
+ }\r
+ catch (Exception exc)\r
+ {\r
+ Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);\r
+ throw;\r
+ }\r
+\r
+ Log.InfoFormat("[GET] END {0}", objectName); \r
+\r
+\r
+ }\r
+\r
+ public async Task<IList<string>> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash)\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative","container");\r
+ if (objectName == null)\r
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
+ if (objectName.IsAbsoluteUri)\r
+ throw new ArgumentException("The objectName must be relative","objectName");\r
+ if (hash == null)\r
+ throw new ArgumentNullException("hash");\r
+ if (String.IsNullOrWhiteSpace(Token))\r
+ throw new InvalidOperationException("Invalid Token");\r
+ if (StorageUrl == null)\r
+ throw new InvalidOperationException("Invalid Storage Url");\r
+ Contract.EndContractBlock();\r
+\r
+ \r
+\r
+ //The container and objectName are relative names. They are joined with the client's\r
+ //BaseAddress to create the object's absolute address\r
+\r
+ var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);\r
+ \r
+\r
+ var uri = new Uri(String.Format("{0}?format=json&hashmap",targetUri),UriKind.Absolute);\r
+\r
+ \r
+ //Send the tree hash as Json to the server \r
+ var jsonHash = hash.ToJson();\r
+ if (Log.IsDebugEnabled)\r
+ Log.DebugFormat("Hashes:\r\n{0}", jsonHash);\r
+\r
+ var message = new HttpRequestMessage(HttpMethod.Put, uri)\r
+ {\r
+ Content = new StringContent(jsonHash)\r
+ };\r
+ message.Headers.Add("ETag",hash.TopHash.ToHashString());\r
+ \r
+ //Don't use a timeout because putting the hashmap may be a long process\r
+\r
+ using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false))\r
+ {\r
+ var empty = (IList<string>)new List<string>();\r
+ \r
+ switch (response.StatusCode)\r
+ {\r
+ case HttpStatusCode.Created:\r
+ //The server will respond either with 201-created if all blocks were already on the server\r
+ return empty;\r
+ case HttpStatusCode.Conflict:\r
+ //or with a 409-conflict and return the list of missing parts\r
+ using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))\r
+ using(var reader=stream.GetLoggedReader(Log))\r
+ { \r
+ var serializer = new JsonSerializer(); \r
+ serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);\r
+ var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));\r
+ return hashes;\r
+ } \r
+ default:\r
+ //All other cases are unexpected\r
+ //Ensure that failure codes raise exceptions\r
+ response.EnsureSuccessStatusCode();\r
+ //And log any other codes as warngings, but continute processing\r
+ Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",response.StatusCode,response.ReasonPhrase);\r
+ return empty;\r
+ }\r
+ }\r
+\r
+ }\r
+\r
+\r
+ public async Task<byte[]> GetBlock(string account, Uri container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)\r
+ {\r
+ if (String.IsNullOrWhiteSpace(Token))\r
+ throw new InvalidOperationException("Invalid Token");\r
+ if (StorageUrl == null)\r
+ throw new InvalidOperationException("Invalid Storage Url");\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative","container");\r
+ if (relativeUrl == null)\r
+ throw new ArgumentNullException("relativeUrl");\r
+ if (end.HasValue && end < 0)\r
+ throw new ArgumentOutOfRangeException("end");\r
+ if (start < 0)\r
+ throw new ArgumentOutOfRangeException("start");\r
+ Contract.EndContractBlock();\r
+\r
+\r
+ var targetUri = GetTargetUri(account).Combine(container).Combine(relativeUrl);\r
+ var message = new HttpRequestMessage(HttpMethod.Get, targetUri);\r
+ message.Headers.Range=new RangeHeaderValue(start,end);\r
+\r
+ //Don't use a timeout because putting the hashmap may be a long process\r
+\r
+ IProgress<DownloadArgs> progress = new Progress<DownloadArgs>(args =>\r
+ {\r
+ Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",\r
+ targetUri.Segments.Last(), args.ProgressPercentage,\r
+ args.BytesReceived,\r
+ args.TotalBytesToReceive);\r
+\r
+ if (DownloadProgressChanged!=null)\r
+ DownloadProgressChanged(this, args);\r
+ });\r
+\r
+\r
+ using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false,HttpCompletionOption.ResponseHeadersRead,\r
+ cancellationToken).ConfigureAwait(false))\r
+ using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))\r
+ using(var targetStream=new MemoryStream())\r
+ {\r
+ \r
+ long totalSize = response.Content.Headers.ContentLength ?? 0;\r
+ long total = 0;\r
+ var buffer = new byte[65536];\r
+ int read;\r
+ while ((read = await stream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) != 0)\r
+ {\r
+ total += read;\r
+ progress.Report(new DownloadArgs(total, totalSize));\r
+ await targetStream.WriteAsync(buffer, 0, read).ConfigureAwait(false);\r
+ }\r
+\r
+ var result = targetStream.ToArray();\r
+ return result;\r
+ }\r
+ \r
+ }\r
+\r
+ public event EventHandler<UploadArgs> UploadProgressChanged;\r
+ public event EventHandler<DownloadArgs> DownloadProgressChanged;\r
+ \r
+\r
+ public async Task PostBlock(string account, Uri container, byte[] block, int offset, int count,string blockHash,CancellationToken token)\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative","container");\r
+ if (block == null)\r
+ throw new ArgumentNullException("block");\r
+ if (offset < 0 || offset >= block.Length)\r
+ throw new ArgumentOutOfRangeException("offset");\r
+ if (count < 0 || count > block.Length)\r
+ throw new ArgumentOutOfRangeException("count");\r
+ if (String.IsNullOrWhiteSpace(Token))\r
+ throw new InvalidOperationException("Invalid Token");\r
+ if (StorageUrl == null)\r
+ throw new InvalidOperationException("Invalid Storage Url"); \r
+ Contract.EndContractBlock();\r
+\r
+\r
+ try\r
+ {\r
+ var containerUri = GetTargetUri(account).Combine(container);\r
+ var targetUri = new Uri(String.Format("{0}?update", containerUri));\r
+\r
+\r
+ //Don't use a timeout because putting the hashmap may be a long process\r
+\r
+\r
+ Log.InfoFormat("[BLOCK POST] START");\r
+\r
+\r
+ var progress = new Progress<UploadArgs>(args =>\r
+ {\r
+ Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",\r
+ args.ProgressPercentage,\r
+ args.BytesSent,\r
+ args.TotalBytesToSend);\r
+ if (UploadProgressChanged != null)\r
+ UploadProgressChanged(this,args);\r
+ });\r
+\r
+ var message = new HttpRequestMessage(HttpMethod.Post, targetUri)\r
+ {\r
+ Content = new ByteArrayContentWithProgress(block, offset, count,progress)\r
+ };\r
+ message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream");\r
+\r
+ //Send the block\r
+ using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3,false,HttpCompletionOption.ResponseContentRead,token).ConfigureAwait(false))\r
+ { \r
+ Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");\r
+ response.EnsureSuccessStatusCode();\r
+ var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false);\r
+ var cleanHash = responseHash.TrimEnd();\r
+ Debug.Assert(blockHash==cleanHash);\r
+ if (responseHash.Equals(cleanHash,StringComparison.OrdinalIgnoreCase))\r
+ Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]",account,container,blockHash,responseHash);\r
+ }\r
+ Log.InfoFormat("[BLOCK POST] END"); \r
+ }\r
+ catch (TaskCanceledException )\r
+ {\r
+ Log.Info("Aborting block");\r
+ throw;\r
+ }\r
+ catch (Exception exc)\r
+ {\r
+ Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);\r
+ throw;\r
+ }\r
+ }\r
+\r
+\r
+ public async Task<TreeHash> GetHashMap(string account, Uri container, Uri objectName)\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative","container");\r
+ if (objectName == null)\r
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
+ if (objectName.IsAbsoluteUri)\r
+ throw new ArgumentException("The objectName must be relative","objectName");\r
+ if (String.IsNullOrWhiteSpace(Token))\r
+ throw new InvalidOperationException("Invalid Token");\r
+ if (StorageUrl == null)\r
+ throw new InvalidOperationException("Invalid Storage Url");\r
+ Contract.EndContractBlock();\r
+\r
+ try\r
+ {\r
+\r
+ var objectUri = GetTargetUri(account).Combine(container).Combine(objectName);\r
+ var targetUri = new Uri(String.Format("{0}?format=json&hashmap", objectUri));\r
+\r
+ //Start downloading the object asynchronously\r
+ var json = await GetStringAsync(targetUri, "").ConfigureAwait(false);\r
+ var treeHash = TreeHash.Parse(json);\r
+ Log.InfoFormat("[GET HASH] END {0}", objectName);\r
+ return treeHash;\r
+\r
+ }\r
+ catch (Exception exc)\r
+ {\r
+ Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);\r
+ throw;\r
+ }\r
+\r
+ }\r
+\r
+\r
+ /// <summary>\r
+ /// \r
+ /// </summary>\r
+ /// <param name="account"></param>\r
+ /// <param name="container"></param>\r
+ /// <param name="objectName"></param>\r
+ /// <param name="fileName"></param>\r
+ /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>\r
+ /// <param name="contentType"> </param>\r
+ /// <remarks>>This method should have no timeout or a very long one</remarks>\r
+ public async Task PutObject(string account, Uri container, Uri objectName, string fileName, string hash = Signature.MERKLE_EMPTY, string contentType = "application/octet-stream")\r
+ {\r
+ if (container == null)\r
+ throw new ArgumentNullException("container", "The container property can't be empty");\r
+ if (container.IsAbsoluteUri)\r
+ throw new ArgumentException("The container must be relative","container");\r
+ if (objectName == null)\r
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
+ if (objectName.IsAbsoluteUri)\r
+ throw new ArgumentException("The objectName must be relative","objectName");\r
+ if (String.IsNullOrWhiteSpace(fileName))\r
+ throw new ArgumentNullException("fileName", "The fileName property can't be empty");\r
+ try\r
+ {\r
+\r
+ using (var client = new RestClient(_baseClient) { Timeout = 0 })\r
+ {\r
+ if (!String.IsNullOrWhiteSpace(account))\r
+ client.BaseAddress = GetAccountUrl(account);\r
+\r
+ var builder = client.GetAddressBuilder(container, objectName);\r
+ var uri = builder.Uri;\r
+\r
+ string etag = hash ;\r
+\r
+ client.Headers.Add("Content-Type", contentType);\r
+ if (contentType!=ObjectInfo.CONTENT_TYPE_DIRECTORY)\r
+ client.Headers.Add("ETag", etag);\r
+\r
+\r
+ Log.InfoFormat("[PUT] START {0}", objectName);\r
+ client.UploadProgressChanged += (sender, args) =>\r
+ {\r
+ using (ThreadContext.Stacks["PUT"].Push("Progress"))\r
+ {\r
+ Log.InfoFormat("{0} {1}% {2} of {3}", fileName,\r
+ args.ProgressPercentage,\r
+ args.BytesSent, args.TotalBytesToSend);\r
+ }\r
+ };\r
+\r
+ client.UploadFileCompleted += (sender, args) =>\r
+ {\r
+ using (ThreadContext.Stacks["PUT"].Push("Progress"))\r
+ {\r
+ Log.InfoFormat("Completed {0}", fileName);\r
+ }\r
+ }; \r
+ \r
+ if (contentType==ObjectInfo.CONTENT_TYPE_DIRECTORY)\r
+ await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false);\r
+ else\r
+ await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false);\r
+ }\r
+\r
+ Log.InfoFormat("[PUT] END {0}", objectName);\r
+ }\r
+ catch (Exception exc)\r
+ {\r
+ Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);\r
+ throw;\r
+ } \r
+\r
+ }\r
+ \r
+ public void MoveObject(string account, Uri sourceContainer, Uri oldObjectName, Uri targetContainer, Uri newObjectName)\r
+ {\r
+ if (sourceContainer == null)\r
+ throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");\r
+ if (sourceContainer.IsAbsoluteUri)\r
+ throw new ArgumentException("The sourceContainer must be relative","sourceContainer");\r
+ if (oldObjectName == null)\r
+ throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");\r
+ if (oldObjectName.IsAbsoluteUri)\r
+ throw new ArgumentException("The oldObjectName must be relative","oldObjectName");\r
+ if (targetContainer == null)\r
+ throw new ArgumentNullException("targetContainer", "The targetContainer property can't be empty");\r
+ if (targetContainer.IsAbsoluteUri)\r
+ throw new ArgumentException("The targetContainer must be relative","targetContainer");\r
+ if (newObjectName == null)\r
+ throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");\r
+ if (newObjectName.IsAbsoluteUri)\r
+ throw new ArgumentException("The newObjectName must be relative","newObjectName");\r
+ Contract.EndContractBlock();\r
+\r
+ var baseUri = GetTargetUri(account);\r
+ var targetUri = baseUri.Combine(targetContainer).Combine(newObjectName);\r
+ var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, oldObjectName),UriKind.Relative);\r
+\r
+ var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
+ message.Headers.Add("X-Move-From", sourceUri.ToString());\r
+ using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)\r
+ {\r
+ var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
+ if (!expectedCodes.Contains(response.StatusCode))\r
+ throw CreateWebException("MoveObject", response.StatusCode);\r
+ }\r
+ }\r
+\r
+ public async Task DeleteObject(string account, Uri sourceContainer, Uri objectName, bool isDirectory)\r
+ {\r
+ if (sourceContainer == null)\r
+ throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");\r
+ if (sourceContainer.IsAbsoluteUri)\r
+ throw new ArgumentException("The sourceContainer must be relative","sourceContainer");\r
+ if (objectName == null)\r
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");\r
+ if (objectName.IsAbsoluteUri)\r
+ throw new ArgumentException("The objectName must be relative","objectName");\r
+ Contract.EndContractBlock();\r
+\r
+\r
+\r
+ var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, objectName),UriKind.Relative);\r
+\r
+ \r
+ if (objectName.OriginalString.EndsWith(".ignore"))\r
+ using(var response = await _baseHttpClient.DeleteAsync(sourceUri)){}\r
+ else\r
+ {\r
+ var relativeUri = new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName),\r
+ UriKind.Relative);\r
+\r
+/*\r
+ var relativeUri = isDirectory\r
+ ? new Uri(\r
+ String.Format("{0}/{1}?delimiter=/", FolderConstants.TrashContainer,\r
+ objectName), UriKind.Relative)\r
+ : new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName),\r
+ UriKind.Relative);\r
+\r
+*/\r
+ var targetUri = GetTargetUri(account).Combine(relativeUri);\r
+\r
+\r
+ var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
+ message.Headers.Add("X-Move-From", sourceUri.ToString());\r
+\r
+ Log.InfoFormat("[TRASH] [{0}] to [{1}]", sourceUri, targetUri);\r
+ using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3))\r
+ {\r
+ var expectedCodes = new[]\r
+ {\r
+ HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,\r
+ HttpStatusCode.NotFound\r
+ };\r
+ if (!expectedCodes.Contains(response.StatusCode))\r
+ throw CreateWebException("DeleteObject", response.StatusCode);\r
+ }\r
+ }\r
+/*\r
+ \r
+\r
+ var targetUrl = FolderConstants.TrashContainer + "/" + objectName;\r
+/*\r
+ if (isDirectory)\r
+ targetUrl = targetUrl + "?delimiter=/";\r
+#1#\r
+\r
+ var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);\r
+\r
+ using (var client = new RestClient(_baseClient))\r
+ {\r
+ if (!String.IsNullOrWhiteSpace(account))\r
+ client.BaseAddress = GetAccountUrl(account);\r
+\r
+ client.Headers.Add("X-Move-From", sourceUrl);\r
+ client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);\r
+ Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);\r
+ client.PutWithRetry(new Uri(targetUrl,UriKind.Relative), 3);\r
+\r
+ var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};\r
+ if (!expectedCodes.Contains(client.StatusCode))\r
+ throw CreateWebException("DeleteObject", client.StatusCode);\r
+ }\r
+*/\r
+ }\r
+\r
+ \r
+ private static WebException CreateWebException(string operation, HttpStatusCode statusCode)\r
+ {\r
+ return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));\r
+ }\r
+\r
+\r
+ public async Task<bool> CanUpload(string account, ObjectInfo cloudFile)\r
+ {\r
+ Contract.Requires(!String.IsNullOrWhiteSpace(account));\r
+ Contract.Requires(cloudFile!=null);\r
+\r
+ var parts = cloudFile.Name.ToString().Split('/');\r
+ var folder = String.Join("/", parts,0,parts.Length-1);\r
+\r
+ var fileName = String.Format("{0}/{1}.pithos.ignore", folder, Guid.NewGuid());\r
+ var fileUri=fileName.ToEscapedUri(); \r
+\r
+ try\r
+ {\r
+ var relativeUri = cloudFile.Container.Combine(fileUri);\r
+ var targetUri = GetTargetUri(account).Combine(relativeUri);\r
+ var message = new HttpRequestMessage(HttpMethod.Put, targetUri);\r
+ message.Content.Headers.ContentType =new MediaTypeHeaderValue("application/octet-stream");\r
+ var response=await _baseHttpClient.SendAsyncWithRetries(message, 3); \r
+ var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};\r
+ var result=(expectedCodes.Contains(response.StatusCode));\r
+ await DeleteObject(account, cloudFile.Container, fileUri, cloudFile.IsDirectory);\r
+ return result;\r
+ }\r
+ catch\r
+ {\r
+ return false;\r
+ }\r
+ \r
+ }\r
+\r
+ ~CloudFilesClient()\r
+ {\r
+ Dispose(false);\r
+ }\r
+\r
+ public void Dispose()\r
+ {\r
+ Dispose(true);\r
+ GC.SuppressFinalize(this);\r
+ }\r
+\r
+ protected virtual void Dispose(bool disposing)\r
+ {\r
+ if (disposing)\r
+ {\r
+ if (_httpClientHandler!=null)\r
+ _httpClientHandler.Dispose();\r
+ if (_baseClient!=null)\r
+ _baseClient.Dispose();\r
+ if(_baseHttpClient!=null)\r
+ _baseHttpClient.Dispose();\r
+ if (_baseHttpClientNoTimeout!=null)\r
+ _baseHttpClientNoTimeout.Dispose();\r
+ }\r
+ _httpClientHandler = null;\r
+ _baseClient = null;\r
+ _baseHttpClient = null;\r
+ _baseHttpClientNoTimeout = null;\r
+ }\r
+ }\r
+\r
+ public class ShareAccountInfo\r
+ {\r
+ public DateTime? last_modified { get; set; }\r
+ public string name { get; set; }\r
+ }\r
+}\r
projects / pithos-ms-client / blobdiff