X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/blobdiff_plain/07ffe53596b17db4876a471907f82317937faad9..cfb091032c1ea51d109dccf68322ce2ee63eaab5:/trunk/Pithos.Network/CloudFilesClient.cs
diff --git a/trunk/Pithos.Network/CloudFilesClient.cs b/trunk/Pithos.Network/CloudFilesClient.cs
index 4bc2a7c..cef59f5 100644
--- a/trunk/Pithos.Network/CloudFilesClient.cs
+++ b/trunk/Pithos.Network/CloudFilesClient.cs
@@ -1,1565 +1,1884 @@
-#region
-/* -----------------------------------------------------------------------
- *
- *
- * Copyright 2011-2012 GRNET S.A. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * 1. Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- *
- * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- *
- * The views and conclusions contained in the software and
- * documentation are those of the authors and should not be
- * interpreted as representing official policies, either expressed
- * or implied, of GRNET S.A.
- *
- * -----------------------------------------------------------------------
- */
-#endregion
-
-// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
-//
-// The class provides methods to upload/download files, delete files, manage containers
-
-
-using System;
-using System.Collections.Generic;
-using System.Collections.Specialized;
-using System.ComponentModel.Composition;
-using System.Diagnostics;
-using System.Diagnostics.Contracts;
-using System.IO;
-using System.Linq;
-using System.Net;
-using System.Reflection;
-using System.Security.Cryptography;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
-using Newtonsoft.Json;
-using Pithos.Interfaces;
-using log4net;
-
-namespace Pithos.Network
-{
- [Export(typeof(ICloudClient))]
- public class CloudFilesClient:ICloudClient
- {
- private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
-
- //CloudFilesClient uses *_baseClient* internally to communicate with the server
- //RestClient provides a REST-friendly interface over the standard WebClient.
- private RestClient _baseClient;
-
-
- //During authentication the client provides a UserName
- public string UserName { get; set; }
-
- //and and ApiKey to the server
- public string ApiKey { get; set; }
-
- //And receives an authentication Token. This token must be provided in ALL other operations,
- //in the X-Auth-Token header
- private string _token;
- private readonly string _emptyGuid = Guid.Empty.ToString();
-
-
- public string Token
- {
- get { return _token; }
- set
- {
- _token = value;
- _baseClient.Headers["X-Auth-Token"] = value;
- }
- }
-
- //The client also receives a StorageUrl after authentication. All subsequent operations must
- //use this url
- public Uri StorageUrl { get; set; }
-
-
- public Uri RootAddressUri { get; set; }
-
- /* private WebProxy _proxy;
- public WebProxy Proxy
- {
- get { return _proxy; }
- set
- {
- _proxy = value;
- if (_baseClient != null)
- _baseClient.Proxy = value;
- }
- }
-*/
-
- /* private Uri _proxy;
- public Uri Proxy
- {
- get { return _proxy; }
- set
- {
- _proxy = value;
- if (_baseClient != null)
- _baseClient.Proxy = new WebProxy(value);
- }
- }*/
-
- public double DownloadPercentLimit { get; set; }
- public double UploadPercentLimit { get; set; }
-
- public string AuthenticationUrl { get; set; }
-
-
- public string VersionPath
- {
- get { return UsePithos ? "v1" : "v1.0"; }
- }
-
- public bool UsePithos { get; set; }
-
-
-
- public CloudFilesClient(string userName, string apiKey)
- {
- UserName = userName;
- ApiKey = apiKey;
- }
-
- public CloudFilesClient(AccountInfo accountInfo)
- {
- if (accountInfo==null)
- throw new ArgumentNullException("accountInfo");
- Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
- Contract.Ensures(StorageUrl != null);
- Contract.Ensures(_baseClient != null);
- Contract.Ensures(RootAddressUri != null);
- Contract.EndContractBlock();
-
- _baseClient = new RestClient
- {
- BaseAddress = accountInfo.StorageUri.ToString(),
- Timeout = 30000,
- Retries = 3,
- };
- StorageUrl = accountInfo.StorageUri;
- Token = accountInfo.Token;
- UserName = accountInfo.UserName;
-
- //Get the root address (StorageUrl without the account)
- var storageUrl = StorageUrl.AbsoluteUri;
- var usernameIndex = storageUrl.LastIndexOf(UserName);
- var rootUrl = storageUrl.Substring(0, usernameIndex);
- RootAddressUri = new Uri(rootUrl);
- }
-
-
- public AccountInfo Authenticate()
- {
- if (String.IsNullOrWhiteSpace(UserName))
- throw new InvalidOperationException("UserName is empty");
- if (String.IsNullOrWhiteSpace(ApiKey))
- throw new InvalidOperationException("ApiKey is empty");
- if (String.IsNullOrWhiteSpace(AuthenticationUrl))
- throw new InvalidOperationException("AuthenticationUrl is empty");
- Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
- Contract.Ensures(StorageUrl != null);
- Contract.Ensures(_baseClient != null);
- Contract.Ensures(RootAddressUri != null);
- Contract.EndContractBlock();
-
-
- Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
-
- var groups = new List();
-
- using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
- {
- /* if (Proxy != null)
- authClient.Proxy = Proxy;*/
-
- Contract.Assume(authClient.Headers!=null);
-
- authClient.Headers.Add("X-Auth-User", UserName);
- authClient.Headers.Add("X-Auth-Key", ApiKey);
- //TODO: Remove after testing. Added to overcome server auth bug
- //authClient.Headers.Add("X-Auth-Token", ApiKey);
-
- authClient.DownloadStringWithRetry(VersionPath, 3);
-
- authClient.AssertStatusOK("Authentication failed");
-
- var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
- if (String.IsNullOrWhiteSpace(storageUrl))
- throw new InvalidOperationException("Failed to obtain storage url");
-
- _baseClient = new RestClient
- {
- BaseAddress = storageUrl,
- Timeout = 10000,
- Retries = 3,
- //Proxy=Proxy
- };
-
- StorageUrl = new Uri(storageUrl);
-
- //Get the root address (StorageUrl without the account)
- var usernameIndex=storageUrl.LastIndexOf(UserName);
- var rootUrl = storageUrl.Substring(0, usernameIndex);
- RootAddressUri = new Uri(rootUrl);
-
- var token = authClient.GetHeaderValue("X-Auth-Token");
- if (String.IsNullOrWhiteSpace(token))
- throw new InvalidOperationException("Failed to obtain token url");
- Token = token;
-
- /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
- groups = (from key in keys
- where key.StartsWith("X-Account-Group-")
- let name = key.Substring(16)
- select new Group(name, authClient.ResponseHeaders[key]))
- .ToList();
-
-*/
- }
-
- Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
- Debug.Assert(_baseClient!=null);
-
- return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};
-
- }
-
-
-
- public IList ListContainers(string account)
- {
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
- var content = client.DownloadStringWithRetry("", 3);
- client.AssertStatusOK("List Containers failed");
-
- if (client.StatusCode == HttpStatusCode.NoContent)
- return new List();
- var infos = JsonConvert.DeserializeObject>(content);
-
- foreach (var info in infos)
- {
- info.Account = account;
- }
- return infos;
- }
-
- }
-
- private string GetAccountUrl(string account)
- {
- return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
- }
-
- public IList ListSharingAccounts(DateTime? since=null)
- {
- using (ThreadContext.Stacks["Share"].Push("List Accounts"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
- client.IfModifiedSince = since;
-
- //Extract the username from the base address
- client.BaseAddress = RootAddressUri.AbsoluteUri;
-
- var content = client.DownloadStringWithRetry(@"", 3);
-
- client.AssertStatusOK("ListSharingAccounts failed");
-
- //If the result is empty, return an empty list,
- var infos = String.IsNullOrWhiteSpace(content)
- ? new List()
- //Otherwise deserialize the account list into a list of ShareAccountInfos
- : JsonConvert.DeserializeObject>(content);
-
- Log.DebugFormat("END");
- return infos;
- }
- }
- }
-
-
- ///
- /// Request listing of all objects in a container modified since a specific time.
- /// If the *since* value is missing, return all objects
- ///
- /// Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new
- /// and should be polled anyway
- ///
- ///
- ///
- public IList ListSharedObjects(HashSet knownContainers,DateTime? since = null )
- {
-
- using (ThreadContext.Stacks["Share"].Push("List Objects"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
- //'since' is not used here because we need to have ListObjects return a NoChange result
- //for all shared accounts,containers
-
- Func GetKey = c => String.Format("{0}\\{1}", c.Account, c.Name);
-
- var accounts = ListSharingAccounts();
- var containers = (from account in accounts
- let conts = ListContainers(account.name)
- from container in conts
- select container).ToList();
- var items = from container in containers
- let actualSince=knownContainers.Contains(GetKey(container))?since:null
- select ListObjects(container.Account , container.Name, actualSince);
- var objects=items.SelectMany(r=> r).ToList();
-
- //For each object
- //Check parents recursively up to (but not including) the container.
- //If parents are missing, add them to the list
- //Need function to calculate all parent URLs
- objects = AddMissingParents(objects);
-
- //Store any new containers
- foreach (var container in containers)
- {
- knownContainers.Add(GetKey(container));
- }
-
-
-
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
- return objects;
- }
- }
-
- private List AddMissingParents(List objects)
- {
- //TODO: Remove short-circuit when we decide to use Missing Parents functionality
- //return objects;
-
- var existingUris = objects.ToDictionary(o => o.Uri, o => o);
- foreach (var objectInfo in objects)
- {
- //Can be null when retrieving objects to show in selective sync
- if (objectInfo.Name == null)
- continue;
-
- var parts = objectInfo.Name.Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries);
- //If there is no parent, skip
- if (parts.Length == 1)
- continue;
- var baseParts = new[]
- {
- objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container
- };
- for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++)
- {
- var nameparts = parts.Range(0, partIdx).ToArray();
- var parentName= String.Join("/", nameparts);
-
- var parentParts = baseParts.Concat(nameparts);
- var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts);
-
- var parentUri = new Uri(parentUrl, UriKind.Absolute);
-
- ObjectInfo existingInfo;
- if (!existingUris.TryGetValue(parentUri,out existingInfo))
- {
- var h = parentUrl.GetHashCode();
- var reverse = new string(parentUrl.Reverse().ToArray());
- var rh = reverse.GetHashCode();
- var b1 = BitConverter.GetBytes(h);
- var b2 = BitConverter.GetBytes(rh);
- var g = new Guid(0,0,0,b1.Concat(b2).ToArray());
-
-
- existingUris[parentUri] = new ObjectInfo
- {
- Account = objectInfo.Account,
- Container = objectInfo.Container,
- Content_Type = @"application/directory",
- ETag = Signature.MD5_EMPTY,
- X_Object_Hash = Signature.MERKLE_EMPTY,
- Name=parentName,
- StorageUri=objectInfo.StorageUri,
- Bytes = 0,
- UUID=g.ToString(),
- };
- }
- }
- }
- return existingUris.Values.ToList();
- }
-
- public void SetTags(ObjectInfo target,IDictionary tags)
- {
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("The Token is not set");
- if (StorageUrl == null)
- throw new InvalidOperationException("The StorageUrl is not set");
- if (target == null)
- throw new ArgumentNullException("target");
- Contract.EndContractBlock();
-
- using (ThreadContext.Stacks["Share"].Push("Share Object"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
-
- client.BaseAddress = GetAccountUrl(target.Account);
-
- client.Parameters.Clear();
- client.Parameters.Add("update", "");
-
- foreach (var tag in tags)
- {
- var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
- client.Headers.Add(headerTag, tag.Value);
- }
-
- client.DownloadStringWithRetry(target.Container, 3);
-
-
- client.AssertStatusOK("SetTags failed");
- //If the status is NOT ACCEPTED we have a problem
- if (client.StatusCode != HttpStatusCode.Accepted)
- {
- Log.Error("Failed to set tags");
- throw new Exception("Failed to set tags");
- }
-
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
- }
- }
-
-
- }
-
- public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
- {
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("The Token is not set");
- if (StorageUrl==null)
- throw new InvalidOperationException("The StorageUrl is not set");
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName");
- if (String.IsNullOrWhiteSpace(account))
- throw new ArgumentNullException("account");
- if (String.IsNullOrWhiteSpace(shareTo))
- throw new ArgumentNullException("shareTo");
- Contract.EndContractBlock();
-
- using (ThreadContext.Stacks["Share"].Push("Share Object"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
-
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
-
- string permission = "";
- if (write)
- permission = String.Format("write={0}", shareTo);
- else if (read)
- permission = String.Format("read={0}", shareTo);
- client.Headers.Add("X-Object-Sharing", permission);
-
- var content = client.DownloadStringWithRetry(container, 3);
-
- client.AssertStatusOK("ShareObject failed");
-
- //If the result is empty, return an empty list,
- var infos = String.IsNullOrWhiteSpace(content)
- ? new List()
- //Otherwise deserialize the object list into a list of ObjectInfos
- : JsonConvert.DeserializeObject>(content);
-
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
- }
- }
-
-
- }
-
- public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
- {
- if (accountInfo==null)
- throw new ArgumentNullException("accountInfo");
- Contract.EndContractBlock();
-
- using (ThreadContext.Stacks["Account"].Push("GetPolicies"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
- client.BaseAddress = GetAccountUrl(accountInfo.UserName);
-
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
- client.Head(String.Empty, 3);
-
- var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
- var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
-
- long quota, bytes;
- if (long.TryParse(quotaValue, out quota))
- accountInfo.Quota = quota;
- if (long.TryParse(bytesValue, out bytes))
- accountInfo.BytesUsed = bytes;
-
- return accountInfo;
-
- }
-
- }
- }
-
- public void UpdateMetadata(ObjectInfo objectInfo)
- {
- if (objectInfo == null)
- throw new ArgumentNullException("objectInfo");
- Contract.EndContractBlock();
-
- using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
-
- using(var client=new RestClient(_baseClient))
- {
-
- client.BaseAddress = GetAccountUrl(objectInfo.Account);
-
- client.Parameters.Clear();
-
-
- //Set Tags
- foreach (var tag in objectInfo.Tags)
- {
- var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
- client.Headers.Add(headerTag, tag.Value);
- }
-
- //Set Permissions
-
- var permissions=objectInfo.GetPermissionString();
- client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);
-
- client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
- client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
- client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
- var isPublic = objectInfo.IsPublic.ToString().ToLower();
- client.Headers.Add("X-Object-Public", isPublic);
-
-
- /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);
- uriBuilder.Query = "update=";
- var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/
- var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);
- client.PostWithRetry(address,"application/xml");
-
- //client.UploadValues(uri,new NameValueCollection());
-
-
- client.AssertStatusOK("UpdateMetadata failed");
- //If the status is NOT ACCEPTED or OK we have a problem
- if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
- {
- Log.Error("Failed to update metadata");
- throw new Exception("Failed to update metadata");
- }
-
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
- }
- }
-
- }
-
- public void UpdateMetadata(ContainerInfo containerInfo)
- {
- if (containerInfo == null)
- throw new ArgumentNullException("containerInfo");
- Contract.EndContractBlock();
-
- using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
-
- using(var client=new RestClient(_baseClient))
- {
-
- client.BaseAddress = GetAccountUrl(containerInfo.Account);
-
- client.Parameters.Clear();
-
-
- //Set Tags
- foreach (var tag in containerInfo.Tags)
- {
- var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
- client.Headers.Add(headerTag, tag.Value);
- }
-
-
- //Set Policies
- foreach (var policy in containerInfo.Policies)
- {
- var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
- client.Headers.Add(headerPolicy, policy.Value);
- }
-
-
- var uriBuilder = client.GetAddressBuilder(containerInfo.Name,"");
- var uri = uriBuilder.Uri;
-
- client.UploadValues(uri,new NameValueCollection());
-
-
- client.AssertStatusOK("UpdateMetadata failed");
- //If the status is NOT ACCEPTED or OK we have a problem
- if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
- {
- Log.Error("Failed to update metadata");
- throw new Exception("Failed to update metadata");
- }
-
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
- }
- }
-
- }
-
-
- public IList ListObjects(string account, string container, DateTime? since = null)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- Contract.EndContractBlock();
-
- using (ThreadContext.Stacks["Objects"].Push("List"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
- client.IfModifiedSince = since;
- var content = client.DownloadStringWithRetry(container, 3);
-
- client.AssertStatusOK("ListObjects failed");
-
- if (client.StatusCode==HttpStatusCode.NotModified)
- return new[]{new NoModificationInfo(account,container)};
- //If the result is empty, return an empty list,
- var infos = String.IsNullOrWhiteSpace(content)
- ? new List()
- //Otherwise deserialize the object list into a list of ObjectInfos
- : JsonConvert.DeserializeObject>(content);
-
- foreach (var info in infos)
- {
- info.Container = container;
- info.Account = account;
- info.StorageUri = this.StorageUrl;
- }
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
- return infos;
- }
- }
- }
-
- public IList ListObjects(string account, string container, string folder, DateTime? since = null)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
-/*
- if (String.IsNullOrWhiteSpace(folder))
- throw new ArgumentNullException("folder");
-*/
- Contract.EndContractBlock();
-
- using (ThreadContext.Stacks["Objects"].Push("List"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Parameters.Add("format", "json");
- client.Parameters.Add("path", folder);
- client.IfModifiedSince = since;
- var content = client.DownloadStringWithRetry(container, 3);
- client.AssertStatusOK("ListObjects failed");
-
- if (client.StatusCode==HttpStatusCode.NotModified)
- return new[]{new NoModificationInfo(account,container,folder)};
-
- var infos = JsonConvert.DeserializeObject>(content);
- foreach (var info in infos)
- {
- info.Account = account;
- if (info.Container == null)
- info.Container = container;
- info.StorageUri = this.StorageUrl;
- }
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
- return infos;
- }
- }
- }
-
-
- public bool ContainerExists(string account, string container)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- Contract.EndContractBlock();
-
- using (ThreadContext.Stacks["Containters"].Push("Exists"))
- {
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Head(container, 3);
-
- bool result;
- switch (client.StatusCode)
- {
- case HttpStatusCode.OK:
- case HttpStatusCode.NoContent:
- result=true;
- break;
- case HttpStatusCode.NotFound:
- result=false;
- break;
- default:
- throw CreateWebException("ContainerExists", client.StatusCode);
- }
- if (Log.IsDebugEnabled) Log.DebugFormat("END");
-
- return result;
- }
-
- }
- }
-
- public bool ObjectExists(string account, string container, string objectName)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The objectName property can't be empty");
- Contract.EndContractBlock();
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Head(container + "/" + objectName, 3);
-
- switch (client.StatusCode)
- {
- case HttpStatusCode.OK:
- case HttpStatusCode.NoContent:
- return true;
- case HttpStatusCode.NotFound:
- return false;
- default:
- throw CreateWebException("ObjectExists", client.StatusCode);
- }
- }
-
- }
-
- public ObjectInfo GetObjectInfo(string account, string container, string objectName)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The objectName property can't be empty");
- Contract.EndContractBlock();
-
- using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
- {
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
- try
- {
- client.Parameters.Clear();
-
- client.Head(container + "/" + objectName, 3);
-
- if (client.TimedOut)
- return ObjectInfo.Empty;
-
- switch (client.StatusCode)
- {
- case HttpStatusCode.OK:
- case HttpStatusCode.NoContent:
- var keys = client.ResponseHeaders.AllKeys.AsQueryable();
- var tags = client.GetMeta("X-Object-Meta-");
- var extensions = (from key in keys
- where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
- select new {Name = key, Value = client.ResponseHeaders[key]})
- .ToDictionary(t => t.Name, t => t.Value);
-
- var permissions=client.GetHeaderValue("X-Object-Sharing", true);
-
-
- var info = new ObjectInfo
- {
- Account = account,
- Container = container,
- Name = objectName,
- ETag = client.GetHeaderValue("ETag"),
- UUID=client.GetHeaderValue("X-Object-UUID"),
- X_Object_Hash = client.GetHeaderValue("X-Object-Hash"),
- Content_Type = client.GetHeaderValue("Content-Type"),
- Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),
- Tags = tags,
- Last_Modified = client.LastModified,
- Extensions = extensions,
- ContentEncoding=client.GetHeaderValue("Content-Encoding",true),
- ContendDisposition = client.GetHeaderValue("Content-Disposition",true),
- Manifest=client.GetHeaderValue("X-Object-Manifest",true),
- PublicUrl=client.GetHeaderValue("X-Object-Public",true),
- StorageUri=this.StorageUrl,
- };
- info.SetPermissions(permissions);
- return info;
- case HttpStatusCode.NotFound:
- return ObjectInfo.Empty;
- default:
- throw new WebException(
- String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
- objectName, client.StatusCode));
- }
-
- }
- catch (RetryException)
- {
- Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);
- return ObjectInfo.Empty;
- }
- catch (WebException e)
- {
- Log.Error(
- String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
- objectName, client.StatusCode), e);
- throw;
- }
- }
- }
-
- }
-
- public void CreateFolder(string account, string container, string folder)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(folder))
- throw new ArgumentNullException("folder", "The folder property can't be empty");
- Contract.EndContractBlock();
-
- var folderUrl=String.Format("{0}/{1}",container,folder);
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Parameters.Clear();
- client.Headers.Add("Content-Type", @"application/directory");
- client.Headers.Add("Content-Length", "0");
- client.PutWithRetry(folderUrl, 3);
-
- if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
- throw CreateWebException("CreateFolder", client.StatusCode);
- }
- }
-
-
-
- public ContainerInfo GetContainerInfo(string account, string container)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- Contract.EndContractBlock();
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Head(container);
- switch (client.StatusCode)
- {
- case HttpStatusCode.OK:
- case HttpStatusCode.NoContent:
- var tags = client.GetMeta("X-Container-Meta-");
- var policies = client.GetMeta("X-Container-Policy-");
-
- var containerInfo = new ContainerInfo
- {
- Account=account,
- Name = container,
- StorageUrl=this.StorageUrl.ToString(),
- Count =
- long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
- Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
- BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
- BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),
- Last_Modified=client.LastModified,
- Tags=tags,
- Policies=policies
- };
-
-
- return containerInfo;
- case HttpStatusCode.NotFound:
- return ContainerInfo.Empty;
- default:
- throw CreateWebException("GetContainerInfo", client.StatusCode);
- }
- }
- }
-
- public void CreateContainer(string account, string container)
- {
- if (String.IsNullOrWhiteSpace(account))
- throw new ArgumentNullException("account");
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- Contract.EndContractBlock();
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.PutWithRetry(container, 3);
- var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
- if (!expectedCodes.Contains(client.StatusCode))
- throw CreateWebException("CreateContainer", client.StatusCode);
- }
- }
-
- public void DeleteContainer(string account, string container)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- Contract.EndContractBlock();
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.DeleteWithRetry(container, 3);
- var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
- if (!expectedCodes.Contains(client.StatusCode))
- throw CreateWebException("DeleteContainer", client.StatusCode);
- }
-
- }
-
- ///
- ///
- ///
- ///
- ///
- ///
- ///
- ///
- /// This method should have no timeout or a very long one
- //Asynchronously download the object specified by *objectName* in a specific *container* to
- // a local file
- public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The objectName property can't be empty");
- Contract.EndContractBlock();
-
- try
- {
- //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
- //object to avoid concurrency errors.
- //
- //Download operations take a long time therefore they have no timeout.
- using(var client = new RestClient(_baseClient) { Timeout = 0 })
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- //The container and objectName are relative names. They are joined with the client's
- //BaseAddress to create the object's absolute address
- var builder = client.GetAddressBuilder(container, objectName);
- var uri = builder.Uri;
-
- //Download progress is reported to the Trace log
- Log.InfoFormat("[GET] START {0}", objectName);
- /*client.DownloadProgressChanged += (sender, args) =>
- Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
- fileName, args.ProgressPercentage,
- args.BytesReceived,
- args.TotalBytesToReceive);*/
- var progress = new Progress(args =>
- {
- Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
- fileName, args.ProgressPercentage,
- args.BytesReceived,
- args.TotalBytesToReceive);
- DownloadProgressChanged(this, args);
- });
-
- //Start downloading the object asynchronously
- await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false);
-
- //Once the download completes
- //Delete the local client object
- }
- //And report failure or completion
- }
- catch (Exception exc)
- {
- Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);
- throw;
- }
-
- Log.InfoFormat("[GET] END {0}", objectName);
-
-
- }
-
- public Task> PutHashMap(string account, string container, string objectName, TreeHash hash)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName");
- if (hash==null)
- throw new ArgumentNullException("hash");
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("Invalid Token");
- if (StorageUrl == null)
- throw new InvalidOperationException("Invalid Storage Url");
- Contract.EndContractBlock();
-
-
- //Don't use a timeout because putting the hashmap may be a long process
- var client = new RestClient(_baseClient) { Timeout = 0 };
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- //The container and objectName are relative names. They are joined with the client's
- //BaseAddress to create the object's absolute address
- var builder = client.GetAddressBuilder(container, objectName);
- builder.Query = "format=json&hashmap";
- var uri = builder.Uri;
-
-
- //Send the tree hash as Json to the server
- client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
- var jsonHash = hash.ToJson();
-
- client.Headers.Add("ETag",hash.MD5);
- var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);
- if (Log.IsDebugEnabled)
- Log.DebugFormat("Hashes:\r\n{0}", jsonHash);
- return uploadTask.ContinueWith(t =>
- {
-
- var empty = (IList)new List();
-
-
- //The server will respond either with 201-created if all blocks were already on the server
- if (client.StatusCode == HttpStatusCode.Created)
- {
- //in which case we return an empty hash list
- return empty;
- }
- //or with a 409-conflict and return the list of missing parts
- //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception
- if (t.IsFaulted)
- {
- var ex = t.Exception.InnerException;
- var we = ex as WebException;
- var response = we.Response as HttpWebResponse;
- if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
- {
- //In case of 409 the missing parts will be in the response content
- using (var stream = response.GetResponseStream())
- using(var reader=stream.GetLoggedReader(Log))
- {
- //We used to have to cleanup the content before returning it because it contains
- //error content after the list of hashes
- //
- //As of 30/1/2012, the result is a proper Json array so we don't need to read the content
- //line by line
-
- var serializer = new JsonSerializer();
- serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);
- var hashes = (List)serializer.Deserialize(reader, typeof(List));
- return hashes;
- }
- }
- //Any other status code is unexpected and the exception should be rethrown
- Log.LogError(response);
- throw ex;
-
- }
-
- //Any other status code is unexpected but there was no exception. We can probably continue processing
- Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
-
- return empty;
- });
-
- }
-
-
- public async Task GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)
- {
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("Invalid Token");
- if (StorageUrl == null)
- throw new InvalidOperationException("Invalid Storage Url");
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- if (relativeUrl == null)
- throw new ArgumentNullException("relativeUrl");
- if (end.HasValue && end < 0)
- throw new ArgumentOutOfRangeException("end");
- if (start < 0)
- throw new ArgumentOutOfRangeException("start");
- Contract.EndContractBlock();
-
- //Don't use a timeout because putting the hashmap may be a long process
- using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end})
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
- var uri = builder.Uri;
-
-/* client.DownloadProgressChanged += (sender, args) =>
- {
- Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
- uri.Segments.Last(), args.ProgressPercentage,
- args.BytesReceived,
- args.TotalBytesToReceive);
- DownloadProgressChanged(sender, args);
- };*/
- var progress = new Progress(args =>
- {
- Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
- uri.Segments.Last(), args.ProgressPercentage,
- args.BytesReceived,
- args.TotalBytesToReceive);
- DownloadProgressChanged(this, args);
- });
-
-
- var result = await client.DownloadDataTaskAsync(uri, cancellationToken,progress).ConfigureAwait(false);
- return result;
- }
- }
-
- public event UploadProgressChangedEventHandler UploadProgressChanged;
- public event DownloadProgressChangedEventHandler DownloadProgressChanged;
-
- public async Task PostBlock(string account, string container, byte[] block, int offset, int count,CancellationToken token)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- if (block == null)
- throw new ArgumentNullException("block");
- if (offset < 0 || offset >= block.Length)
- throw new ArgumentOutOfRangeException("offset");
- if (count < 0 || count > block.Length)
- throw new ArgumentOutOfRangeException("count");
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("Invalid Token");
- if (StorageUrl == null)
- throw new InvalidOperationException("Invalid Storage Url");
- Contract.EndContractBlock();
-
-
- try
- {
-
- //Don't use a timeout because putting the hashmap may be a long process
- using (var client = new RestClient(_baseClient) { Timeout = 0 })
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- token.Register(client.CancelAsync);
-
- var builder = client.GetAddressBuilder(container, "");
- //We are doing an update
- builder.Query = "update";
- var uri = builder.Uri;
-
- client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
-
- Log.InfoFormat("[BLOCK POST] START");
-
-/*
- client.UploadProgressChanged += (sender, args) =>
- {
- Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
- args.ProgressPercentage, args.BytesSent,
- args.TotalBytesToSend);
- UploadProgressChanged(sender, args);
- };
-*/
- client.UploadFileCompleted += (sender, args) =>
- Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
-
- var progress=new Progress(args=>
- {
- Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
- args.ProgressPercentage, args.BytesSent,
- args.TotalBytesToSend);
- UploadProgressChanged(this, args);
- });
- var buffer = new byte[count];
- Buffer.BlockCopy(block, offset, buffer, 0, count);
- //Send the block
- await client.UploadDataTaskAsync(uri, "POST", buffer,token,progress).ConfigureAwait(false);
- Log.InfoFormat("[BLOCK POST] END");
- }
- }
- catch (TaskCanceledException )
- {
- Log.Info("Aborting block");
- throw;
- }
- catch (Exception exc)
- {
- Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);
- throw;
- }
- }
-
-
- public async Task GetHashMap(string account, string container, string objectName)
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName");
- if (String.IsNullOrWhiteSpace(Token))
- throw new InvalidOperationException("Invalid Token");
- if (StorageUrl == null)
- throw new InvalidOperationException("Invalid Storage Url");
- Contract.EndContractBlock();
-
- try
- {
- //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
- //object to avoid concurrency errors.
- //
- //Download operations take a long time therefore they have no timeout.
- //TODO: Do they really? this is a hashmap operation, not a download
-
- //Start downloading the object asynchronously
- using (var client = new RestClient(_baseClient) { Timeout = 0 })
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- //The container and objectName are relative names. They are joined with the client's
- //BaseAddress to create the object's absolute address
- var builder = client.GetAddressBuilder(container, objectName);
- builder.Query = "format=json&hashmap";
- var uri = builder.Uri;
-
-
- var json = await client.DownloadStringTaskAsync(uri).ConfigureAwait(false);
- var treeHash = TreeHash.Parse(json);
- Log.InfoFormat("[GET HASH] END {0}", objectName);
- return treeHash;
- }
- }
- catch (Exception exc)
- {
- Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
- throw;
- }
-
- }
-
-
- ///
- ///
- ///
- ///
- ///
- ///
- ///
- /// Optional hash value for the file. If no hash is provided, the method calculates a new hash
- /// >This method should have no timeout or a very long one
- public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream")
- {
- if (String.IsNullOrWhiteSpace(container))
- throw new ArgumentNullException("container", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The objectName property can't be empty");
- if (String.IsNullOrWhiteSpace(fileName))
- throw new ArgumentNullException("fileName", "The fileName property can't be empty");
-/*
- if (!File.Exists(fileName) && !Directory.Exists(fileName))
- throw new FileNotFoundException("The file or directory does not exist",fileName);
-*/
-
- try
- {
-
- using (var client = new RestClient(_baseClient) { Timeout = 0 })
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- var builder = client.GetAddressBuilder(container, objectName);
- var uri = builder.Uri;
-
- string etag = hash ?? CalculateHash(fileName);
-
- client.Headers.Add("Content-Type", contentType);
- client.Headers.Add("ETag", etag);
-
-
- Log.InfoFormat("[PUT] START {0}", objectName);
- client.UploadProgressChanged += (sender, args) =>
- {
- using (ThreadContext.Stacks["PUT"].Push("Progress"))
- {
- Log.InfoFormat("{0} {1}% {2} of {3}", fileName,
- args.ProgressPercentage,
- args.BytesSent, args.TotalBytesToSend);
- }
- };
-
- client.UploadFileCompleted += (sender, args) =>
- {
- using (ThreadContext.Stacks["PUT"].Push("Progress"))
- {
- Log.InfoFormat("Completed {0}", fileName);
- }
- };
- if (contentType=="application/directory")
- await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false);
- else
- await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false);
- }
-
- Log.InfoFormat("[PUT] END {0}", objectName);
- }
- catch (Exception exc)
- {
- Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
- throw;
- }
-
- }
-
-
- private static string CalculateHash(string fileName)
- {
- Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
- Contract.EndContractBlock();
-
- string hash;
- using (var hasher = MD5.Create())
- using(var stream=File.OpenRead(fileName))
- {
- var hashBuilder=new StringBuilder();
- foreach (byte b in hasher.ComputeHash(stream))
- hashBuilder.Append(b.ToString("x2").ToLower());
- hash = hashBuilder.ToString();
- }
- return hash;
- }
-
- public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
- {
- if (String.IsNullOrWhiteSpace(sourceContainer))
- throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(oldObjectName))
- throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
- if (String.IsNullOrWhiteSpace(targetContainer))
- throw new ArgumentNullException("targetContainer", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(newObjectName))
- throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
- Contract.EndContractBlock();
-
- var targetUrl = targetContainer + "/" + newObjectName;
- var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Headers.Add("X-Move-From", sourceUrl);
- client.PutWithRetry(targetUrl, 3);
-
- var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
- if (!expectedCodes.Contains(client.StatusCode))
- throw CreateWebException("MoveObject", client.StatusCode);
- }
- }
-
- public void DeleteObject(string account, string sourceContainer, string objectName, bool isDirectory)
- {
- if (String.IsNullOrWhiteSpace(sourceContainer))
- throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
- if (String.IsNullOrWhiteSpace(objectName))
- throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
- Contract.EndContractBlock();
-
- var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
-/*
- if (isDirectory)
- targetUrl = targetUrl + "?delimiter=/";
-*/
-
- var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
- client.Headers.Add("X-Move-From", sourceUrl);
- client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
- Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);
- client.PutWithRetry(targetUrl, 3);
-
- var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
- if (!expectedCodes.Contains(client.StatusCode))
- throw CreateWebException("DeleteObject", client.StatusCode);
- }
- }
-
-
- private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
- {
- return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
- }
-
-
-/*
- public IEnumerable ListDirectories(ContainerInfo container)
- {
- var directories=this.ListObjects(container.Account, container.Name, "/");
- }
-*/
-
- public bool CanUpload(string account, ObjectInfo cloudFile)
- {
- Contract.Requires(!String.IsNullOrWhiteSpace(account));
- Contract.Requires(cloudFile!=null);
-
- using (var client = new RestClient(_baseClient))
- {
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
-
-
- var parts = cloudFile.Name.Split('/');
- var folder = String.Join("/", parts,0,parts.Length-1);
-
- var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid());
-
- client.Parameters.Clear();
- try
- {
- client.PutWithRetry(fileUrl, 3, @"application/octet-stream");
-
- var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
- var result=(expectedCodes.Contains(client.StatusCode));
- DeleteObject(account, cloudFile.Container, fileUrl, cloudFile.IsDirectory);
- return result;
- }
- catch
- {
- return false;
- }
- }
- }
- }
-
- public class ShareAccountInfo
- {
- public DateTime? last_modified { get; set; }
- public string name { get; set; }
- }
-}
+#region
+/* -----------------------------------------------------------------------
+ *
+ *
+ * Copyright 2011-2012 GRNET S.A. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and
+ * documentation are those of the authors and should not be
+ * interpreted as representing official policies, either expressed
+ * or implied, of GRNET S.A.
+ *
+ * -----------------------------------------------------------------------
+ */
+#endregion
+
+// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
+//
+// The class provides methods to upload/download files, delete files, manage containers
+
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.ComponentModel.Composition;
+using System.Diagnostics;
+using System.Diagnostics.Contracts;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Pithos.Interfaces;
+using Pithos.Network;
+using log4net;
+
+namespace Pithos.Network
+{
+
+ [Export(typeof(ICloudClient))]
+ public class CloudFilesClient:ICloudClient,IDisposable
+ {
+ private const string TOKEN_HEADER = "X-Auth-Token";
+ private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
+ //CloudFilesClient uses *_baseClient* internally to communicate with the server
+ //RestClient provides a REST-friendly interface over the standard WebClient.
+ private RestClient _baseClient;
+
+ private HttpClient _baseHttpClient;
+ private HttpClient _baseHttpClientNoTimeout;
+
+
+ //During authentication the client provides a UserName
+ public string UserName { get; set; }
+
+ //and and ApiKey to the server
+ public string ApiKey { get; set; }
+
+ //And receives an authentication Token. This token must be provided in ALL other operations,
+ //in the X-Auth-Token header
+ private string _token;
+ private readonly string _emptyGuid = Guid.Empty.ToString();
+ private readonly Uri _emptyUri = new Uri("",UriKind.Relative);
+
+ private HttpClientHandler _httpClientHandler = new HttpClientHandler
+ {
+ AllowAutoRedirect = true,
+ AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
+ UseCookies = true,
+ };
+
+
+ public string Token
+ {
+ get { return _token; }
+ set
+ {
+ _token = value;
+ _baseClient.Headers[TOKEN_HEADER] = value;
+ }
+ }
+
+ //The client also receives a StorageUrl after authentication. All subsequent operations must
+ //use this url
+ public Uri StorageUrl { get; set; }
+
+
+ public Uri RootAddressUri { get; set; }
+
+
+ public double DownloadPercentLimit { get; set; }
+ public double UploadPercentLimit { get; set; }
+
+ public string AuthenticationUrl { get; set; }
+
+
+ public string VersionPath
+ {
+ get { return UsePithos ? "v1" : "v1.0"; }
+ }
+
+ public bool UsePithos { get; set; }
+
+
+
+ public CloudFilesClient(string userName, string apiKey)
+ {
+ UserName = userName;
+ ApiKey = apiKey;
+ }
+
+ public CloudFilesClient(AccountInfo accountInfo)
+ {
+ if (accountInfo==null)
+ throw new ArgumentNullException("accountInfo");
+ Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
+ Contract.Ensures(StorageUrl != null);
+ Contract.Ensures(_baseClient != null);
+ Contract.Ensures(RootAddressUri != null);
+ Contract.EndContractBlock();
+
+ _baseClient = new RestClient
+ {
+ BaseAddress = accountInfo.StorageUri.ToString(),
+ Timeout = 30000,
+ Retries = 3,
+ };
+ StorageUrl = accountInfo.StorageUri;
+ Token = accountInfo.Token;
+ UserName = accountInfo.UserName;
+
+ //Get the root address (StorageUrl without the account)
+ var storageUrl = StorageUrl.AbsoluteUri;
+ var usernameIndex = storageUrl.LastIndexOf(UserName);
+ var rootUrl = storageUrl.Substring(0, usernameIndex);
+ RootAddressUri = new Uri(rootUrl);
+
+ var httpClientHandler = new HttpClientHandler
+ {
+ AllowAutoRedirect = true,
+ AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
+ UseCookies = true,
+ };
+
+
+ _baseHttpClient = new HttpClient(httpClientHandler)
+ {
+ BaseAddress = StorageUrl,
+ Timeout = TimeSpan.FromSeconds(30)
+ };
+ _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);
+
+ _baseHttpClientNoTimeout = new HttpClient(httpClientHandler)
+ {
+ BaseAddress = StorageUrl,
+ Timeout = TimeSpan.FromMilliseconds(-1)
+ };
+ _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);
+
+
+ }
+
+
+ private static void AssertStatusOK(HttpResponseMessage response, string message)
+ {
+ var statusCode = response.StatusCode;
+ if (statusCode >= HttpStatusCode.BadRequest)
+ throw new WebException(String.Format("{0} with code {1} - {2}", message, statusCode, response.ReasonPhrase));
+ }
+
+ public async Task Authenticate()
+ {
+ if (String.IsNullOrWhiteSpace(UserName))
+ throw new InvalidOperationException("UserName is empty");
+ if (String.IsNullOrWhiteSpace(ApiKey))
+ throw new InvalidOperationException("ApiKey is empty");
+ if (String.IsNullOrWhiteSpace(AuthenticationUrl))
+ throw new InvalidOperationException("AuthenticationUrl is empty");
+ Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
+ Contract.Ensures(StorageUrl != null);
+ Contract.Ensures(_baseClient != null);
+ Contract.Ensures(RootAddressUri != null);
+ Contract.EndContractBlock();
+
+
+ Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
+
+ var groups = new List();
+
+ using (var authClient = new HttpClient(_httpClientHandler,false){ BaseAddress = new Uri(AuthenticationUrl),Timeout=TimeSpan.FromSeconds(30) })
+ {
+
+ authClient.DefaultRequestHeaders.Add("X-Auth-User", UserName);
+ authClient.DefaultRequestHeaders.Add("X-Auth-Key", ApiKey);
+
+ string storageUrl;
+ string token;
+
+ using (var response = await authClient.GetAsyncWithRetries(new Uri(VersionPath, UriKind.Relative),3).ConfigureAwait(false)) // .DownloadStringWithRetryRelative(new Uri(VersionPath, UriKind.Relative), 3);
+ {
+ AssertStatusOK(response,"Authentication failed");
+
+ storageUrl = response.Headers.GetFirstValue("X-Storage-Url");
+ if (String.IsNullOrWhiteSpace(storageUrl))
+ throw new InvalidOperationException("Failed to obtain storage url");
+
+ token = response.Headers.GetFirstValue(TOKEN_HEADER);
+ if (String.IsNullOrWhiteSpace(token))
+ throw new InvalidOperationException("Failed to obtain token url");
+
+ }
+
+
+ _baseClient = new RestClient
+ {
+ BaseAddress = storageUrl,
+ Timeout = 30000,
+ Retries = 3,
+ };
+
+ StorageUrl = new Uri(storageUrl);
+ Token = token;
+
+
+
+
+ //Get the root address (StorageUrl without the account)
+ var usernameIndex=storageUrl.LastIndexOf(UserName);
+ var rootUrl = storageUrl.Substring(0, usernameIndex);
+ RootAddressUri = new Uri(rootUrl);
+
+
+ _baseHttpClient = new HttpClient(_httpClientHandler,false)
+ {
+ BaseAddress = StorageUrl,
+ Timeout = TimeSpan.FromSeconds(30)
+ };
+ _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, token);
+
+ _baseHttpClientNoTimeout = new HttpClient(_httpClientHandler,false)
+ {
+ BaseAddress = StorageUrl,
+ Timeout = TimeSpan.FromMilliseconds(-1)
+ };
+ _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, token);
+
+ /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
+ groups = (from key in keys
+ where key.StartsWith("X-Account-Group-")
+ let name = key.Substring(16)
+ select new Group(name, authClient.ResponseHeaders[key]))
+ .ToList();
+
+*/
+ }
+
+ Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
+ Debug.Assert(_baseClient!=null);
+
+ return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};
+
+ }
+
+ private static void TraceStart(string method, Uri actualAddress)
+ {
+ Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
+ }
+
+ private async Task GetStringAsync(Uri targetUri, string errorMessage,DateTimeOffset? since=null)
+ {
+ TraceStart("GET",targetUri);
+ var request = new HttpRequestMessage(HttpMethod.Get, targetUri);
+ if (since.HasValue)
+ {
+ request.Headers.IfModifiedSince = since.Value;
+ }
+ using (var response = await _baseHttpClient.SendAsyncWithRetries(request,3).ConfigureAwait(false))
+ {
+ AssertStatusOK(response, errorMessage);
+
+ if (response.StatusCode == HttpStatusCode.NoContent)
+ return String.Empty;
+
+ var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
+ return content;
+ }
+ }
+
+ public async Task> ListContainers(string account)
+ {
+
+ var targetUrl = GetTargetUrl(account);
+ var targetUri = new Uri(String.Format("{0}?format=json", targetUrl));
+ var result = await GetStringAsync(targetUri, "List Containers failed").ConfigureAwait(false);
+ if (String.IsNullOrWhiteSpace(result))
+ return new List();
+ var infos = JsonConvert.DeserializeObject>(result);
+ foreach (var info in infos)
+ {
+ info.Account = account;
+ }
+ return infos;
+ }
+
+
+ private string GetAccountUrl(string account)
+ {
+ return RootAddressUri.Combine(account).AbsoluteUri;
+ }
+
+ public IList ListSharingAccounts(DateTime? since=null)
+ {
+ using (ThreadContext.Stacks["Share"].Push("List Accounts"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+ var targetUri = new Uri(String.Format("{0}?format=json", RootAddressUri), UriKind.Absolute);
+ var content=TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListSharingAccounts failed", since).ConfigureAwait(false)).Result;
+
+ //If the result is empty, return an empty list,
+ var infos = String.IsNullOrWhiteSpace(content)
+ ? new List()
+ //Otherwise deserialize the account list into a list of ShareAccountInfos
+ : JsonConvert.DeserializeObject>(content);
+
+ Log.DebugFormat("END");
+ return infos;
+ }
+ }
+
+
+ ///
+ /// Request listing of all objects in a container modified since a specific time.
+ /// If the *since* value is missing, return all objects
+ ///
+ /// Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new
+ /// and should be polled anyway
+ ///
+ ///
+ ///
+ public IList ListSharedObjects(HashSet knownContainers, DateTimeOffset? since)
+ {
+
+ using (ThreadContext.Stacks["Share"].Push("List Objects"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+ //'since' is not used here because we need to have ListObjects return a NoChange result
+ //for all shared accounts,containers
+
+ Func getKey = c => String.Format("{0}\\{1}", c.Account, c.Name);
+
+ var containers = (from account in ListSharingAccounts()
+ let conts = ListContainers(account.name).Result
+ from container in conts
+ select container).ToList();
+ var items = from container in containers
+ let actualSince=knownContainers.Contains(getKey(container))?since:null
+ select ListObjects(container.Account , container.Name, actualSince);
+ var objects=items.SelectMany(r=> r).ToList();
+
+ //For each object
+ //Check parents recursively up to (but not including) the container.
+ //If parents are missing, add them to the list
+ //Need function to calculate all parent URLs
+ objects = AddMissingParents(objects);
+
+ //Store any new containers
+ foreach (var container in containers)
+ {
+ knownContainers.Add(getKey(container));
+ }
+
+
+
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ return objects;
+ }
+ }
+
+ private List AddMissingParents(List objects)
+ {
+ //TODO: Remove short-circuit when we decide to use Missing Parents functionality
+ //return objects;
+
+ var existingUris = objects.ToDictionary(o => o.Uri, o => o);
+ foreach (var objectInfo in objects)
+ {
+ //Can be null when retrieving objects to show in selective sync
+ if (objectInfo.Name == null)
+ continue;
+
+ //No need to unescape here, the parts will be used to create new ObjectInfos
+ var parts = objectInfo.Name.ToString().Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries);
+ //If there is no parent, skip
+ if (parts.Length == 1)
+ continue;
+ var baseParts = new[]
+ {
+ objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container.ToString()
+ };
+ for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++)
+ {
+ var nameparts = parts.Range(0, partIdx).ToArray();
+ var parentName= String.Join("/", nameparts);
+
+ var parentParts = baseParts.Concat(nameparts);
+ var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts);
+
+ var parentUri = new Uri(parentUrl, UriKind.Absolute);
+
+ ObjectInfo existingInfo;
+ if (!existingUris.TryGetValue(parentUri,out existingInfo))
+ {
+ var h = parentUrl.GetHashCode();
+ var reverse = new string(parentUrl.Reverse().ToArray());
+ var rh = reverse.GetHashCode();
+ var b1 = BitConverter.GetBytes(h);
+ var b2 = BitConverter.GetBytes(rh);
+ var g = new Guid(0,0,0,b1.Concat(b2).ToArray());
+
+
+ existingUris[parentUri] = new ObjectInfo
+ {
+ Account = objectInfo.Account,
+ Container = objectInfo.Container,
+ Content_Type = ObjectInfo.CONTENT_TYPE_DIRECTORY,
+ ETag = Signature.MERKLE_EMPTY,
+ X_Object_Hash = Signature.MERKLE_EMPTY,
+ Name=new Uri(parentName,UriKind.Relative),
+ StorageUri=objectInfo.StorageUri,
+ Bytes = 0,
+ UUID=g.ToString(),
+ };
+ }
+ }
+ }
+ return existingUris.Values.ToList();
+ }
+
+ public void SetTags(ObjectInfo target,IDictionary tags)
+ {
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("The Token is not set");
+ if (StorageUrl == null)
+ throw new InvalidOperationException("The StorageUrl is not set");
+ if (target == null)
+ throw new ArgumentNullException("target");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Share"].Push("Share Object"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+ using (var client = new RestClient(_baseClient))
+ {
+
+ client.BaseAddress = GetAccountUrl(target.Account);
+
+ client.Parameters.Clear();
+ client.Parameters.Add("update", "");
+
+ foreach (var tag in tags)
+ {
+ var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
+ client.Headers.Add(headerTag, tag.Value);
+ }
+
+ client.DownloadStringWithRetryRelative(target.Container, 3);
+
+
+ client.AssertStatusOK("SetTags failed");
+ //If the status is NOT ACCEPTED we have a problem
+ if (client.StatusCode != HttpStatusCode.Accepted)
+ {
+ Log.Error("Failed to set tags");
+ throw new Exception("Failed to set tags");
+ }
+
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ }
+ }
+
+
+ }
+
+ public void ShareObject(string account, Uri container, Uri objectName, string shareTo, bool read, bool write)
+ {
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("The Token is not set");
+ if (StorageUrl==null)
+ throw new InvalidOperationException("The StorageUrl is not set");
+ if (container==null)
+ throw new ArgumentNullException("container");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("container");
+ if (objectName==null)
+ throw new ArgumentNullException("objectName");
+ if (objectName.IsAbsoluteUri)
+ throw new ArgumentException("objectName");
+ if (String.IsNullOrWhiteSpace(account))
+ throw new ArgumentNullException("account");
+ if (String.IsNullOrWhiteSpace(shareTo))
+ throw new ArgumentNullException("shareTo");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Share"].Push("Share Object"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+ using (var client = new RestClient(_baseClient))
+ {
+
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Parameters.Clear();
+ client.Parameters.Add("format", "json");
+
+ string permission = "";
+ if (write)
+ permission = String.Format("write={0}", shareTo);
+ else if (read)
+ permission = String.Format("read={0}", shareTo);
+ client.Headers.Add("X-Object-Sharing", permission);
+
+ var content = client.DownloadStringWithRetryRelative(container, 3);
+
+ client.AssertStatusOK("ShareObject failed");
+
+ //If the result is empty, return an empty list,
+ var infos = String.IsNullOrWhiteSpace(content)
+ ? new List()
+ //Otherwise deserialize the object list into a list of ObjectInfos
+ : JsonConvert.DeserializeObject>(content);
+
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ }
+ }
+
+
+ }
+
+ public async Task GetAccountPolicies(AccountInfo accountInfo)
+ {
+ if (accountInfo==null)
+ throw new ArgumentNullException("accountInfo");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Account"].Push("GetPolicies"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+/*
+ if (_baseClient == null)
+ {
+ _baseClient = new RestClient
+ {
+ BaseAddress = accountInfo.StorageUri.ToString(),
+ Timeout = 30000,
+ Retries = 3,
+ };
+ }
+
+*/
+ var containerUri = GetTargetUri(accountInfo.UserName);
+ var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);
+ using(var response=await _baseHttpClient.HeadAsyncWithRetries(targetUri,3).ConfigureAwait(false))
+ {
+
+ var quotaValue=response.Headers.GetFirstValue("X-Account-Policy-Quota");
+ var bytesValue = response.Headers.GetFirstValue("X-Account-Bytes-Used");
+ long quota, bytes;
+ if (long.TryParse(quotaValue, out quota))
+ accountInfo.Quota = quota;
+ if (long.TryParse(bytesValue, out bytes))
+ accountInfo.BytesUsed = bytes;
+
+ return accountInfo;
+ }
+
+
+ //using (var client = new RestClient(_baseClient))
+ //{
+ // if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
+ // client.BaseAddress = GetAccountUrl(accountInfo.UserName);
+
+ // client.Parameters.Clear();
+ // client.Parameters.Add("format", "json");
+ // client.Head(_emptyUri, 3);
+
+ // var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
+ // var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
+
+ // long quota, bytes;
+ // if (long.TryParse(quotaValue, out quota))
+ // accountInfo.Quota = quota;
+ // if (long.TryParse(bytesValue, out bytes))
+ // accountInfo.BytesUsed = bytes;
+
+ // return accountInfo;
+
+ //}
+
+ }
+ }
+
+ public void UpdateMetadata(ObjectInfo objectInfo)
+ {
+ if (objectInfo == null)
+ throw new ArgumentNullException("objectInfo");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+
+ using(var client=new RestClient(_baseClient))
+ {
+
+ client.BaseAddress = GetAccountUrl(objectInfo.Account);
+
+ client.Parameters.Clear();
+
+
+ //Set Tags
+ foreach (var tag in objectInfo.Tags)
+ {
+ var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
+ client.Headers.Add(headerTag, tag.Value);
+ }
+
+ //Set Permissions
+
+ var permissions=objectInfo.GetPermissionString();
+ client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);
+
+ client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
+ client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
+ client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
+ var isPublic = objectInfo.IsPublic.ToString().ToLower();
+ client.Headers.Add("X-Object-Public", isPublic);
+
+
+ var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);
+ client.PostWithRetry(new Uri(address,UriKind.Relative),"application/xml");
+
+ client.AssertStatusOK("UpdateMetadata failed");
+ //If the status is NOT ACCEPTED or OK we have a problem
+ if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
+ {
+ Log.Error("Failed to update metadata");
+ throw new Exception("Failed to update metadata");
+ }
+
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ }
+ }
+
+ }
+
+ public void UpdateMetadata(ContainerInfo containerInfo)
+ {
+ if (containerInfo == null)
+ throw new ArgumentNullException("containerInfo");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+
+ using(var client=new RestClient(_baseClient))
+ {
+
+ client.BaseAddress = GetAccountUrl(containerInfo.Account);
+
+ client.Parameters.Clear();
+
+
+ //Set Tags
+ foreach (var tag in containerInfo.Tags)
+ {
+ var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
+ client.Headers.Add(headerTag, tag.Value);
+ }
+
+
+ //Set Policies
+ foreach (var policy in containerInfo.Policies)
+ {
+ var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
+ client.Headers.Add(headerPolicy, policy.Value);
+ }
+
+
+ var uriBuilder = client.GetAddressBuilder(containerInfo.Name,_emptyUri);
+ var uri = uriBuilder.Uri;
+
+ client.UploadValues(uri,new NameValueCollection());
+
+
+ client.AssertStatusOK("UpdateMetadata failed");
+ //If the status is NOT ACCEPTED or OK we have a problem
+ if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
+ {
+ Log.Error("Failed to update metadata");
+ throw new Exception("Failed to update metadata");
+ }
+
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ }
+ }
+
+ }
+
+
+
+
+ public IList ListObjects(string account, Uri container, DateTimeOffset? since = null)
+ {
+ if (container==null)
+ throw new ArgumentNullException("container");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("container");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Objects"].Push("List"))
+ {
+
+ var containerUri = GetTargetUri(account).Combine(container);
+ var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);
+
+ var content =TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result;
+
+ //304 will result in an empty string. Empty containers return an empty json array
+ if (String.IsNullOrWhiteSpace(content))
+ return new[] {new NoModificationInfo(account, container)};
+
+ //If the result is empty, return an empty list,
+ var infos = String.IsNullOrWhiteSpace(content)
+ ? new List()
+ //Otherwise deserialize the object list into a list of ObjectInfos
+ : JsonConvert.DeserializeObject>(content);
+
+ foreach (var info in infos)
+ {
+ info.Container = container;
+ info.Account = account;
+ info.StorageUri = StorageUrl;
+ }
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ return infos;
+ }
+ }
+
+ public IList ListObjects(string account, Uri container, Uri folder, DateTimeOffset? since = null)
+ {
+ if (container==null)
+ throw new ArgumentNullException("container");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("container");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Objects"].Push("List"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+ var containerUri = GetTargetUri(account).Combine(container);
+ var targetUri = new Uri(String.Format("{0}?format=json&path={1}", containerUri,folder), UriKind.Absolute);
+ var content = TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result;
+
+ //304 will result in an empty string. Empty containers return an empty json array
+ if (String.IsNullOrWhiteSpace(content))
+ return new[] { new NoModificationInfo(account, container) };
+
+
+ var infos = JsonConvert.DeserializeObject>(content);
+ foreach (var info in infos)
+ {
+ info.Account = account;
+ if (info.Container == null)
+ info.Container = container;
+ info.StorageUri = StorageUrl;
+ }
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ return infos;
+/*
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Parameters.Clear();
+ client.Parameters.Add("format", "json");
+ client.Parameters.Add("path", folder.ToString());
+ client.IfModifiedSince = since;
+ var content = client.DownloadStringWithRetryRelative(container, 3);
+ client.AssertStatusOK("ListObjects failed");
+
+ if (client.StatusCode==HttpStatusCode.NotModified)
+ return new[]{new NoModificationInfo(account,container,folder)};
+
+ var infos = JsonConvert.DeserializeObject>(content);
+ foreach (var info in infos)
+ {
+ info.Account = account;
+ if (info.Container == null)
+ info.Container = container;
+ info.StorageUri = StorageUrl;
+ }
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+ return infos;
+ }
+*/
+ }
+ }
+
+
+ public bool ContainerExists(string account, Uri container)
+ {
+ if (container==null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException( "The container must be relative","container");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Containters"].Push("Exists"))
+ {
+ if (Log.IsDebugEnabled) Log.DebugFormat("START");
+
+ var targetUri = GetTargetUri(account).Combine(container);
+
+ using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result)
+ {
+
+ bool result;
+ switch (response.StatusCode)
+ {
+ case HttpStatusCode.OK:
+ case HttpStatusCode.NoContent:
+ result = true;
+ break;
+ case HttpStatusCode.NotFound:
+ result = false;
+ break;
+ default:
+ throw CreateWebException("ContainerExists", response.StatusCode);
+ }
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+
+ return result;
+ }
+/*
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Parameters.Clear();
+ client.Head(container, 3);
+
+ bool result;
+ switch (client.StatusCode)
+ {
+ case HttpStatusCode.OK:
+ case HttpStatusCode.NoContent:
+ result=true;
+ break;
+ case HttpStatusCode.NotFound:
+ result=false;
+ break;
+ default:
+ throw CreateWebException("ContainerExists", client.StatusCode);
+ }
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
+
+ return result;
+ }
+*/
+
+ }
+ }
+
+ private Uri GetTargetUri(string account)
+ {
+ return new Uri(GetTargetUrl(account),UriKind.Absolute);
+ }
+
+ private string GetTargetUrl(string account)
+ {
+ return String.IsNullOrWhiteSpace(account)
+ ? _baseHttpClient.BaseAddress.ToString()
+ : GetAccountUrl(account);
+ }
+
+ public bool ObjectExists(string account, Uri container, Uri objectName)
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative","container");
+ if (objectName == null)
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");
+ if (objectName.IsAbsoluteUri)
+ throw new ArgumentException("The objectName must be relative","objectName");
+ Contract.EndContractBlock();
+
+ var targetUri=GetTargetUri(account).Combine(container).Combine(objectName);
+
+ using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result)
+ {
+ switch (response.StatusCode)
+ {
+ case HttpStatusCode.OK:
+ case HttpStatusCode.NoContent:
+ return true;
+ case HttpStatusCode.NotFound:
+ return false;
+ default:
+ throw CreateWebException("ObjectExists", response.StatusCode);
+ }
+ }
+
+/*
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Parameters.Clear();
+ client.Head(container.Combine(objectName), 3);
+
+ switch (client.StatusCode)
+ {
+ case HttpStatusCode.OK:
+ case HttpStatusCode.NoContent:
+ return true;
+ case HttpStatusCode.NotFound:
+ return false;
+ default:
+ throw CreateWebException("ObjectExists", client.StatusCode);
+ }
+ }
+*/
+
+ }
+
+ public async Task GetObjectInfo(string account, Uri container, Uri objectName)
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative", "container");
+ if (objectName == null)
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");
+ if (objectName.IsAbsoluteUri)
+ throw new ArgumentException("The objectName must be relative", "objectName");
+ Contract.EndContractBlock();
+
+ using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
+ {
+
+ var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);
+ try
+ {
+ using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true))
+ {
+ switch (response.StatusCode)
+ {
+ case HttpStatusCode.OK:
+ case HttpStatusCode.NoContent:
+ var tags = response.Headers.GetMeta("X-Object-Meta-");
+ var extensions = (from header in response.Headers
+ where
+ header.Key.StartsWith("X-Object-") &&
+ !header.Key.StartsWith("X-Object-Meta-")
+ select new {Name = header.Key, Value = header.Value.FirstOrDefault()})
+ .ToDictionary(t => t.Name, t => t.Value);
+
+ var permissions = response.Headers.GetFirstValue("X-Object-Sharing");
+
+
+ var info = new ObjectInfo
+ {
+ Account = account,
+ Container = container,
+ Name = objectName,
+ ETag = response.Headers.ETag.NullSafe(e=>e.Tag),
+ UUID = response.Headers.GetFirstValue("X-Object-UUID"),
+ X_Object_Hash = response.Headers.GetFirstValue("X-Object-Hash"),
+ Content_Type = response.Headers.GetFirstValue("Content-Type"),
+ Bytes = Convert.ToInt64(response.Content.Headers.ContentLength),
+ Tags = tags,
+ Last_Modified = response.Content.Headers.LastModified,
+ Extensions = extensions,
+ ContentEncoding =
+ response.Content.Headers.ContentEncoding.FirstOrDefault(),
+ ContendDisposition =
+ response.Content.Headers.ContentDisposition.NullSafe(c=>c.ToString()),
+ Manifest = response.Headers.GetFirstValue("X-Object-Manifest"),
+ PublicUrl = response.Headers.GetFirstValue("X-Object-Public"),
+ StorageUri = StorageUrl,
+ };
+ info.SetPermissions(permissions);
+ return info;
+ case HttpStatusCode.NotFound:
+ return ObjectInfo.Empty;
+ default:
+ throw new WebException(
+ String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
+ objectName, response.StatusCode));
+ }
+ }
+ }
+ catch (RetryException)
+ {
+ Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.", objectName);
+ return ObjectInfo.Empty;
+ }
+ catch (WebException e)
+ {
+ Log.Error(
+ String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status {1}",
+ objectName, e.Status), e);
+ throw;
+ }
+ } /* using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+ try
+ {
+ client.Parameters.Clear();
+
+ client.Head(container.Combine(objectName), 3);
+
+ if (client.TimedOut)
+ return ObjectInfo.Empty;
+
+ switch (client.StatusCode)
+ {
+ case HttpStatusCode.OK:
+ case HttpStatusCode.NoContent:
+ var keys = client.ResponseHeaders.AllKeys.AsQueryable();
+ var tags = client.GetMeta("X-Object-Meta-");
+ var extensions = (from key in keys
+ where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
+ select new {Name = key, Value = client.ResponseHeaders[key]})
+ .ToDictionary(t => t.Name, t => t.Value);
+
+ var permissions=client.GetHeaderValue("X-Object-Sharing", true);
+
+
+ var info = new ObjectInfo
+ {
+ Account = account,
+ Container = container,
+ Name = objectName,
+ ETag = client.GetHeaderValue("ETag"),
+ UUID=client.GetHeaderValue("X-Object-UUID"),
+ X_Object_Hash = client.GetHeaderValue("X-Object-Hash"),
+ Content_Type = client.GetHeaderValue("Content-Type"),
+ Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),
+ Tags = tags,
+ Last_Modified = client.LastModified,
+ Extensions = extensions,
+ ContentEncoding=client.GetHeaderValue("Content-Encoding",true),
+ ContendDisposition = client.GetHeaderValue("Content-Disposition",true),
+ Manifest=client.GetHeaderValue("X-Object-Manifest",true),
+ PublicUrl=client.GetHeaderValue("X-Object-Public",true),
+ StorageUri=StorageUrl,
+ };
+ info.SetPermissions(permissions);
+ return info;
+ case HttpStatusCode.NotFound:
+ return ObjectInfo.Empty;
+ default:
+ throw new WebException(
+ String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
+ objectName, client.StatusCode));
+ }
+
+ }
+ catch (RetryException)
+ {
+ Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);
+ return ObjectInfo.Empty;
+ }
+ catch (WebException e)
+ {
+ Log.Error(
+ String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
+ objectName, client.StatusCode), e);
+ throw;
+ }
+ } */
+
+
+ }
+
+
+
+ public void CreateFolder(string account, Uri container, Uri folder)
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative","container");
+ if (folder == null)
+ throw new ArgumentNullException("folder", "The objectName property can't be empty");
+ if (folder.IsAbsoluteUri)
+ throw new ArgumentException("The objectName must be relative","folder");
+ Contract.EndContractBlock();
+
+ var folderUri=container.Combine(folder);
+ var targetUri = GetTargetUri(account).Combine(folderUri);
+ var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
+
+ message.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY);
+ message.Headers.Add("Content-Length", "0");
+ using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)
+ {
+ if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)
+ throw CreateWebException("CreateFolder", response.StatusCode);
+ }
+/*
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Parameters.Clear();
+ client.Headers.Add("Content-Type", ObjectInfo.CONTENT_TYPE_DIRECTORY);
+ client.Headers.Add("Content-Length", "0");
+ client.PutWithRetry(folderUri, 3);
+
+ if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
+ throw CreateWebException("CreateFolder", client.StatusCode);
+ }
+*/
+ }
+
+ private Dictionary GetMeta(HttpResponseMessage response,string metaPrefix)
+ {
+ if (String.IsNullOrWhiteSpace(metaPrefix))
+ throw new ArgumentNullException("metaPrefix");
+ Contract.EndContractBlock();
+
+ var dict = (from header in response.Headers
+ where header.Key.StartsWith(metaPrefix)
+ select new { Name = header.Key, Value = String.Join(",", header.Value) })
+ .ToDictionary(t => t.Name, t => t.Value);
+
+
+ return dict;
+ }
+
+
+ public ContainerInfo GetContainerInfo(string account, Uri container)
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative","container");
+ Contract.EndContractBlock();
+
+ var targetUri = GetTargetUri(account).Combine(container);
+ using (var response = _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).Result)
+ {
+ if (Log.IsDebugEnabled)
+ Log.DebugFormat("ContainerInfo data: {0}\n{1}",response,response.Content.ReadAsStringAsync().Result);
+ switch (response.StatusCode)
+ {
+ case HttpStatusCode.OK:
+ case HttpStatusCode.NoContent:
+ var tags = GetMeta(response,"X-Container-Meta-");
+ var policies = GetMeta(response,"X-Container-Policy-");
+
+ var containerInfo = new ContainerInfo
+ {
+ Account = account,
+ Name = container,
+ StorageUrl = StorageUrl.ToString(),
+ Count =long.Parse(response.Headers.GetFirstValue("X-Container-Object-Count")),
+ Bytes = long.Parse(response.Headers.GetFirstValue("X-Container-Bytes-Used")),
+ BlockHash = response.Headers.GetFirstValue("X-Container-Block-Hash"),
+ BlockSize =
+ int.Parse(response.Headers.GetFirstValue("X-Container-Block-Size")),
+ Last_Modified = response.Content.Headers.LastModified,
+ Tags = tags,
+ Policies = policies
+ };
+
+
+ return containerInfo;
+ case HttpStatusCode.NotFound:
+ return ContainerInfo.Empty;
+ default:
+ throw CreateWebException("GetContainerInfo", response.StatusCode);
+ }
+ }
+ }
+
+ public void CreateContainer(string account, Uri container)
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative","container");
+ Contract.EndContractBlock();
+
+ var targetUri=GetTargetUri(account).Combine(container);
+ var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
+ message.Headers.Add("Content-Length", "0");
+ using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)
+ {
+ var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
+ if (!expectedCodes.Contains(response.StatusCode))
+ throw CreateWebException("CreateContainer", response.StatusCode);
+ }
+/*
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.PutWithRetry(container, 3);
+ var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
+ if (!expectedCodes.Contains(client.StatusCode))
+ throw CreateWebException("CreateContainer", client.StatusCode);
+ }
+*/
+ }
+
+ public async Task WipeContainer(string account, Uri container)
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative", "container");
+ Contract.EndContractBlock();
+
+ await DeleteContainer(account, new Uri(String.Format("{0}?delimiter=/", container), UriKind.Relative)).ConfigureAwait(false);
+ }
+
+
+ public async Task DeleteContainer(string account, Uri container)
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative","container");
+ Contract.EndContractBlock();
+
+ var targetUri = GetTargetUri(account).Combine(container);
+ var message = new HttpRequestMessage(HttpMethod.Delete, targetUri);
+ using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
+ {
+ var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent };
+ if (!expectedCodes.Contains(response.StatusCode))
+ throw CreateWebException("DeleteContainer", response.StatusCode);
+ }
+
+ }
+
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ /// This method should have no timeout or a very long one
+ //Asynchronously download the object specified by *objectName* in a specific *container* to
+ // a local file
+ public async Task GetObject(string account, Uri container, Uri objectName, string fileName,CancellationToken cancellationToken)
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative","container");
+ if (objectName == null)
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");
+ if (objectName.IsAbsoluteUri)
+ throw new ArgumentException("The objectName must be relative","objectName");
+ Contract.EndContractBlock();
+
+
+ try
+ {
+ //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
+ //object to avoid concurrency errors.
+ //
+ //Download operations take a long time therefore they have no timeout.
+ using(var client = new RestClient(_baseClient) { Timeout = 0 })
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ //The container and objectName are relative names. They are joined with the client's
+ //BaseAddress to create the object's absolute address
+ var builder = client.GetAddressBuilder(container, objectName);
+ var uri = builder.Uri;
+
+ //Download progress is reported to the Trace log
+ Log.InfoFormat("[GET] START {0}", objectName);
+ /*client.DownloadProgressChanged += (sender, args) =>
+ Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
+ fileName, args.ProgressPercentage,
+ args.BytesReceived,
+ args.TotalBytesToReceive);*/
+ var progress = new Progress(args =>
+ {
+ Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
+ fileName, args.ProgressPercentage,
+ args.BytesReceived,
+ args.TotalBytesToReceive);
+ if (DownloadProgressChanged!=null)
+ DownloadProgressChanged(this, new DownloadArgs(args));
+ });
+
+ //Start downloading the object asynchronously
+ await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false);
+
+ //Once the download completes
+ //Delete the local client object
+ }
+ //And report failure or completion
+ }
+ catch (Exception exc)
+ {
+ Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);
+ throw;
+ }
+
+ Log.InfoFormat("[GET] END {0}", objectName);
+
+
+ }
+
+ public async Task> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash)
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative","container");
+ if (objectName == null)
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");
+ if (objectName.IsAbsoluteUri)
+ throw new ArgumentException("The objectName must be relative","objectName");
+ if (hash == null)
+ throw new ArgumentNullException("hash");
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("Invalid Token");
+ if (StorageUrl == null)
+ throw new InvalidOperationException("Invalid Storage Url");
+ Contract.EndContractBlock();
+
+
+
+ //The container and objectName are relative names. They are joined with the client's
+ //BaseAddress to create the object's absolute address
+
+ var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);
+
+
+ var uri = new Uri(String.Format("{0}?format=json&hashmap",targetUri),UriKind.Absolute);
+
+
+ //Send the tree hash as Json to the server
+ var jsonHash = hash.ToJson();
+ if (Log.IsDebugEnabled)
+ Log.DebugFormat("Hashes:\r\n{0}", jsonHash);
+
+ var message = new HttpRequestMessage(HttpMethod.Put, uri)
+ {
+ Content = new StringContent(jsonHash)
+ };
+ message.Headers.Add("ETag",hash.TopHash.ToHashString());
+
+ //Don't use a timeout because putting the hashmap may be a long process
+
+ using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
+ {
+ var empty = (IList)new List();
+
+ switch (response.StatusCode)
+ {
+ case HttpStatusCode.Created:
+ //The server will respond either with 201-created if all blocks were already on the server
+ return empty;
+ case HttpStatusCode.Conflict:
+ //or with a 409-conflict and return the list of missing parts
+ using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
+ using(var reader=stream.GetLoggedReader(Log))
+ {
+ var serializer = new JsonSerializer();
+ serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);
+ var hashes = (List)serializer.Deserialize(reader, typeof(List));
+ return hashes;
+ }
+ default:
+ //All other cases are unexpected
+ //Ensure that failure codes raise exceptions
+ response.EnsureSuccessStatusCode();
+ //And log any other codes as warngings, but continute processing
+ Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",response.StatusCode,response.ReasonPhrase);
+ return empty;
+ }
+ }
+
+ }
+
+
+ public async Task GetBlock(string account, Uri container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)
+ {
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("Invalid Token");
+ if (StorageUrl == null)
+ throw new InvalidOperationException("Invalid Storage Url");
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative","container");
+ if (relativeUrl == null)
+ throw new ArgumentNullException("relativeUrl");
+ if (end.HasValue && end < 0)
+ throw new ArgumentOutOfRangeException("end");
+ if (start < 0)
+ throw new ArgumentOutOfRangeException("start");
+ Contract.EndContractBlock();
+
+
+ var targetUri = GetTargetUri(account).Combine(container).Combine(relativeUrl);
+ var message = new HttpRequestMessage(HttpMethod.Get, targetUri);
+ message.Headers.Range=new RangeHeaderValue(start,end);
+
+ //Don't use a timeout because putting the hashmap may be a long process
+
+ IProgress progress = new Progress(args =>
+ {
+ Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
+ targetUri.Segments.Last(), args.ProgressPercentage,
+ args.BytesReceived,
+ args.TotalBytesToReceive);
+
+ if (DownloadProgressChanged!=null)
+ DownloadProgressChanged(this, args);
+ });
+
+
+ using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false,HttpCompletionOption.ResponseHeadersRead,
+ cancellationToken).ConfigureAwait(false))
+ using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
+ using(var targetStream=new MemoryStream())
+ {
+
+ long totalSize = response.Content.Headers.ContentLength ?? 0;
+ long total = 0;
+ var buffer = new byte[65536];
+ int read;
+ while ((read = await stream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) != 0)
+ {
+ total += read;
+ progress.Report(new DownloadArgs(total, totalSize));
+ await targetStream.WriteAsync(buffer, 0, read).ConfigureAwait(false);
+ }
+
+ var result = targetStream.ToArray();
+ return result;
+ }
+
+ }
+
+ public event EventHandler UploadProgressChanged;
+ public event EventHandler DownloadProgressChanged;
+
+
+ public async Task PostBlock(string account, Uri container, byte[] block, int offset, int count,string blockHash,CancellationToken token)
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative","container");
+ if (block == null)
+ throw new ArgumentNullException("block");
+ if (offset < 0 || offset >= block.Length)
+ throw new ArgumentOutOfRangeException("offset");
+ if (count < 0 || count > block.Length)
+ throw new ArgumentOutOfRangeException("count");
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("Invalid Token");
+ if (StorageUrl == null)
+ throw new InvalidOperationException("Invalid Storage Url");
+ Contract.EndContractBlock();
+
+
+ try
+ {
+ var containerUri = GetTargetUri(account).Combine(container);
+ var targetUri = new Uri(String.Format("{0}?update", containerUri));
+
+
+ //Don't use a timeout because putting the hashmap may be a long process
+
+
+ Log.InfoFormat("[BLOCK POST] START");
+
+
+ var progress = new Progress(args =>
+ {
+ Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
+ args.ProgressPercentage,
+ args.BytesSent,
+ args.TotalBytesToSend);
+ if (UploadProgressChanged != null)
+ UploadProgressChanged(this,args);
+ });
+
+ var message = new HttpRequestMessage(HttpMethod.Post, targetUri)
+ {
+ Content = new ByteArrayContentWithProgress(block, offset, count,progress)
+ };
+ message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream");
+
+ //Send the block
+ using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3,false,HttpCompletionOption.ResponseContentRead,token).ConfigureAwait(false))
+ {
+ Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
+ response.EnsureSuccessStatusCode();
+ var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
+ var cleanHash = responseHash.TrimEnd();
+ Debug.Assert(blockHash==cleanHash);
+ if (responseHash.Equals(cleanHash,StringComparison.OrdinalIgnoreCase))
+ Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]",account,container,blockHash,responseHash);
+ }
+ Log.InfoFormat("[BLOCK POST] END");
+ }
+ catch (TaskCanceledException )
+ {
+ Log.Info("Aborting block");
+ throw;
+ }
+ catch (Exception exc)
+ {
+ Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);
+ throw;
+ }
+ }
+
+
+ public async Task GetHashMap(string account, Uri container, Uri objectName)
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative","container");
+ if (objectName == null)
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");
+ if (objectName.IsAbsoluteUri)
+ throw new ArgumentException("The objectName must be relative","objectName");
+ if (String.IsNullOrWhiteSpace(Token))
+ throw new InvalidOperationException("Invalid Token");
+ if (StorageUrl == null)
+ throw new InvalidOperationException("Invalid Storage Url");
+ Contract.EndContractBlock();
+
+ try
+ {
+
+ var objectUri = GetTargetUri(account).Combine(container).Combine(objectName);
+ var targetUri = new Uri(String.Format("{0}?format=json&hashmap", objectUri));
+
+ //Start downloading the object asynchronously
+ var json = await GetStringAsync(targetUri, "").ConfigureAwait(false);
+ var treeHash = TreeHash.Parse(json);
+ Log.InfoFormat("[GET HASH] END {0}", objectName);
+ return treeHash;
+
+ }
+ catch (Exception exc)
+ {
+ Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
+ throw;
+ }
+
+ }
+
+
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ /// Optional hash value for the file. If no hash is provided, the method calculates a new hash
+ ///
+ /// >This method should have no timeout or a very long one
+ public async Task PutObject(string account, Uri container, Uri objectName, string fileName, string hash = Signature.MERKLE_EMPTY, string contentType = "application/octet-stream")
+ {
+ if (container == null)
+ throw new ArgumentNullException("container", "The container property can't be empty");
+ if (container.IsAbsoluteUri)
+ throw new ArgumentException("The container must be relative","container");
+ if (objectName == null)
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");
+ if (objectName.IsAbsoluteUri)
+ throw new ArgumentException("The objectName must be relative","objectName");
+ if (String.IsNullOrWhiteSpace(fileName))
+ throw new ArgumentNullException("fileName", "The fileName property can't be empty");
+ try
+ {
+
+ using (var client = new RestClient(_baseClient) { Timeout = 0 })
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ var builder = client.GetAddressBuilder(container, objectName);
+ var uri = builder.Uri;
+
+ string etag = hash ;
+
+ client.Headers.Add("Content-Type", contentType);
+ if (contentType!=ObjectInfo.CONTENT_TYPE_DIRECTORY)
+ client.Headers.Add("ETag", etag);
+
+
+ Log.InfoFormat("[PUT] START {0}", objectName);
+ client.UploadProgressChanged += (sender, args) =>
+ {
+ using (ThreadContext.Stacks["PUT"].Push("Progress"))
+ {
+ Log.InfoFormat("{0} {1}% {2} of {3}", fileName,
+ args.ProgressPercentage,
+ args.BytesSent, args.TotalBytesToSend);
+ }
+ };
+
+ client.UploadFileCompleted += (sender, args) =>
+ {
+ using (ThreadContext.Stacks["PUT"].Push("Progress"))
+ {
+ Log.InfoFormat("Completed {0}", fileName);
+ }
+ };
+
+ if (contentType==ObjectInfo.CONTENT_TYPE_DIRECTORY)
+ await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false);
+ else
+ await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false);
+ }
+
+ Log.InfoFormat("[PUT] END {0}", objectName);
+ }
+ catch (Exception exc)
+ {
+ Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
+ throw;
+ }
+
+ }
+
+ public void MoveObject(string account, Uri sourceContainer, Uri oldObjectName, Uri targetContainer, Uri newObjectName)
+ {
+ if (sourceContainer == null)
+ throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");
+ if (sourceContainer.IsAbsoluteUri)
+ throw new ArgumentException("The sourceContainer must be relative","sourceContainer");
+ if (oldObjectName == null)
+ throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
+ if (oldObjectName.IsAbsoluteUri)
+ throw new ArgumentException("The oldObjectName must be relative","oldObjectName");
+ if (targetContainer == null)
+ throw new ArgumentNullException("targetContainer", "The targetContainer property can't be empty");
+ if (targetContainer.IsAbsoluteUri)
+ throw new ArgumentException("The targetContainer must be relative","targetContainer");
+ if (newObjectName == null)
+ throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
+ if (newObjectName.IsAbsoluteUri)
+ throw new ArgumentException("The newObjectName must be relative","newObjectName");
+ Contract.EndContractBlock();
+
+ var baseUri = GetTargetUri(account);
+ var targetUri = baseUri.Combine(targetContainer).Combine(newObjectName);
+ var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, oldObjectName),UriKind.Relative);
+
+ var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
+ message.Headers.Add("X-Move-From", sourceUri.ToString());
+ using (var response = _baseHttpClient.SendAsyncWithRetries(message, 3).Result)
+ {
+ var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
+ if (!expectedCodes.Contains(response.StatusCode))
+ throw CreateWebException("MoveObject", response.StatusCode);
+ }
+ }
+
+ public async Task DeleteObject(string account, Uri sourceContainer, Uri objectName, bool isDirectory)
+ {
+ if (sourceContainer == null)
+ throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");
+ if (sourceContainer.IsAbsoluteUri)
+ throw new ArgumentException("The sourceContainer must be relative","sourceContainer");
+ if (objectName == null)
+ throw new ArgumentNullException("objectName", "The objectName property can't be empty");
+ if (objectName.IsAbsoluteUri)
+ throw new ArgumentException("The objectName must be relative","objectName");
+ Contract.EndContractBlock();
+
+
+
+ var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, objectName),UriKind.Relative);
+
+
+ if (objectName.OriginalString.EndsWith(".ignore"))
+ using(var response = await _baseHttpClient.DeleteAsync(sourceUri)){}
+ else
+ {
+ var relativeUri = new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName),
+ UriKind.Relative);
+
+/*
+ var relativeUri = isDirectory
+ ? new Uri(
+ String.Format("{0}/{1}?delimiter=/", FolderConstants.TrashContainer,
+ objectName), UriKind.Relative)
+ : new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName),
+ UriKind.Relative);
+
+*/
+ var targetUri = GetTargetUri(account).Combine(relativeUri);
+
+
+ var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
+ message.Headers.Add("X-Move-From", sourceUri.ToString());
+
+ Log.InfoFormat("[TRASH] [{0}] to [{1}]", sourceUri, targetUri);
+ using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3))
+ {
+ var expectedCodes = new[]
+ {
+ HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,
+ HttpStatusCode.NotFound
+ };
+ if (!expectedCodes.Contains(response.StatusCode))
+ throw CreateWebException("DeleteObject", response.StatusCode);
+ }
+ }
+/*
+
+
+ var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
+/*
+ if (isDirectory)
+ targetUrl = targetUrl + "?delimiter=/";
+#1#
+
+ var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
+
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ client.Headers.Add("X-Move-From", sourceUrl);
+ client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
+ Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);
+ client.PutWithRetry(new Uri(targetUrl,UriKind.Relative), 3);
+
+ var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
+ if (!expectedCodes.Contains(client.StatusCode))
+ throw CreateWebException("DeleteObject", client.StatusCode);
+ }
+*/
+ }
+
+
+ private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
+ {
+ return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
+ }
+
+
+ public async Task CanUpload(string account, ObjectInfo cloudFile)
+ {
+ Contract.Requires(!String.IsNullOrWhiteSpace(account));
+ Contract.Requires(cloudFile!=null);
+
+ var parts = cloudFile.Name.ToString().Split('/');
+ var folder = String.Join("/", parts,0,parts.Length-1);
+
+ var fileName = String.Format("{0}/{1}.pithos.ignore", folder, Guid.NewGuid());
+ var fileUri=fileName.ToEscapedUri();
+
+ try
+ {
+ var relativeUri = cloudFile.Container.Combine(fileUri);
+ var targetUri = GetTargetUri(account).Combine(relativeUri);
+ var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
+ message.Content.Headers.ContentType =new MediaTypeHeaderValue("application/octet-stream");
+ var response=await _baseHttpClient.SendAsyncWithRetries(message, 3);
+ var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
+ var result=(expectedCodes.Contains(response.StatusCode));
+ await DeleteObject(account, cloudFile.Container, fileUri, cloudFile.IsDirectory);
+ return result;
+ }
+ catch
+ {
+ return false;
+ }
+
+ }
+
+ ~CloudFilesClient()
+ {
+ Dispose(false);
+ }
+
+ public void Dispose()
+ {
+ Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+
+ protected virtual void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ if (_httpClientHandler!=null)
+ _httpClientHandler.Dispose();
+ if (_baseClient!=null)
+ _baseClient.Dispose();
+ if(_baseHttpClient!=null)
+ _baseHttpClient.Dispose();
+ if (_baseHttpClientNoTimeout!=null)
+ _baseHttpClientNoTimeout.Dispose();
+ }
+ _httpClientHandler = null;
+ _baseClient = null;
+ _baseHttpClient = null;
+ _baseHttpClientNoTimeout = null;
+ }
+ }
+
+ public class ShareAccountInfo
+ {
+ public DateTime? last_modified { get; set; }
+ public string name { get; set; }
+ }
+}