Synnefo » Pithos MS Client

#region

/* -----------------------------------------------------------------------

 * <copyright file="CloudFilesClient.cs" company="GRNet">

 * 

 * Copyright 2011-2012 GRNET S.A. All rights reserved.

 *

 * Redistribution and use in source and binary forms, with or

 * without modification, are permitted provided that the following

 * conditions are met:

 *

 *   1. Redistributions of source code must retain the above

 *      copyright notice, this list of conditions and the following

 *      disclaimer.

 *

 *   2. Redistributions in binary form must reproduce the above

 *      copyright notice, this list of conditions and the following

 *      disclaimer in the documentation and/or other materials

 *      provided with the distribution.

 *

 *

 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

 * POSSIBILITY OF SUCH DAMAGE.

 *

 * The views and conclusions contained in the software and

 * documentation are those of the authors and should not be

 * interpreted as representing official policies, either expressed

 * or implied, of GRNET S.A.

 * </copyright>

 * -----------------------------------------------------------------------

 */

#endregion

// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos

//

// The class provides methods to upload/download files, delete files, manage containers

using System;

using System.Collections.Generic;

using System.Collections.Specialized;

using System.ComponentModel.Composition;

using System.Diagnostics;

using System.Diagnostics.Contracts;

using System.IO;

using System.Linq;

using System.Net;

using System.Reflection;

using System.Security.Cryptography;

using System.Text;

using System.Threading;

using System.Threading.Tasks;

using Newtonsoft.Json;

using Pithos.Interfaces;

using log4net;

namespace Pithos.Network

{

    [Export(typeof(ICloudClient))]

    public class CloudFilesClient:ICloudClient

    {

        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);

        //CloudFilesClient uses *_baseClient* internally to communicate with the server

        //RestClient provides a REST-friendly interface over the standard WebClient.

        private RestClient _baseClient;

        //During authentication the client provides a UserName 

        public string UserName { get; set; }

        //and and ApiKey to the server

        public string ApiKey { get; set; }

        //And receives an authentication Token. This token must be provided in ALL other operations,

        //in the X-Auth-Token header

        private string _token;

        public string Token

        {

            get { return _token; }

            set

            {

                _token = value;

                _baseClient.Headers["X-Auth-Token"] = value;

            }

        }

        //The client also receives a StorageUrl after authentication. All subsequent operations must

        //use this url

        public Uri StorageUrl { get; set; }

        protected Uri RootAddressUri { get; set; }

       /* private WebProxy _proxy;

        public WebProxy Proxy

        {

            get { return _proxy; }

            set

            {

                _proxy = value;

                if (_baseClient != null)

                    _baseClient.Proxy = value;                

            }

        }

*/

        /* private Uri _proxy;

        public Uri Proxy

        {

            get { return _proxy; }

            set

            {

                _proxy = value;

                if (_baseClient != null)

                    _baseClient.Proxy = new WebProxy(value);                

            }

        }*/

        public double DownloadPercentLimit { get; set; }

        public double UploadPercentLimit { get; set; }

        public string AuthenticationUrl { get; set; }

        public string VersionPath

        {

            get { return UsePithos ? "v1" : "v1.0"; }

        }

        public bool UsePithos { get; set; }

        public CloudFilesClient(string userName, string apiKey)

        {

            UserName = userName;

            ApiKey = apiKey;

        }

        public CloudFilesClient(AccountInfo accountInfo)

        {

            if (accountInfo==null)

                throw new ArgumentNullException("accountInfo");

            Contract.Ensures(!String.IsNullOrWhiteSpace(Token));

            Contract.Ensures(StorageUrl != null);

            Contract.Ensures(_baseClient != null);

            Contract.Ensures(RootAddressUri != null);

            Contract.EndContractBlock();          

            _baseClient = new RestClient

            {

                BaseAddress = accountInfo.StorageUri.ToString(),

                Timeout = 10000,

                Retries = 3,

            };

            StorageUrl = accountInfo.StorageUri;

            Token = accountInfo.Token;

            UserName = accountInfo.UserName;

            //Get the root address (StorageUrl without the account)

            var storageUrl = StorageUrl.AbsoluteUri;

            var usernameIndex = storageUrl.LastIndexOf(UserName);

            var rootUrl = storageUrl.Substring(0, usernameIndex);

            RootAddressUri = new Uri(rootUrl);

        }

        public AccountInfo Authenticate()

        {

            if (String.IsNullOrWhiteSpace(UserName))

                throw new InvalidOperationException("UserName is empty");

            if (String.IsNullOrWhiteSpace(ApiKey))

                throw new InvalidOperationException("ApiKey is empty");

            if (String.IsNullOrWhiteSpace(AuthenticationUrl))

                throw new InvalidOperationException("AuthenticationUrl is empty");

            Contract.Ensures(!String.IsNullOrWhiteSpace(Token));

            Contract.Ensures(StorageUrl != null);

            Contract.Ensures(_baseClient != null);

            Contract.Ensures(RootAddressUri != null);

            Contract.EndContractBlock();

            Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);

            var groups = new List<Group>();

            using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})

            {                

               /* if (Proxy != null)

                    authClient.Proxy = Proxy;*/

                Contract.Assume(authClient.Headers!=null);

                authClient.Headers.Add("X-Auth-User", UserName);

                authClient.Headers.Add("X-Auth-Key", ApiKey);

                authClient.DownloadStringWithRetry(VersionPath, 3);

                authClient.AssertStatusOK("Authentication failed");

                var storageUrl = authClient.GetHeaderValue("X-Storage-Url");

                if (String.IsNullOrWhiteSpace(storageUrl))

                    throw new InvalidOperationException("Failed to obtain storage url");

                _baseClient = new RestClient

                {

                    BaseAddress = storageUrl,

                    Timeout = 10000,

                    Retries = 3,

                    //Proxy=Proxy

                };

                StorageUrl = new Uri(storageUrl);

                //Get the root address (StorageUrl without the account)

                var usernameIndex=storageUrl.LastIndexOf(UserName);

                var rootUrl = storageUrl.Substring(0, usernameIndex);

                RootAddressUri = new Uri(rootUrl);

                var token = authClient.GetHeaderValue("X-Auth-Token");

                if (String.IsNullOrWhiteSpace(token))

                    throw new InvalidOperationException("Failed to obtain token url");

                Token = token;

               /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();

                groups = (from key in keys

                            where key.StartsWith("X-Account-Group-")

                            let name = key.Substring(16)

                            select new Group(name, authClient.ResponseHeaders[key]))

                        .ToList();

*/

            }

            Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);

            Debug.Assert(_baseClient!=null);

            return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};            

        }

        public IList<ContainerInfo> ListContainers(string account)

        {

            using (var client = new RestClient(_baseClient))

            {

                if (!String.IsNullOrWhiteSpace(account))

                    client.BaseAddress = GetAccountUrl(account);

                client.Parameters.Clear();

                client.Parameters.Add("format", "json");

                var content = client.DownloadStringWithRetry("", 3);

                client.AssertStatusOK("List Containers failed");

                if (client.StatusCode == HttpStatusCode.NoContent)

                    return new List<ContainerInfo>();

                var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);

                foreach (var info in infos)

                {

                    info.Account = account;

                }

                return infos;

            }

        }

        private string GetAccountUrl(string account)

        {

            return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;

        }

        public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)

        {

            using (ThreadContext.Stacks["Share"].Push("List Accounts"))

            {

                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))

                {

                    client.Parameters.Clear();

                    client.Parameters.Add("format", "json");

                    client.IfModifiedSince = since;

                    //Extract the username from the base address

                    client.BaseAddress = RootAddressUri.AbsoluteUri;

                    var content = client.DownloadStringWithRetry(@"", 3);

                    client.AssertStatusOK("ListSharingAccounts failed");

                    //If the result is empty, return an empty list,

                    var infos = String.IsNullOrWhiteSpace(content)

                                    ? new List<ShareAccountInfo>()

                                //Otherwise deserialize the account list into a list of ShareAccountInfos

                                    : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);

                    Log.DebugFormat("END");

                    return infos;

                }

            }

        }

        //Request listing of all objects in a container modified since a specific time.

        //If the *since* value is missing, return all objects

        public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)

        {

            using (ThreadContext.Stacks["Share"].Push("List Objects"))

            {

                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                //'since' is not used here because we need to have ListObjects return a NoChange result

                //for all shared accounts,containers

                var accounts = ListSharingAccounts();

                var items = from account in accounts 

                            let containers = ListContainers(account.name) 

                            from container in containers 

                            select ListObjects(account.name, container.Name,since);

                var objects=items.SelectMany(r=> r).ToList();

/*

                var objects = new List<ObjectInfo>();

                foreach (var containerObjects in items)

                {

                    objects.AddRange(containerObjects);

                }

*/

                if (Log.IsDebugEnabled) Log.DebugFormat("END");

                return objects;

            }

        }

        public void SetTags(ObjectInfo target,IDictionary<string,string> tags)

        {

            if (String.IsNullOrWhiteSpace(Token))

                throw new InvalidOperationException("The Token is not set");

            if (StorageUrl == null)

                throw new InvalidOperationException("The StorageUrl is not set");

            if (target == null)

                throw new ArgumentNullException("target");

            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Share"].Push("Share Object"))

            {

                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))

                {

                    client.BaseAddress = GetAccountUrl(target.Account);

                    client.Parameters.Clear();

                    client.Parameters.Add("update", "");

                    foreach (var tag in tags)

                    {

                        var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);

                        client.Headers.Add(headerTag, tag.Value);

                    }

                    client.DownloadStringWithRetry(target.Container, 3);

                    client.AssertStatusOK("SetTags failed");

                    //If the status is NOT ACCEPTED we have a problem

                    if (client.StatusCode != HttpStatusCode.Accepted)

                    {

                        Log.Error("Failed to set tags");

                        throw new Exception("Failed to set tags");

                    }

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");

                }

            }

        }

        public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)

        {

            if (String.IsNullOrWhiteSpace(Token))

                throw new InvalidOperationException("The Token is not set");

            if (StorageUrl==null)

                throw new InvalidOperationException("The StorageUrl is not set");

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName");

            if (String.IsNullOrWhiteSpace(account))

                throw new ArgumentNullException("account");

            if (String.IsNullOrWhiteSpace(shareTo))

                throw new ArgumentNullException("shareTo");

            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Share"].Push("Share Object"))

            {

                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))

                {

                    client.BaseAddress = GetAccountUrl(account);

                    client.Parameters.Clear();

                    client.Parameters.Add("format", "json");

                    string permission = "";

                    if (write)

                        permission = String.Format("write={0}", shareTo);

                    else if (read)

                        permission = String.Format("read={0}", shareTo);

                    client.Headers.Add("X-Object-Sharing", permission);

                    var content = client.DownloadStringWithRetry(container, 3);

                    client.AssertStatusOK("ShareObject failed");

                    //If the result is empty, return an empty list,

                    var infos = String.IsNullOrWhiteSpace(content)

                                    ? new List<ObjectInfo>()

                                //Otherwise deserialize the object list into a list of ObjectInfos

                                    : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");

                }

            }

        }

        public AccountInfo GetAccountPolicies(AccountInfo accountInfo)

        {

            if (accountInfo==null)

                throw new ArgumentNullException("accountInfo");

            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Account"].Push("GetPolicies"))

            {

                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))

                {

                    if (!String.IsNullOrWhiteSpace(accountInfo.UserName))

                        client.BaseAddress = GetAccountUrl(accountInfo.UserName);

                    client.Parameters.Clear();

                    client.Parameters.Add("format", "json");                    

                    client.Head(String.Empty, 3);

                    var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];

                    var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];

                    long quota, bytes;

                    if (long.TryParse(quotaValue, out quota))

                        accountInfo.Quota = quota;

                    if (long.TryParse(bytesValue, out bytes))

                        accountInfo.BytesUsed = bytes;

                    return accountInfo;

                }

            }

        }

        public void UpdateMetadata(ObjectInfo objectInfo)

        {

            if (objectInfo == null)

                throw new ArgumentNullException("objectInfo");

            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))

            {

                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using(var client=new RestClient(_baseClient))

                {

                    client.BaseAddress = GetAccountUrl(objectInfo.Account);

                    client.Parameters.Clear();

                    //Set Tags

                    foreach (var tag in objectInfo.Tags)

                    {

                        var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);

                        client.Headers.Add(headerTag, tag.Value);

                    }

                    //Set Permissions

                    var permissions=objectInfo.GetPermissionString();

                    client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);

                    client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);

                    client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);

                    client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);

                    var isPublic = objectInfo.IsPublic.ToString().ToLower();

                    client.Headers.Add("X-Object-Public", isPublic);

                    /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);

                    uriBuilder.Query = "update=";

                    var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/

                    var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);

                    client.PostWithRetry(address,"application/xml");

                    //client.UploadValues(uri,new NameValueCollection());

                    client.AssertStatusOK("UpdateMetadata failed");

                    //If the status is NOT ACCEPTED or OK we have a problem

                    if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))

                    {

                        Log.Error("Failed to update metadata");

                        throw new Exception("Failed to update metadata");

                    }

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");

                }

            }

        }

        public void UpdateMetadata(ContainerInfo containerInfo)

        {

            if (containerInfo == null)

                throw new ArgumentNullException("containerInfo");

            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))

            {

                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using(var client=new RestClient(_baseClient))

                {

                    client.BaseAddress = GetAccountUrl(containerInfo.Account);

                    client.Parameters.Clear();

                    //Set Tags

                    foreach (var tag in containerInfo.Tags)

                    {

                        var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);

                        client.Headers.Add(headerTag, tag.Value);

                    }

                    //Set Policies

                    foreach (var policy in containerInfo.Policies)

                    {

                        var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);

                        client.Headers.Add(headerPolicy, policy.Value);

                    }

                    var uriBuilder = client.GetAddressBuilder(containerInfo.Name,"");

                    var uri = uriBuilder.Uri;

                    client.UploadValues(uri,new NameValueCollection());

                    client.AssertStatusOK("UpdateMetadata failed");

                    //If the status is NOT ACCEPTED or OK we have a problem

                    if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))

                    {

                        Log.Error("Failed to update metadata");

                        throw new Exception("Failed to update metadata");

                    }

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");

                }

            }

        }

        public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Objects"].Push("List"))

            {

                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))

                {

                    if (!String.IsNullOrWhiteSpace(account))

                        client.BaseAddress = GetAccountUrl(account);

                    client.Parameters.Clear();

                    client.Parameters.Add("format", "json");

                    client.IfModifiedSince = since;

                    var content = client.DownloadStringWithRetry(container, 3);

                    client.AssertStatusOK("ListObjects failed");

                    if (client.StatusCode==HttpStatusCode.NotModified)

                        return new[]{new NoModificationInfo(account,container)};

                    //If the result is empty, return an empty list,

                    var infos = String.IsNullOrWhiteSpace(content)

                                    ? new List<ObjectInfo>()

                                //Otherwise deserialize the object list into a list of ObjectInfos

                                    : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);

                    foreach (var info in infos)

                    {

                        info.Container = container;

                        info.Account = account;

                        info.StorageUri = this.StorageUrl;

                    }

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");

                    return infos;

                }

            }

        }

        public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

/*

            if (String.IsNullOrWhiteSpace(folder))

                throw new ArgumentNullException("folder");

*/

            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Objects"].Push("List"))

            {

                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))

                {

                    if (!String.IsNullOrWhiteSpace(account))

                        client.BaseAddress = GetAccountUrl(account);

                    client.Parameters.Clear();

                    client.Parameters.Add("format", "json");

                    client.Parameters.Add("path", folder);

                    client.IfModifiedSince = since;

                    var content = client.DownloadStringWithRetry(container, 3);

                    client.AssertStatusOK("ListObjects failed");

                    if (client.StatusCode==HttpStatusCode.NotModified)

                        return new[]{new NoModificationInfo(account,container,folder)};

                    var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);

                    foreach (var info in infos)

                    {

                        info.Account = account;

                        if (info.Container == null)

                            info.Container = container;

                        info.StorageUri = this.StorageUrl;

                    }

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");

                    return infos;

                }

            }

        }

        public bool ContainerExists(string account, string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Containters"].Push("Exists"))

            {

                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))

                {

                    if (!String.IsNullOrWhiteSpace(account))

                        client.BaseAddress = GetAccountUrl(account);

                    client.Parameters.Clear();

                    client.Head(container, 3);

                    bool result;

                    switch (client.StatusCode)

                    {

                        case HttpStatusCode.OK:

                        case HttpStatusCode.NoContent:

                            result=true;

                            break;

                        case HttpStatusCode.NotFound:

                            result=false;

                            break;

                        default:

                            throw CreateWebException("ContainerExists", client.StatusCode);

                    }

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");

                    return result;

                }

            }

        }

        public bool ObjectExists(string account, string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            Contract.EndContractBlock();

            using (var client = new RestClient(_baseClient))

            {

                if (!String.IsNullOrWhiteSpace(account))

                    client.BaseAddress = GetAccountUrl(account);

                client.Parameters.Clear();

                client.Head(container + "/" + objectName, 3);

                switch (client.StatusCode)

                {

                    case HttpStatusCode.OK:

                    case HttpStatusCode.NoContent:

                        return true;

                    case HttpStatusCode.NotFound:

                        return false;

                    default:

                        throw CreateWebException("ObjectExists", client.StatusCode);

                }

            }

        }

        public ObjectInfo GetObjectInfo(string account, string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))

            {                

                using (var client = new RestClient(_baseClient))

                {

                    if (!String.IsNullOrWhiteSpace(account))

                        client.BaseAddress = GetAccountUrl(account);

                    try

                    {

                        client.Parameters.Clear();

                        client.Head(container + "/" + objectName, 3);

                        if (client.TimedOut)

                            return ObjectInfo.Empty;

                        switch (client.StatusCode)

                        {

                            case HttpStatusCode.OK:

                            case HttpStatusCode.NoContent:

                                var keys = client.ResponseHeaders.AllKeys.AsQueryable();

                                var tags = client.GetMeta("X-Object-Meta-");

                                var extensions = (from key in keys

                                                  where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")

                                                  select new {Name = key, Value = client.ResponseHeaders[key]})

                                    .ToDictionary(t => t.Name, t => t.Value);

                                var permissions=client.GetHeaderValue("X-Object-Sharing", true);

                                var info = new ObjectInfo

                                               {

                                                   Account = account,

                                                   Container = container,

                                                   Name = objectName,

                                                   Hash = client.GetHeaderValue("ETag"),

                                                   Content_Type = client.GetHeaderValue("Content-Type"),

                                                   Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),

                                                   Tags = tags,

                                                   Last_Modified = client.LastModified,

                                                   Extensions = extensions,

                                                   ContentEncoding=client.GetHeaderValue("Content-Encoding",true),

                                                   ContendDisposition = client.GetHeaderValue("Content-Disposition",true),

                                                   Manifest=client.GetHeaderValue("X-Object-Manifest",true),

                                                   PublicUrl=client.GetHeaderValue("X-Object-Public",true),  

                                                   StorageUri=this.StorageUrl,

                                               };

                                info.SetPermissions(permissions);

                                return info;

                            case HttpStatusCode.NotFound:

                                return ObjectInfo.Empty;

                            default:

                                throw new WebException(

                                    String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",

                                                  objectName, client.StatusCode));

                        }

                    }

                    catch (RetryException)

                    {

                        Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);

                        return ObjectInfo.Empty;

                    }

                    catch (WebException e)

                    {

                        Log.Error(

                            String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",

                                          objectName, client.StatusCode), e);

                        throw;

                    }

                }                

            }

        }

        public void CreateFolder(string account, string container, string folder)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(folder))

                throw new ArgumentNullException("folder", "The folder property can't be empty");

            Contract.EndContractBlock();

            var folderUrl=String.Format("{0}/{1}",container,folder);

            using (var client = new RestClient(_baseClient))

            {

                if (!String.IsNullOrWhiteSpace(account))

                    client.BaseAddress = GetAccountUrl(account);

                client.Parameters.Clear();

                client.Headers.Add("Content-Type", @"application/directory");

                client.Headers.Add("Content-Length", "0");

                client.PutWithRetry(folderUrl, 3);

                if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)

                    throw CreateWebException("CreateFolder", client.StatusCode);

            }

        }

        public ContainerInfo GetContainerInfo(string account, string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            Contract.EndContractBlock();

            using (var client = new RestClient(_baseClient))

            {

                if (!String.IsNullOrWhiteSpace(account))

                    client.BaseAddress = GetAccountUrl(account);                

                client.Head(container);

                switch (client.StatusCode)

                {

                    case HttpStatusCode.OK:

                    case HttpStatusCode.NoContent:

                        var tags = client.GetMeta("X-Container-Meta-");

                        var policies = client.GetMeta("X-Container-Policy-");

                        var containerInfo = new ContainerInfo

                                                {

                                                    Account=account,

                                                    Name = container,

                                                    StorageUrl=this.StorageUrl.ToString(),

                                                    Count =

                                                        long.Parse(client.GetHeaderValue("X-Container-Object-Count")),

                                                    Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),

                                                    BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),

                                                    BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),

                                                    Last_Modified=client.LastModified,

                                                    Tags=tags,

                                                    Policies=policies

                                                };

                        return containerInfo;

                    case HttpStatusCode.NotFound:

                        return ContainerInfo.Empty;

                    default:

                        throw CreateWebException("GetContainerInfo", client.StatusCode);

                }

            }

        }

        public void CreateContainer(string account, string container)

        {

            if (String.IsNullOrWhiteSpace(account))

                throw new ArgumentNullException("account");

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

            Contract.EndContractBlock();

            using (var client = new RestClient(_baseClient))

            {

                if (!String.IsNullOrWhiteSpace(account))

                    client.BaseAddress = GetAccountUrl(account);

                client.PutWithRetry(container, 3);

                var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};

                if (!expectedCodes.Contains(client.StatusCode))

                    throw CreateWebException("CreateContainer", client.StatusCode);

            }

        }

        public void DeleteContainer(string account, string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            Contract.EndContractBlock();

            using (var client = new RestClient(_baseClient))

            {

                if (!String.IsNullOrWhiteSpace(account))

                    client.BaseAddress = GetAccountUrl(account);

                client.DeleteWithRetry(container, 3);

                var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};

                if (!expectedCodes.Contains(client.StatusCode))

                    throw CreateWebException("DeleteContainer", client.StatusCode);

            }

        }

        /// <summary>

        /// 

        /// </summary>

        /// <param name="account"></param>

        /// <param name="container"></param>

        /// <param name="objectName"></param>

        /// <param name="fileName"></param>

        /// <returns></returns>

        /// <remarks>This method should have no timeout or a very long one</remarks>

        //Asynchronously download the object specified by *objectName* in a specific *container* to 

        // a local file

        public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");            

            Contract.EndContractBlock();

            try

            {

                //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient

                //object to avoid concurrency errors.

                //

                //Download operations take a long time therefore they have no timeout.

                using(var client = new RestClient(_baseClient) { Timeout = 0 })

                {

                    if (!String.IsNullOrWhiteSpace(account))

                        client.BaseAddress = GetAccountUrl(account);

                    //The container and objectName are relative names. They are joined with the client's

                    //BaseAddress to create the object's absolute address

                    var builder = client.GetAddressBuilder(container, objectName);

                    var uri = builder.Uri;

                    //Download progress is reported to the Trace log

                    Log.InfoFormat("[GET] START {0}", objectName);

                    client.DownloadProgressChanged += (sender, args) =>

                                                      Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",

                                                                     fileName, args.ProgressPercentage,

                                                                     args.BytesReceived,

                                                                     args.TotalBytesToReceive);

                    //Start downloading the object asynchronously

                    await client.DownloadFileTaskAsync(uri, fileName,cancellationToken);

                    //Once the download completes

                    //Delete the local client object

                }

                //And report failure or completion

            }

            catch (Exception exc)

            {

                Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);

                throw;

            }

            Log.InfoFormat("[GET] END {0}", objectName);                                             

        }

        public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName");

            if (hash==null)

                throw new ArgumentNullException("hash");

            if (String.IsNullOrWhiteSpace(Token))

                throw new InvalidOperationException("Invalid Token");

            if (StorageUrl == null)

                throw new InvalidOperationException("Invalid Storage Url");

            Contract.EndContractBlock();

            //Don't use a timeout because putting the hashmap may be a long process

            var client = new RestClient(_baseClient) { Timeout = 0 };           

            if (!String.IsNullOrWhiteSpace(account))

                client.BaseAddress = GetAccountUrl(account);

            //The container and objectName are relative names. They are joined with the client's

            //BaseAddress to create the object's absolute address

            var builder = client.GetAddressBuilder(container, objectName);

            builder.Query = "format=json&hashmap";

            var uri = builder.Uri;

            //Send the tree hash as Json to the server            

            client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";

            var jsonHash = hash.ToJson();

            var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);

            if (Log.IsDebugEnabled)

                Log.DebugFormat("Hashes:\r\n{0}", jsonHash);

            return uploadTask.ContinueWith(t =>

            {

                var empty = (IList<string>)new List<string>();

                //The server will respond either with 201-created if all blocks were already on the server

                if (client.StatusCode == HttpStatusCode.Created)                    

                {

                    //in which case we return an empty hash list

                    return empty;

                }

                //or with a 409-conflict and return the list of missing parts

                //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception                

                if (t.IsFaulted)

                {

                    var ex = t.Exception.InnerException;

                    var we = ex as WebException;

                    var response = we.Response as HttpWebResponse;

                    if (response!=null && response.StatusCode==HttpStatusCode.Conflict)

                    {

                        //In case of 409 the missing parts will be in the response content                        

                        using (var stream = response.GetResponseStream())

                        using(var reader=stream.GetLoggedReader(Log))

                        {

                            //We used to have to cleanup the content before returning it because it contains

                            //error content after the list of hashes

                            //

                            //As of 30/1/2012, the result is a proper Json array so we don't need to read the content

                            //line by line

                            var serializer = new JsonSerializer();                            

                            serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);

                            var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));

                            return hashes;

                        }                        

                    }                    

                    //Any other status code is unexpected and the exception should be rethrown

                    Log.LogError(response);

                    throw ex;

                }

                //Any other status code is unexpected but there was no exception. We can probably continue processing

                Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);                    

                return empty;

            });

        }

        public async Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)

        {

            if (String.IsNullOrWhiteSpace(Token))

                throw new InvalidOperationException("Invalid Token");

            if (StorageUrl == null)

                throw new InvalidOperationException("Invalid Storage Url");

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

            if (relativeUrl == null)

                throw new ArgumentNullException("relativeUrl");

            if (end.HasValue && end < 0)

                throw new ArgumentOutOfRangeException("end");

            if (start < 0)

                throw new ArgumentOutOfRangeException("start");

            Contract.EndContractBlock();

            //Don't use a timeout because putting the hashmap may be a long process

            using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end})

            {

                if (!String.IsNullOrWhiteSpace(account))

                    client.BaseAddress = GetAccountUrl(account);

                var builder = client.GetAddressBuilder(container, relativeUrl.ToString());

                var uri = builder.Uri;

                var result = await client.DownloadDataTaskAsync(uri, cancellationToken);

                return result;

            }

        }

        public async Task PostBlock(string account, string container, byte[] block, int offset, int count)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

            if (block == null)

                throw new ArgumentNullException("block");

            if (offset < 0 || offset >= block.Length)

                throw new ArgumentOutOfRangeException("offset");

            if (count < 0 || count > block.Length)

                throw new ArgumentOutOfRangeException("count");

            if (String.IsNullOrWhiteSpace(Token))

                throw new InvalidOperationException("Invalid Token");

            if (StorageUrl == null)

                throw new InvalidOperationException("Invalid Storage Url");                        

            Contract.EndContractBlock();

            try

            {

            //Don't use a timeout because putting the hashmap may be a long process

                using (var client = new RestClient(_baseClient) { Timeout = 0 })

                {                    

                    if (!String.IsNullOrWhiteSpace(account))

                        client.BaseAddress = GetAccountUrl(account);

                    var builder = client.GetAddressBuilder(container, "");

                    //We are doing an update

                    builder.Query = "update";

                    var uri = builder.Uri;

                    client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";

                    Log.InfoFormat("[BLOCK POST] START");

                    client.UploadProgressChanged += (sender, args) =>

                                                    Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",

                                                                   args.ProgressPercentage, args.BytesSent,

                                                                   args.TotalBytesToSend);

                    client.UploadFileCompleted += (sender, args) =>

                                                  Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");

                    var buffer = new byte[count];

                    Buffer.BlockCopy(block, offset, buffer, 0, count);

                    //Send the block

                    await client.UploadDataTaskAsync(uri, "POST", buffer);                    

                    Log.InfoFormat("[BLOCK POST] END");

                }

            }

            catch (Exception exc)

            {

                Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);                                        

                throw;

            }

        }

        public async Task<TreeHash> GetHashMap(string account, string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName");

            if (String.IsNullOrWhiteSpace(Token))

                throw new InvalidOperationException("Invalid Token");

            if (StorageUrl == null)

                throw new InvalidOperationException("Invalid Storage Url");

            Contract.EndContractBlock();

            try

            {

                //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient

                //object to avoid concurrency errors.

                //

                //Download operations take a long time therefore they have no timeout.

                //TODO: Do they really? this is a hashmap operation, not a download

                //Start downloading the object asynchronously

                using (var client = new RestClient(_baseClient) { Timeout = 0 })

                {

                    if (!String.IsNullOrWhiteSpace(account))

                        client.BaseAddress = GetAccountUrl(account);

                    //The container and objectName are relative names. They are joined with the client's

                    //BaseAddress to create the object's absolute address

                    var builder = client.GetAddressBuilder(container, objectName);

                    builder.Query = "format=json&hashmap";

                    var uri = builder.Uri;

                    var json = await client.DownloadStringTaskAsync(uri);

                    var treeHash = TreeHash.Parse(json);

                    Log.InfoFormat("[GET HASH] END {0}", objectName);

                    return treeHash;

                }

            }

            catch (Exception exc)

            {

                Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);

                throw;

            }

        }

        /// <summary>

        /// 

        /// </summary>

        /// <param name="account"></param>

        /// <param name="container"></param>

        /// <param name="objectName"></param>

        /// <param name="fileName"></param>

        /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>

        /// <remarks>>This method should have no timeout or a very long one</remarks>

        public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream")

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            if (String.IsNullOrWhiteSpace(fileName))

                throw new ArgumentNullException("fileName", "The fileName property can't be empty");

/*

            if (!File.Exists(fileName) && !Directory.Exists(fileName))

                throw new FileNotFoundException("The file or directory does not exist",fileName);

*/

            try

            {

                using (var client = new RestClient(_baseClient) { Timeout = 0 })

                {

                    if (!String.IsNullOrWhiteSpace(account))

                        client.BaseAddress = GetAccountUrl(account);

                    var builder = client.GetAddressBuilder(container, objectName);

                    var uri = builder.Uri;

                    string etag = hash ?? CalculateHash(fileName);

                    client.Headers.Add("Content-Type", contentType);

                    client.Headers.Add("ETag", etag);

                    Log.InfoFormat("[PUT] START {0}", objectName);

                    client.UploadProgressChanged += (sender, args) =>

                                                        {

                                                            using (ThreadContext.Stacks["PUT"].Push("Progress"))

                                                            {

                                                                Log.InfoFormat("{0} {1}% {2} of {3}", fileName,

                                                                               args.ProgressPercentage,

                                                                               args.BytesSent, args.TotalBytesToSend);

                                                            }

                                                        };

                    client.UploadFileCompleted += (sender, args) =>

                                                      {

                                                          using (ThreadContext.Stacks["PUT"].Push("Progress"))

                                                          {

                                                              Log.InfoFormat("Completed {0}", fileName);

                                                          }

                                                      };                    

                    if (contentType=="application/directory")

                        await client.UploadDataTaskAsync(uri, "PUT", new byte[0]);

                    else

                        await client.UploadFileTaskAsync(uri, "PUT", fileName);

                }

                Log.InfoFormat("[PUT] END {0}", objectName);

            }

            catch (Exception exc)

            {

                Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);

                throw;

            }                

        }

        private static string CalculateHash(string fileName)

        {

            Contract.Requires(!String.IsNullOrWhiteSpace(fileName));

            Contract.EndContractBlock();

            string hash;

            using (var hasher = MD5.Create())

            using(var stream=File.OpenRead(fileName))

            {

                var hashBuilder=new StringBuilder();

                foreach (byte b in hasher.ComputeHash(stream))

                    hashBuilder.Append(b.ToString("x2").ToLower());

                hash = hashBuilder.ToString();                

            }

            return hash;

        }

        public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)

        {

            if (String.IsNullOrWhiteSpace(sourceContainer))

                throw new ArgumentNullException("sourceContainer", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(oldObjectName))

                throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");

            if (String.IsNullOrWhiteSpace(targetContainer))

                throw new ArgumentNullException("targetContainer", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(newObjectName))

                throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");

            Contract.EndContractBlock();

            var targetUrl = targetContainer + "/" + newObjectName;

            var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);

            using (var client = new RestClient(_baseClient))

            {

                if (!String.IsNullOrWhiteSpace(account))

                    client.BaseAddress = GetAccountUrl(account);

                client.Headers.Add("X-Move-From", sourceUrl);

                client.PutWithRetry(targetUrl, 3);

                var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};

                if (!expectedCodes.Contains(client.StatusCode))

                    throw CreateWebException("MoveObject", client.StatusCode);

            }

        }

        public void DeleteObject(string account, string sourceContainer, string objectName)

        {            

            if (String.IsNullOrWhiteSpace(sourceContainer))

                throw new ArgumentNullException("sourceContainer", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");

            Contract.EndContractBlock();

            var targetUrl = FolderConstants.TrashContainer + "/" + objectName;

            var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);

            using (var client = new RestClient(_baseClient))

            {

                if (!String.IsNullOrWhiteSpace(account))

                    client.BaseAddress = GetAccountUrl(account);

                client.Headers.Add("X-Move-From", sourceUrl);

                client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);

                Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);

                client.PutWithRetry(targetUrl, 3);

                var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};

                if (!expectedCodes.Contains(client.StatusCode))

                    throw CreateWebException("DeleteObject", client.StatusCode);

            }

        }

        private static WebException CreateWebException(string operation, HttpStatusCode statusCode)

        {

            return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));

        }

/*

        public IEnumerable<ObjectInfo> ListDirectories(ContainerInfo container)

        {

            var directories=this.ListObjects(container.Account, container.Name, "/");

        }

*/

        public bool CanUpload(string account, ObjectInfo cloudFile)

        {

            Contract.Requires(!String.IsNullOrWhiteSpace(account));

            Contract.Requires(cloudFile!=null);

            using (var client = new RestClient(_baseClient))

            {

                if (!String.IsNullOrWhiteSpace(account))

                    client.BaseAddress = GetAccountUrl(account);

                var parts = cloudFile.Name.Split('/');

                var folder = String.Join("/", parts,0,parts.Length-1);

                var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid());

                client.Parameters.Clear();

                try

                {

                    client.PutWithRetry(fileUrl, 3, @"application/octet-stream");

                    var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};

                    var result=(expectedCodes.Contains(client.StatusCode));

                    DeleteObject(account, cloudFile.Container, fileUrl);

                    return result;

                }

                catch

                {

                    return false;

                }

            }

        }

    }

    public class ShareAccountInfo

    {

        public DateTime? last_modified { get; set; }

        public string name { get; set; }

    }

}