[pithos-ms-client] / trunk%2FPithos.Network%2FCloudFilesClient.cs

// -----------------------------------------------------------------------
// <copyright file="CloudFilesClient.cs" company="GRNET">
// Copyright 2011-2012 GRNET S.A. All rights reserved.
// 
// Redistribution and use in source and binary forms, with or
// without modification, are permitted provided that the following
// conditions are met:
// 
//   1. Redistributions of source code must retain the above
//      copyright notice, this list of conditions and the following
//      disclaimer.
// 
//   2. Redistributions in binary form must reproduce the above
//      copyright notice, this list of conditions and the following
//      disclaimer in the documentation and/or other materials
//      provided with the distribution.
// 
// THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
// USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
// AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
// 
// The views and conclusions contained in the software and
// documentation are those of the authors and should not be
// interpreted as representing official policies, either expressed
// or implied, of GRNET S.A.
// </copyright>
// -----------------------------------------------------------------------


// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
//
// The class provides methods to upload/download files, delete files, manage containers


using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.Diagnostics.Contracts;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Pithos.Interfaces;
using log4net;

namespace Pithos.Network
{
    [Export(typeof(ICloudClient))]
    public class CloudFilesClient:ICloudClient
    {
        //CloudFilesClient uses *_baseClient* internally to communicate with the server
        //RestClient provides a REST-friendly interface over the standard WebClient.
        private RestClient _baseClient;
        

        //During authentication the client provides a UserName 
        public string UserName { get; set; }
        
        //and and ApiKey to the server
        public string ApiKey { get; set; }
        
        //And receives an authentication Token. This token must be provided in ALL other operations,
        //in the X-Auth-Token header
        private string _token;
        public string Token
        {
            get { return _token; }
            set
            {
                _token = value;
                _baseClient.Headers["X-Auth-Token"] = value;
            }
        }

        //The client also receives a StorageUrl after authentication. All subsequent operations must
        //use this url
        public Uri StorageUrl { get; set; }


        protected Uri RootAddressUri { get; set; }

       /* private WebProxy _proxy;
        public WebProxy Proxy
        {
            get { return _proxy; }
            set
            {
                _proxy = value;
                if (_baseClient != null)
                    _baseClient.Proxy = value;                
            }
        }
*/

        /* private Uri _proxy;
        public Uri Proxy
        {
            get { return _proxy; }
            set
            {
                _proxy = value;
                if (_baseClient != null)
                    _baseClient.Proxy = new WebProxy(value);                
            }
        }*/

        public double DownloadPercentLimit { get; set; }
        public double UploadPercentLimit { get; set; }

        public string AuthenticationUrl { get; set; }

 
        public string VersionPath
        {
            get { return UsePithos ? "v1" : "v1.0"; }
        }

        public bool UsePithos { get; set; }


        private static readonly ILog Log = LogManager.GetLogger("CloudFilesClient");

        public CloudFilesClient(string userName, string apiKey)
        {
            UserName = userName;
            ApiKey = apiKey;
        }

        public CloudFilesClient(AccountInfo accountInfo)
        {
            if (accountInfo==null)
                throw new ArgumentNullException("accountInfo");
            Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
            Contract.Ensures(StorageUrl != null);
            Contract.Ensures(_baseClient != null);
            Contract.Ensures(RootAddressUri != null);
            Contract.EndContractBlock();          

            _baseClient = new RestClient
            {
                BaseAddress = accountInfo.StorageUri.ToString(),
                Timeout = 10000,
                Retries = 3,
            };
            StorageUrl = accountInfo.StorageUri;
            Token = accountInfo.Token;
            UserName = accountInfo.UserName;

            //Get the root address (StorageUrl without the account)
            var storageUrl = StorageUrl.AbsoluteUri;
            var usernameIndex = storageUrl.LastIndexOf(UserName);
            var rootUrl = storageUrl.Substring(0, usernameIndex);
            RootAddressUri = new Uri(rootUrl);
        }


        public AccountInfo Authenticate()
        {
            if (String.IsNullOrWhiteSpace(UserName))
                throw new InvalidOperationException("UserName is empty");
            if (String.IsNullOrWhiteSpace(ApiKey))
                throw new InvalidOperationException("ApiKey is empty");
            if (String.IsNullOrWhiteSpace(AuthenticationUrl))
                throw new InvalidOperationException("AuthenticationUrl is empty");
            Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
            Contract.Ensures(StorageUrl != null);
            Contract.Ensures(_baseClient != null);
            Contract.Ensures(RootAddressUri != null);
            Contract.EndContractBlock();


            Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);

            var groups = new List<Group>();

            using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
            {                
               /* if (Proxy != null)
                    authClient.Proxy = Proxy;*/

                Contract.Assume(authClient.Headers!=null);

                authClient.Headers.Add("X-Auth-User", UserName);
                authClient.Headers.Add("X-Auth-Key", ApiKey);

                authClient.DownloadStringWithRetry(VersionPath, 3);

                authClient.AssertStatusOK("Authentication failed");

                var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
                if (String.IsNullOrWhiteSpace(storageUrl))
                    throw new InvalidOperationException("Failed to obtain storage url");
                
                _baseClient = new RestClient
                {
                    BaseAddress = storageUrl,
                    Timeout = 10000,
                    Retries = 3,
                    //Proxy=Proxy
                };

                StorageUrl = new Uri(storageUrl);
                
                //Get the root address (StorageUrl without the account)
                var usernameIndex=storageUrl.LastIndexOf(UserName);
                var rootUrl = storageUrl.Substring(0, usernameIndex);
                RootAddressUri = new Uri(rootUrl);
                
                var token = authClient.GetHeaderValue("X-Auth-Token");
                if (String.IsNullOrWhiteSpace(token))
                    throw new InvalidOperationException("Failed to obtain token url");
                Token = token;

               /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
                groups = (from key in keys
                            where key.StartsWith("X-Account-Group-")
                            let name = key.Substring(16)
                            select new Group(name, authClient.ResponseHeaders[key]))
                        .ToList();
                    
*/
            }

            Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
            

            return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};            

        }



        public IList<ContainerInfo> ListContainers(string account)
        {
            using (var client = new RestClient(_baseClient))
            {
                if (!String.IsNullOrWhiteSpace(account))
                    client.BaseAddress = GetAccountUrl(account);
                
                client.Parameters.Clear();
                client.Parameters.Add("format", "json");
                var content = client.DownloadStringWithRetry("", 3);
                client.AssertStatusOK("List Containers failed");

                if (client.StatusCode == HttpStatusCode.NoContent)
                    return new List<ContainerInfo>();
                var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
                
                foreach (var info in infos)
                {
                    info.Account = account;
                }
                return infos;
            }

        }

        private string GetAccountUrl(string account)
        {
            return new Uri(RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
        }

        public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
        {
            using (ThreadContext.Stacks["Share"].Push("List Accounts"))
            {
                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))
                {
                    client.Parameters.Clear();
                    client.Parameters.Add("format", "json");
                    client.IfModifiedSince = since;

                    //Extract the username from the base address
                    client.BaseAddress = RootAddressUri.AbsoluteUri;

                    var content = client.DownloadStringWithRetry(@"", 3);

                    client.AssertStatusOK("ListSharingAccounts failed");

                    //If the result is empty, return an empty list,
                    var infos = String.IsNullOrWhiteSpace(content)
                                    ? new List<ShareAccountInfo>()
                                //Otherwise deserialize the account list into a list of ShareAccountInfos
                                    : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);

                    Log.DebugFormat("END");
                    return infos;
                }
            }
        }

        //Request listing of all objects in a container modified since a specific time.
        //If the *since* value is missing, return all objects
        public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
        {

            using (ThreadContext.Stacks["Share"].Push("List Objects"))
            {
                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                var objects = new List<ObjectInfo>();
                var accounts = ListSharingAccounts(since);
                foreach (var account in accounts)
                {
                    var containers = ListContainers(account.name);
                    foreach (var container in containers)
                    {
                        var containerObjects = ListObjects(account.name, container.Name);
                        objects.AddRange(containerObjects);
                    }
                }
                if (Log.IsDebugEnabled) Log.DebugFormat("END");
                return objects;
            }
        }

        public void SetTags(ObjectInfo target,IDictionary<string,string> tags)
        {
            if (String.IsNullOrWhiteSpace(Token))
                throw new InvalidOperationException("The Token is not set");
            if (StorageUrl == null)
                throw new InvalidOperationException("The StorageUrl is not set");
            if (target == null)
                throw new ArgumentNullException("target");
            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Share"].Push("Share Object"))
            {
                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))
                {

                    client.BaseAddress = GetAccountUrl(target.Account);

                    client.Parameters.Clear();
                    client.Parameters.Add("update", "");

                    foreach (var tag in tags)
                    {
                        var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
                        client.Headers.Add(headerTag, tag.Value);
                    }
                    
                    client.DownloadStringWithRetry(target.Container, 3);

                    
                    client.AssertStatusOK("SetTags failed");
                    //If the status is NOT ACCEPTED we have a problem
                    if (client.StatusCode != HttpStatusCode.Accepted)
                    {
                        Log.Error("Failed to set tags");
                        throw new Exception("Failed to set tags");
                    }

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
                }
            }


        }

        public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
        {
            if (String.IsNullOrWhiteSpace(Token))
                throw new InvalidOperationException("The Token is not set");
            if (StorageUrl==null)
                throw new InvalidOperationException("The StorageUrl is not set");
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container");
            if (String.IsNullOrWhiteSpace(objectName))
                throw new ArgumentNullException("objectName");
            if (String.IsNullOrWhiteSpace(account))
                throw new ArgumentNullException("account");
            if (String.IsNullOrWhiteSpace(shareTo))
                throw new ArgumentNullException("shareTo");
            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Share"].Push("Share Object"))
            {
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
                
                using (var client = new RestClient(_baseClient))
                {

                    client.BaseAddress = GetAccountUrl(account);

                    client.Parameters.Clear();
                    client.Parameters.Add("format", "json");

                    string permission = "";
                    if (write)
                        permission = String.Format("write={0}", shareTo);
                    else if (read)
                        permission = String.Format("read={0}", shareTo);
                    client.Headers.Add("X-Object-Sharing", permission);

                    var content = client.DownloadStringWithRetry(container, 3);

                    client.AssertStatusOK("ShareObject failed");

                    //If the result is empty, return an empty list,
                    var infos = String.IsNullOrWhiteSpace(content)
                                    ? new List<ObjectInfo>()
                                //Otherwise deserialize the object list into a list of ObjectInfos
                                    : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
                }
            }


        }

        public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
        {
            if (accountInfo==null)
                throw new ArgumentNullException("accountInfo");
            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Account"].Push("GetPolicies"))
            {
                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))
                {
                    if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
                        client.BaseAddress = GetAccountUrl(accountInfo.UserName);

                    client.Parameters.Clear();
                    client.Parameters.Add("format", "json");                    
                    client.Head(String.Empty, 3);

                    var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
                    var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];

                    long quota, bytes;
                    if (long.TryParse(quotaValue, out quota))
                        accountInfo.Quota = quota;
                    if (long.TryParse(bytesValue, out bytes))
                        accountInfo.BytesUsed = bytes;
                    
                    return accountInfo;

                }

            }
        }

        public void UpdateMetadata(ObjectInfo objectInfo)
        {
            if (objectInfo == null)
                throw new ArgumentNullException("objectInfo");
            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
            {
                if (Log.IsDebugEnabled) Log.DebugFormat("START");


                using(var client=new RestClient(_baseClient))
                {

                    client.BaseAddress = GetAccountUrl(objectInfo.Account);
                    
                    client.Parameters.Clear();
                    

                    //Set Tags
                    foreach (var tag in objectInfo.Tags)
                    {
                        var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
                        client.Headers.Add(headerTag, tag.Value);
                    }

                    //Set Permissions

                    var permissions=objectInfo.GetPermissionString();
                    client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);

                    client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
                    client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
                    client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
                    var isPublic = objectInfo.IsPublic.ToString().ToLower();
                    client.Headers.Add("X-Object-Public", isPublic);


                    var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);
                    var uri = uriBuilder.Uri;

                    client.UploadValues(uri,new NameValueCollection());


                    client.AssertStatusOK("UpdateMetadata failed");
                    //If the status is NOT ACCEPTED or OK we have a problem
                    if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
                    {
                        Log.Error("Failed to update metadata");
                        throw new Exception("Failed to update metadata");
                    }

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
                }
            }

        }

        public void UpdateMetadata(ContainerInfo containerInfo)
        {
            if (containerInfo == null)
                throw new ArgumentNullException("containerInfo");
            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
            {
                if (Log.IsDebugEnabled) Log.DebugFormat("START");


                using(var client=new RestClient(_baseClient))
                {

                    client.BaseAddress = GetAccountUrl(containerInfo.Account);
                    
                    client.Parameters.Clear();
                    

                    //Set Tags
                    foreach (var tag in containerInfo.Tags)
                    {
                        var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
                        client.Headers.Add(headerTag, tag.Value);
                    }

                    
                    //Set Policies
                    foreach (var policy in containerInfo.Policies)
                    {
                        var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
                        client.Headers.Add(headerPolicy, policy.Value);
                    }


                    var uriBuilder = client.GetAddressBuilder(containerInfo.Name,"");
                    var uri = uriBuilder.Uri;

                    client.UploadValues(uri,new NameValueCollection());


                    client.AssertStatusOK("UpdateMetadata failed");
                    //If the status is NOT ACCEPTED or OK we have a problem
                    if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
                    {
                        Log.Error("Failed to update metadata");
                        throw new Exception("Failed to update metadata");
                    }

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
                }
            }

        }


        public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container");
            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Objects"].Push("List"))
            {
                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))
                {
                    if (!String.IsNullOrWhiteSpace(account))
                        client.BaseAddress = GetAccountUrl(account);

                    client.Parameters.Clear();
                    client.Parameters.Add("format", "json");
                    client.IfModifiedSince = since;
                    var content = client.DownloadStringWithRetry(container, 3);

                    client.AssertStatusOK("ListObjects failed");

                    //If the result is empty, return an empty list,
                    var infos = String.IsNullOrWhiteSpace(content)
                                    ? new List<ObjectInfo>()
                                //Otherwise deserialize the object list into a list of ObjectInfos
                                    : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);

                    foreach (var info in infos)
                    {
                        info.Container = container;
                        info.Account = account;
                    }
                    if (Log.IsDebugEnabled) Log.DebugFormat("START");
                    return infos;
                }
            }
        }



        public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container");
            if (String.IsNullOrWhiteSpace(folder))
                throw new ArgumentNullException("folder");
            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Objects"].Push("List"))
            {
                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))
                {
                    if (!String.IsNullOrWhiteSpace(account))
                        client.BaseAddress = GetAccountUrl(account);

                    client.Parameters.Clear();
                    client.Parameters.Add("format", "json");
                    client.Parameters.Add("path", folder);
                    client.IfModifiedSince = since;
                    var content = client.DownloadStringWithRetry(container, 3);
                    client.AssertStatusOK("ListObjects failed");

                    var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);

                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
                    return infos;
                }
            }
        }

 
        public bool ContainerExists(string account, string container)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container", "The container property can't be empty");
            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Containters"].Push("Exists"))
            {
                if (Log.IsDebugEnabled) Log.DebugFormat("START");

                using (var client = new RestClient(_baseClient))
                {
                    if (!String.IsNullOrWhiteSpace(account))
                        client.BaseAddress = GetAccountUrl(account);

                    client.Parameters.Clear();
                    client.Head(container, 3);
                                        
                    bool result;
                    switch (client.StatusCode)
                    {
                        case HttpStatusCode.OK:
                        case HttpStatusCode.NoContent:
                            result=true;
                            break;
                        case HttpStatusCode.NotFound:
                            result=false;
                            break;
                        default:
                            throw CreateWebException("ContainerExists", client.StatusCode);
                    }
                    if (Log.IsDebugEnabled) Log.DebugFormat("END");

                    return result;
                }
                
            }
        }

        public bool ObjectExists(string account, string container, string objectName)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container", "The container property can't be empty");
            if (String.IsNullOrWhiteSpace(objectName))
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
            Contract.EndContractBlock();

            using (var client = new RestClient(_baseClient))
            {
                if (!String.IsNullOrWhiteSpace(account))
                    client.BaseAddress = GetAccountUrl(account);

                client.Parameters.Clear();
                client.Head(container + "/" + objectName, 3);

                switch (client.StatusCode)
                {
                    case HttpStatusCode.OK:
                    case HttpStatusCode.NoContent:
                        return true;
                    case HttpStatusCode.NotFound:
                        return false;
                    default:
                        throw CreateWebException("ObjectExists", client.StatusCode);
                }
            }

        }

        public ObjectInfo GetObjectInfo(string account, string container, string objectName)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container", "The container property can't be empty");
            if (String.IsNullOrWhiteSpace(objectName))
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
            Contract.EndContractBlock();

            using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
            {                

                using (var client = new RestClient(_baseClient))
                {
                    if (!String.IsNullOrWhiteSpace(account))
                        client.BaseAddress = GetAccountUrl(account);
                    try
                    {
                        client.Parameters.Clear();

                        client.Head(container + "/" + objectName, 3);

                        if (client.TimedOut)
                            return ObjectInfo.Empty;

                        switch (client.StatusCode)
                        {
                            case HttpStatusCode.OK:
                            case HttpStatusCode.NoContent:
                                var keys = client.ResponseHeaders.AllKeys.AsQueryable();
                                var tags = client.GetMeta("X-Object-Meta-");
                                var extensions = (from key in keys
                                                  where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
                                                  select new {Name = key, Value = client.ResponseHeaders[key]})
                                    .ToDictionary(t => t.Name, t => t.Value);

                                var permissions=client.GetHeaderValue("X-Object-Sharing", true);
                                
                                
                                var info = new ObjectInfo
                                               {
                                                   Account = account,
                                                   Container = container,
                                                   Name = objectName,
                                                   Hash = client.GetHeaderValue("ETag"),
                                                   Content_Type = client.GetHeaderValue("Content-Type"),
                                                   Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),
                                                   Tags = tags,
                                                   Last_Modified = client.LastModified,
                                                   Extensions = extensions,
                                                   ContentEncoding=client.GetHeaderValue("Content-Encoding",true),
                                                   ContendDisposition = client.GetHeaderValue("Content-Disposition",true),
                                                   Manifest=client.GetHeaderValue("X-Object-Manifest",true),
                                                   PublicUrl=client.GetHeaderValue("X-Object-Public",true),                                                   
                                               };
                                info.SetPermissions(permissions);
                                return info;
                            case HttpStatusCode.NotFound:
                                return ObjectInfo.Empty;
                            default:
                                throw new WebException(
                                    String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
                                                  objectName, client.StatusCode));
                        }

                    }
                    catch (RetryException)
                    {
                        Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.",objectName);
                        return ObjectInfo.Empty;
                    }
                    catch (WebException e)
                    {
                        Log.Error(
                            String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
                                          objectName, client.StatusCode), e);
                        throw;
                    }
                }                
            }

        }

        public void CreateFolder(string account, string container, string folder)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container", "The container property can't be empty");
            if (String.IsNullOrWhiteSpace(folder))
                throw new ArgumentNullException("folder", "The folder property can't be empty");
            Contract.EndContractBlock();

            var folderUrl=String.Format("{0}/{1}",container,folder);
            using (var client = new RestClient(_baseClient))
            {
                if (!String.IsNullOrWhiteSpace(account))
                    client.BaseAddress = GetAccountUrl(account);

                client.Parameters.Clear();
                client.Headers.Add("Content-Type", @"application/directory");
                client.Headers.Add("Content-Length", "0");
                client.PutWithRetry(folderUrl, 3);

                if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
                    throw CreateWebException("CreateFolder", client.StatusCode);
            }
        }

     

        public ContainerInfo GetContainerInfo(string account, string container)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container", "The container property can't be empty");
            Contract.EndContractBlock();

            using (var client = new RestClient(_baseClient))
            {
                if (!String.IsNullOrWhiteSpace(account))
                    client.BaseAddress = GetAccountUrl(account);                

                client.Head(container);
                switch (client.StatusCode)
                {
                    case HttpStatusCode.OK:
                    case HttpStatusCode.NoContent:
                        var tags = client.GetMeta("X-Container-Meta-");
                        var policies = client.GetMeta("X-Container-Policy-");

                        var containerInfo = new ContainerInfo
                                                {
                                                    Account=account,
                                                    Name = container,
                                                    Count =
                                                        long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
                                                    Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
                                                    BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
                                                    BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),
                                                    Last_Modified=client.LastModified,
                                                    Tags=tags,
                                                    Policies=policies
                                                };
                        

                        return containerInfo;
                    case HttpStatusCode.NotFound:
                        return ContainerInfo.Empty;
                    default:
                        throw CreateWebException("GetContainerInfo", client.StatusCode);
                }
            }
        }

        public void CreateContainer(string account, string container)
        {
            if (String.IsNullOrWhiteSpace(account))
                throw new ArgumentNullException("account");
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container");
            Contract.EndContractBlock();

            using (var client = new RestClient(_baseClient))
            {
                if (!String.IsNullOrWhiteSpace(account))
                    client.BaseAddress = GetAccountUrl(account);

                client.PutWithRetry(container, 3);
                var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
                if (!expectedCodes.Contains(client.StatusCode))
                    throw CreateWebException("CreateContainer", client.StatusCode);
            }
        }

        public void DeleteContainer(string account, string container)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container", "The container property can't be empty");
            Contract.EndContractBlock();

            using (var client = new RestClient(_baseClient))
            {
                if (!String.IsNullOrWhiteSpace(account))
                    client.BaseAddress = GetAccountUrl(account);

                client.DeleteWithRetry(container, 3);
                var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
                if (!expectedCodes.Contains(client.StatusCode))
                    throw CreateWebException("DeleteContainer", client.StatusCode);
            }

        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="account"></param>
        /// <param name="container"></param>
        /// <param name="objectName"></param>
        /// <param name="fileName"></param>
        /// <returns></returns>
        /// <remarks>This method should have no timeout or a very long one</remarks>
        //Asynchronously download the object specified by *objectName* in a specific *container* to 
        // a local file
        public Task GetObject(string account, string container, string objectName, string fileName)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container", "The container property can't be empty");
            if (String.IsNullOrWhiteSpace(objectName))
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");            
            Contract.EndContractBlock();

            try
            {
                //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
                //object to avoid concurrency errors.
                //
                //Download operations take a long time therefore they have no timeout.
                var client = new RestClient(_baseClient) { Timeout = 0 };
                if (!String.IsNullOrWhiteSpace(account))
                    client.BaseAddress = GetAccountUrl(account);

                //The container and objectName are relative names. They are joined with the client's
                //BaseAddress to create the object's absolute address
                var builder = client.GetAddressBuilder(container, objectName);
                var uri = builder.Uri;

                //Download progress is reported to the Trace log
                Log.InfoFormat("[GET] START {0}", objectName);
                client.DownloadProgressChanged += (sender, args) => 
                    Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
                                    fileName, args.ProgressPercentage,
                                    args.BytesReceived,
                                    args.TotalBytesToReceive);                                


                //Start downloading the object asynchronously
                var downloadTask = client.DownloadFileTask(uri, fileName);
                
                //Once the download completes
                return downloadTask.ContinueWith(download =>
                                      {
                                          //Delete the local client object
                                          client.Dispose();
                                          //And report failure or completion
                                          if (download.IsFaulted)
                                          {
                                              Log.ErrorFormat("[GET] FAIL for {0} with \r{1}", objectName,
                                                               download.Exception);
                                          }
                                          else
                                          {
                                              Log.InfoFormat("[GET] END {0}", objectName);                                             
                                          }
                                      });
            }
            catch (Exception exc)
            {
                Log.ErrorFormat("[GET] END {0} with {1}", objectName, exc);
                throw;
            }



        }

        public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container");
            if (String.IsNullOrWhiteSpace(objectName))
                throw new ArgumentNullException("objectName");
            if (hash==null)
                throw new ArgumentNullException("hash");
            if (String.IsNullOrWhiteSpace(Token))
                throw new InvalidOperationException("Invalid Token");
            if (StorageUrl == null)
                throw new InvalidOperationException("Invalid Storage Url");
            Contract.EndContractBlock();


            //Don't use a timeout because putting the hashmap may be a long process
            var client = new RestClient(_baseClient) { Timeout = 0 };           
            if (!String.IsNullOrWhiteSpace(account))
                client.BaseAddress = GetAccountUrl(account);

            //The container and objectName are relative names. They are joined with the client's
            //BaseAddress to create the object's absolute address
            var builder = client.GetAddressBuilder(container, objectName);
            builder.Query = "format=json&hashmap";
            var uri = builder.Uri;


            //Send the tree hash as Json to the server            
            client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
            var jsonHash = hash.ToJson();
            var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);
            
            return uploadTask.ContinueWith(t =>
            {

                var empty = (IList<string>)new List<string>();
                

                //The server will respond either with 201-created if all blocks were already on the server
                if (client.StatusCode == HttpStatusCode.Created)                    
                {
                    //in which case we return an empty hash list
                    return empty;
                }
                //or with a 409-conflict and return the list of missing parts
                //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception                
                if (t.IsFaulted)
                {
                    var ex = t.Exception.InnerException;
                    var we = ex as WebException;
                    var response = we.Response as HttpWebResponse;
                    if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
                    {
                        //In case of 409 the missing parts will be in the response content                        
                        using (var stream = response.GetResponseStream())
                        using(var reader=new StreamReader(stream))
                        {
                            Debug.Assert(stream.Position == 0);
                            //We used to have to cleanup the content before returning it because it contains
                            //error content after the list of hashes
                            //
                            //As of 30/1/2012, the result is a proper Json array so we don't need to read the content
                            //line by line
                            
                            var serializer = new JsonSerializer();                            
                            var hashes=(List<string>)serializer.Deserialize(reader, typeof (List<string>));

                            return hashes;
                        }                        
                    }                    
                    //Any other status code is unexpected and the exception should be rethrown
                    throw ex;
                    
                }
                //Any other status code is unexpected but there was no exception. We can probably continue processing
                Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);                    
                
                return empty;
            });

        }

        public Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end)
        {
            if (String.IsNullOrWhiteSpace(Token))
                throw new InvalidOperationException("Invalid Token");
            if (StorageUrl == null)
                throw new InvalidOperationException("Invalid Storage Url");
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container");
            if (relativeUrl== null)
                throw new ArgumentNullException("relativeUrl");
            if (end.HasValue && end<0)
                throw new ArgumentOutOfRangeException("end");
            if (start<0)
                throw new ArgumentOutOfRangeException("start");
            Contract.EndContractBlock();


            //Don't use a timeout because putting the hashmap may be a long process
            var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
            if (!String.IsNullOrWhiteSpace(account))
                client.BaseAddress = GetAccountUrl(account);

            var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
            var uri = builder.Uri;

            return client.DownloadDataTask(uri)
                .ContinueWith(t=>
                                  {
                                      client.Dispose();
                                      return t.Result;
                                  });
        }


        public async Task PostBlock(string account, string container, byte[] block, int offset, int count)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container");
            if (block == null)
                throw new ArgumentNullException("block");
            if (offset < 0 || offset >= block.Length)
                throw new ArgumentOutOfRangeException("offset");
            if (count < 0 || count > block.Length)
                throw new ArgumentOutOfRangeException("count");
            if (String.IsNullOrWhiteSpace(Token))
                throw new InvalidOperationException("Invalid Token");
            if (StorageUrl == null)
                throw new InvalidOperationException("Invalid Storage Url");                        
            Contract.EndContractBlock();


            try
            {

            //Don't use a timeout because putting the hashmap may be a long process
                using (var client = new RestClient(_baseClient) { Timeout = 0 })
                {
                    if (!String.IsNullOrWhiteSpace(account))
                        client.BaseAddress = GetAccountUrl(account);

                    var builder = client.GetAddressBuilder(container, "");
                    //We are doing an update
                    builder.Query = "update";
                    var uri = builder.Uri;

                    client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";

                    Log.InfoFormat("[BLOCK POST] START");

                    client.UploadProgressChanged += (sender, args) =>
                                                    Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
                                                                   args.ProgressPercentage, args.BytesSent,
                                                                   args.TotalBytesToSend);
                    client.UploadFileCompleted += (sender, args) =>
                                                  Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");

                    var buffer = new byte[count];
                    Buffer.BlockCopy(block, offset, buffer, 0, count);
                    //Send the block
                    await client.UploadDataTask(uri, "POST", buffer);
                    Log.InfoFormat("[BLOCK POST] END");
                }
            }
            catch (Exception exc)
            {
                Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);                                        
                throw;
            }
        }


        public async Task<TreeHash> GetHashMap(string account, string container, string objectName)
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container");
            if (String.IsNullOrWhiteSpace(objectName))
                throw new ArgumentNullException("objectName");
            if (String.IsNullOrWhiteSpace(Token))
                throw new InvalidOperationException("Invalid Token");
            if (StorageUrl == null)
                throw new InvalidOperationException("Invalid Storage Url");
            Contract.EndContractBlock();

            try
            {
                //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
                //object to avoid concurrency errors.
                //
                //Download operations take a long time therefore they have no timeout.
                //TODO: Do they really? this is a hashmap operation, not a download
                
                //Start downloading the object asynchronously
                using (var client = new RestClient(_baseClient) { Timeout = 0 })
                {
                    if (!String.IsNullOrWhiteSpace(account))
                        client.BaseAddress = GetAccountUrl(account);

                    //The container and objectName are relative names. They are joined with the client's
                    //BaseAddress to create the object's absolute address
                    var builder = client.GetAddressBuilder(container, objectName);
                    builder.Query = "format=json&hashmap";
                    var uri = builder.Uri;


                    var json = await client.DownloadStringTaskAsync(uri);
                    var treeHash = TreeHash.Parse(json);
                    Log.InfoFormat("[GET HASH] END {0}", objectName);
                    return treeHash;
                }
            }
            catch (Exception exc)
            {
                Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
                throw;
            }

        }


        /// <summary>
        /// 
        /// </summary>
        /// <param name="account"></param>
        /// <param name="container"></param>
        /// <param name="objectName"></param>
        /// <param name="fileName"></param>
        /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
        /// <remarks>>This method should have no timeout or a very long one</remarks>
        public async Task PutObject(string account, string container, string objectName, string fileName, string hash = null, string contentType = "application/octet-stream")
        {
            if (String.IsNullOrWhiteSpace(container))
                throw new ArgumentNullException("container", "The container property can't be empty");
            if (String.IsNullOrWhiteSpace(objectName))
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
            if (String.IsNullOrWhiteSpace(fileName))
                throw new ArgumentNullException("fileName", "The fileName property can't be empty");
/*
            if (!File.Exists(fileName) && !Directory.Exists(fileName))
                throw new FileNotFoundException("The file or directory does not exist",fileName);
*/
            Contract.EndContractBlock();
            
            try
            {

                using (var client = new RestClient(_baseClient) { Timeout = 0 })
                {
                    if (!String.IsNullOrWhiteSpace(account))
                        client.BaseAddress = GetAccountUrl(account);

                    var builder = client.GetAddressBuilder(container, objectName);
                    var uri = builder.Uri;

                    string etag = hash ?? CalculateHash(fileName);

                    client.Headers.Add("Content-Type", contentType);
                    client.Headers.Add("ETag", etag);


                    Log.InfoFormat("[PUT] START {0}", objectName);
                    client.UploadProgressChanged += (sender, args) =>
                                                        {
                                                            using (ThreadContext.Stacks["PUT"].Push("Progress"))
                                                            {
                                                                Log.InfoFormat("{0} {1}% {2} of {3}", fileName,
                                                                               args.ProgressPercentage,
                                                                               args.BytesSent, args.TotalBytesToSend);
                                                            }
                                                        };

                    client.UploadFileCompleted += (sender, args) =>
                                                      {
                                                          using (ThreadContext.Stacks["PUT"].Push("Progress"))
                                                          {
                                                              Log.InfoFormat("Completed {0}", fileName);
                                                          }
                                                      };
                    if (contentType=="application/directory")
                        await client.UploadDataTaskAsync(uri, "PUT", new byte[0]);
                    else
                        await client.UploadFileTaskAsync(uri, "PUT", fileName);
                }

                Log.InfoFormat("[PUT] END {0}", objectName);
            }
            catch (Exception exc)
            {
                Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
                throw;
            }                

        }
       
        
        private static string CalculateHash(string fileName)
        {
            Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
            Contract.EndContractBlock();

            string hash;
            using (var hasher = MD5.Create())
            using(var stream=File.OpenRead(fileName))
            {
                var hashBuilder=new StringBuilder();
                foreach (byte b in hasher.ComputeHash(stream))
                    hashBuilder.Append(b.ToString("x2").ToLower());
                hash = hashBuilder.ToString();                
            }
            return hash;
        }
        
        public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
        {
            if (String.IsNullOrWhiteSpace(sourceContainer))
                throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
            if (String.IsNullOrWhiteSpace(oldObjectName))
                throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
            if (String.IsNullOrWhiteSpace(targetContainer))
                throw new ArgumentNullException("targetContainer", "The container property can't be empty");
            if (String.IsNullOrWhiteSpace(newObjectName))
                throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
            Contract.EndContractBlock();

            var targetUrl = targetContainer + "/" + newObjectName;
            var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);

            using (var client = new RestClient(_baseClient))
            {
                if (!String.IsNullOrWhiteSpace(account))
                    client.BaseAddress = GetAccountUrl(account);

                client.Headers.Add("X-Move-From", sourceUrl);
                client.PutWithRetry(targetUrl, 3);

                var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
                if (!expectedCodes.Contains(client.StatusCode))
                    throw CreateWebException("MoveObject", client.StatusCode);
            }
        }

        public void DeleteObject(string account, string sourceContainer, string objectName)
        {            
            if (String.IsNullOrWhiteSpace(sourceContainer))
                throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
            if (String.IsNullOrWhiteSpace(objectName))
                throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
            Contract.EndContractBlock();

            var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
            var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);

            using (var client = new RestClient(_baseClient))
            {
                if (!String.IsNullOrWhiteSpace(account))
                    client.BaseAddress = GetAccountUrl(account);

                client.Headers.Add("X-Move-From", sourceUrl);
                client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
                client.PutWithRetry(targetUrl, 3);

                var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
                if (!expectedCodes.Contains(client.StatusCode))
                    throw CreateWebException("DeleteObject", client.StatusCode);
            }
        }

      
        private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
        {
            return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
        }

        
    }

    public class ShareAccountInfo
    {
        public DateTime? last_modified { get; set; }
        public string name { get; set; }
    }
}