Synnefo » Pithos MS Client

// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos

//

// The class provides methods to upload/download files, delete files, manage containers

using System;

using System.Collections.Generic;

using System.ComponentModel.Composition;

using System.Diagnostics;

using System.Diagnostics.Contracts;

using System.Globalization;

using System.IO;

using System.Linq;

using System.Net;

using System.Security.Cryptography;

using System.Text;

using System.Threading.Algorithms;

using System.Threading.Tasks;

using Newtonsoft.Json;

using Pithos.Interfaces;

using WebHeaderCollection = System.Net.WebHeaderCollection;

namespace Pithos.Network

{

    [Export(typeof(ICloudClient))]

    public class CloudFilesClient:ICloudClient

    {

        //CloudFilesClient uses *_baseClient* internally to communicate with the server

        //RestClient provides a REST-friendly interface over the standard WebClient.

        private RestClient _baseClient;

        //Some operations can specify a Timeout. The default value of all timeouts is 10 seconds

        private readonly TimeSpan _shortTimeout = TimeSpan.FromSeconds(10);

        //Some operations can be retried before failing. The default number of retries is 5

        private readonly int _retries = 5;        

        //During authentication the client provides a UserName 

        public string UserName { get; set; }

        //and and ApiKey to the server

        public string ApiKey { get; set; }

        //And receives an authentication Token. This token must be provided in ALL other operations,

        //in the X-Auth-Token header

        public string Token { get; set; }

        //The client also receives a StorageUrl after authentication. All subsequent operations must

        //use this url

        public Uri StorageUrl { get; set; }

        public Uri Proxy { get; set; }

        public double DownloadPercentLimit { get; set; }

        public double UploadPercentLimit { get; set; }

        public string AuthenticationUrl { get; set; }

        public string VersionPath

        {

            get { return UsePithos ? "v1" : "v1.0"; }

        }

        public bool UsePithos { get; set; }

        private bool _authenticated = false;

        //

        public void Authenticate(string userName,string apiKey)

        {

            Trace.TraceInformation("[AUTHENTICATE] Start for {0}", userName);

            if (String.IsNullOrWhiteSpace(userName))

                throw new ArgumentNullException("userName", "The userName property can't be empty");

            if (String.IsNullOrWhiteSpace(apiKey))

                throw new ArgumentNullException("apiKey", "The apiKey property can't be empty");

            if (_authenticated)

                return;

            UserName = userName;

            ApiKey = apiKey;

            using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})

            {

                if (Proxy != null)

                    authClient.Proxy = new WebProxy(Proxy);

                authClient.Headers.Add("X-Auth-User", UserName);

                authClient.Headers.Add("X-Auth-Key", ApiKey);

                authClient.DownloadStringWithRetry(VersionPath, 3);

                authClient.AssertStatusOK("Authentication failed");

                var storageUrl = authClient.GetHeaderValue("X-Storage-Url");

                if (String.IsNullOrWhiteSpace(storageUrl))

                    throw new InvalidOperationException("Failed to obtain storage url");

                StorageUrl = new Uri(storageUrl);

                var token = authClient.GetHeaderValue("X-Auth-Token");

                if (String.IsNullOrWhiteSpace(token))

                    throw new InvalidOperationException("Failed to obtain token url");

                Token = token;

            }

            _baseClient = new RestClient{

                BaseAddress  = StorageUrl.AbsoluteUri,                

                Timeout=10000,

                Retries=3};

            if (Proxy!=null)

                _baseClient.Proxy = new WebProxy(Proxy);

            _baseClient.Headers.Add("X-Auth-Token", Token);

            Trace.TraceInformation("[AUTHENTICATE] End for {0}", userName);

        }

        public IList<ContainerInfo> ListContainers()

        {

            using (var client = new RestClient(_baseClient))

            {

                client.Parameters.Clear();

                client.Parameters.Add("format", "json");

                var content = client.DownloadStringWithRetry("", 3);

                client.AssertStatusOK("List Containers failed");

                if (client.StatusCode == HttpStatusCode.NoContent)

                    return new List<ContainerInfo>();

                var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);

                return infos;

            }

        }

        //Request listing of all objects in a container modified since a specific time.

        //If the *since* value is missing, return all objects

        public IList<ObjectInfo> ListObjects(string container, DateTime? since = null)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

            Contract.EndContractBlock();

            Trace.TraceInformation("[START] ListObjects");

            using (var client = new RestClient(_baseClient))

            {

                client.Parameters.Clear();

                client.Parameters.Add("format", "json");

                client.IfModifiedSince = since;

                var content = client.DownloadStringWithRetry(container, 3);

                client.AssertStatusOK("ListObjects failed");

                //If the result is empty, return an empty list,

                var infos=String.IsNullOrWhiteSpace(content) 

                    ? new List<ObjectInfo>() 

                    //Otherwise deserialize the object list into a list of ObjectInfos

                    : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);

                Trace.TraceInformation("[END] ListObjects");

                return infos;

            }

        }

        public IList<ObjectInfo> ListObjects(string container, string folder, DateTime? since = null)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

            if (String.IsNullOrWhiteSpace(folder))

                throw new ArgumentNullException("folder");

            Contract.EndContractBlock();

            Trace.TraceInformation("[START] ListObjects");

            using (var client = new RestClient(_baseClient))

            {

                client.Parameters.Clear();

                client.Parameters.Add("format", "json");

                client.Parameters.Add("path", folder);

                client.IfModifiedSince = since;

                var content = client.DownloadStringWithRetry(container, 3);

                client.AssertStatusOK("ListObjects failed");

                var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);

                Trace.TraceInformation("[END] ListObjects");

                return infos;

            }

        }

        public bool ContainerExists(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            using (var client = new RestClient(_baseClient))

            {

                client.Parameters.Clear();

                client.Head(container, 3);

                switch (client.StatusCode)

                {

                    case HttpStatusCode.OK:

                    case HttpStatusCode.NoContent:

                        return true;

                    case HttpStatusCode.NotFound:

                        return false;

                    default:

                        throw CreateWebException("ContainerExists", client.StatusCode);

                }

            }

        }

        public bool ObjectExists(string container,string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            using (var client = new RestClient(_baseClient))

            {

                client.Parameters.Clear();

                client.Head(container + "/" + objectName, 3);

                switch (client.StatusCode)

                {

                    case HttpStatusCode.OK:

                    case HttpStatusCode.NoContent:

                        return true;

                    case HttpStatusCode.NotFound:

                        return false;

                    default:

                        throw CreateWebException("ObjectExists", client.StatusCode);

                }

            }

        }

        public ObjectInfo GetObjectInfo(string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            using (var client = new RestClient(_baseClient))

            {

                try

                {

                    client.Parameters.Clear();

                    client.Head(container + "/" + objectName, 3);

                    if (client.TimedOut)

                        return ObjectInfo.Empty;

                    switch (client.StatusCode)

                    {

                        case HttpStatusCode.OK:

                        case HttpStatusCode.NoContent:

                            var keys = client.ResponseHeaders.AllKeys.AsQueryable();

                            var tags = (from key in keys

                                        where key.StartsWith("X-Object-Meta-")

                                        let name = key.Substring(14)

                                        select new {Name = name, Value = client.ResponseHeaders[name]})

                                .ToDictionary(t => t.Name, t => t.Value);

                            var extensions = (from key in keys

                                              where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")

                                              let name = key.Substring(9)

                                              select new {Name = name, Value = client.ResponseHeaders[name]})

                                .ToDictionary(t => t.Name, t => t.Value);

                            return new ObjectInfo

                                       {

                                           Name = objectName,

                                           /*Bytes =

                                               long.Parse(client.GetHeaderValue("Content-Length")),*/

                                           Hash = client.GetHeaderValue("ETag"),

                                           Content_Type = client.GetHeaderValue("Content-Type"),

                                           Tags = tags,

                                           Extensions = extensions

                                       };

                        case HttpStatusCode.NotFound:

                            return ObjectInfo.Empty;

                        default:

                            throw new WebException(

                                String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",

                                              objectName, client.StatusCode));

                    }

                }

                catch(RetryException e)

                {

                    Trace.TraceWarning("[RETRY FAIL] GetObjectInfo for {0} failed.");

                    return ObjectInfo.Empty;

                }

                catch(WebException e)

                {

                    Trace.TraceError(

                        String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",

                                      objectName, client.StatusCode), e);

                    throw;

                }

            }

        }

        public void CreateFolder(string container, string folder)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(folder))

                throw new ArgumentNullException("folder", "The folder property can't be empty");

            var folderUrl=String.Format("{0}/{1}",container,folder);

            using (var client = new RestClient(_baseClient))

            {

                client.Parameters.Clear();

                client.Headers.Add("Content-Type", @"application/directory");

                client.Headers.Add("Content-Length", "0");

                client.PutWithRetry(folderUrl, 3);

                if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)

                    throw CreateWebException("CreateFolder", client.StatusCode);

            }

        }

        public ContainerInfo GetContainerInfo(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            using (var client = new RestClient(_baseClient))

            {

                client.Head(container);

                switch (client.StatusCode)

                {

                    case HttpStatusCode.OK:

                    case HttpStatusCode.NoContent:

                        var containerInfo = new ContainerInfo

                                                {

                                                    Name = container,

                                                    Count =

                                                        long.Parse(client.GetHeaderValue("X-Container-Object-Count")),

                                                    Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),

                                                    BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),

                                                    BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size"))

                                                };

                        return containerInfo;

                    case HttpStatusCode.NotFound:

                        return ContainerInfo.Empty;

                    default:

                        throw CreateWebException("GetContainerInfo", client.StatusCode);

                }

            }

        }

        public void CreateContainer(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            using (var client = new RestClient(_baseClient))

            {

                client.PutWithRetry(container, 3);

                var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};

                if (!expectedCodes.Contains(client.StatusCode))

                    throw CreateWebException("CreateContainer", client.StatusCode);

            }

        }

        public void DeleteContainer(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            using (var client = new RestClient(_baseClient))

            {

                client.DeleteWithRetry(container, 3);

                var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};

                if (!expectedCodes.Contains(client.StatusCode))

                    throw CreateWebException("DeleteContainer", client.StatusCode);

            }

        }

        /// <summary>

        /// 

        /// </summary>

        /// <param name="container"></param>

        /// <param name="objectName"></param>

        /// <param name="fileName"></param>

        /// <returns></returns>

        /// <remarks>This method should have no timeout or a very long one</remarks>

        //Asynchronously download the object specified by *objectName* in a specific *container* to 

        // a local file

        public Task GetObject(string container, string objectName, string fileName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");            

            Contract.EndContractBlock();

            try

            {

                //The container and objectName are relative names. They are joined with the client's

                //BaseAddress to create the object's absolute address

                var builder = GetAddressBuilder(container, objectName);

                var uri = builder.Uri;

                //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient

                //object to avoid concurrency errors.

                //

                //Download operations take a long time therefore they have no timeout.

                var client = new RestClient(_baseClient) { Timeout = 0 };

                //Download progress is reported to the Trace log

                Trace.TraceInformation("[GET] START {0}", objectName);

                client.DownloadProgressChanged += (sender, args) => 

                    Trace.TraceInformation("[GET PROGRESS] {0} {1}% {2} of {3}",

                                    fileName, args.ProgressPercentage,

                                    args.BytesReceived,

                                    args.TotalBytesToReceive);                                

                //Start downloading the object asynchronously

                var downloadTask = client.DownloadFileTask(uri, fileName);

                //Once the download completes

                return downloadTask.ContinueWith(download =>

                                      {

                                          //Delete the local client object

                                          client.Dispose();

                                          //And report failure or completion

                                          if (download.IsFaulted)

                                          {

                                              Trace.TraceError("[GET] FAIL for {0} with \r{1}", objectName,

                                                               download.Exception);

                                          }

                                          else

                                          {

                                              Trace.TraceInformation("[GET] END {0}", objectName);                                             

                                          }

                                      });

            }

            catch (Exception exc)

            {

                Trace.TraceError("[GET] END {0} with {1}", objectName, exc);

                throw;

            }

        }

        public Task<string> PutHashMap(string container, string objectName, TreeHash hash)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName");

            if (hash==null)

                throw new ArgumentNullException("hash");

            if (String.IsNullOrWhiteSpace(Token))

                throw new InvalidOperationException("Invalid Token");

            if (StorageUrl == null)

                throw new InvalidOperationException("Invalid Storage Url");

            Contract.EndContractBlock();

            //The container and objectName are relative names. They are joined with the client's

            //BaseAddress to create the object's absolute address

            var builder = GetAddressBuilder(container, objectName);

            builder.Query = "format=json&hashmap";

            var uri = builder.Uri;

            //Don't use a timeout because putting the hashmap may be a long process

            var client = new RestClient(_baseClient) { Timeout = 0 };

            //Send the tree hash as Json to the server            

            client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";

            var uploadTask=client.UploadStringTask(uri, "PUT", hash.ToJson());

            return uploadTask.ContinueWith(t =>

            {

                //The server will respond either with 201-created if all blocks were already on the server

                if (client.StatusCode == HttpStatusCode.Created)

                    //in which case we return an empty hash list

                    return t.Result;

                //or with a 409-conflict and return the list of missing parts

                //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception                

                if (t.IsFaulted)

                {

                    var ex = t.Exception.InnerException;

                    var we = ex as WebException;

                    var response = we.Response as HttpWebResponse;

                    if (response!=null && response.StatusCode==HttpStatusCode.Conflict)

                    {

                        //In case of 409 the missing parts will be in the response content                        

                        using (var stream = response.GetResponseStream())

                        using(var reader=new StreamReader(stream))

                        {

                            var content=reader.ReadToEnd();

                            return content;

                        }                        

                    }

                    else

                        //Any other status code is unexpected and the exception should be rethrown

                        throw ex;

                }

                //Any other status code is unexpected but there was no exception. We can probably continue processing

                else

                {

                    Trace.TraceWarning("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);                    

                }

                return t.Result;

            });

        }

        public Task<TreeHash> GetHashMap(string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName");

            if (String.IsNullOrWhiteSpace(Token))

                throw new InvalidOperationException("Invalid Token");

            if (StorageUrl == null)

                throw new InvalidOperationException("Invalid Storage Url");

            Contract.EndContractBlock();

            try

            {

                //The container and objectName are relative names. They are joined with the client's

                //BaseAddress to create the object's absolute address

                var builder = GetAddressBuilder(container, objectName);

                builder.Query="format=json&hashmap";

                var uri = builder.Uri;                

                //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient

                //object to avoid concurrency errors.

                //

                //Download operations take a long time therefore they have no timeout.

                //TODO: Do we really? this is a hashmap operation, not a download

                var client = new RestClient(_baseClient) { Timeout = 0 };

                //Start downloading the object asynchronously

                var downloadTask = client.DownloadStringTask(uri);

                //Once the download completes

                return downloadTask.ContinueWith(download =>

                {

                    //Delete the local client object

                    client.Dispose();

                    //And report failure or completion

                    if (download.IsFaulted)

                    {

                        Trace.TraceError("[GET HASH] FAIL for {0} with \r{1}", objectName,

                                        download.Exception);

                        throw download.Exception;

                    }

                    //The server will return an empty string if the file is empty

                    var json = download.Result;

                    var treeHash = TreeHash.Parse(json);

                    Trace.TraceInformation("[GET HASH] END {0}", objectName);                                             

                    return treeHash;

                });

            }

            catch (Exception exc)

            {

                Trace.TraceError("[GET HASH] END {0} with {1}", objectName, exc);

                throw;

            }

        }

        private UriBuilder GetAddressBuilder(string container, string objectName)

        {

            var builder = new UriBuilder(String.Join("/", _baseClient.BaseAddress, container, objectName));

            return builder;

        }

        /// <summary>

        /// 

        /// </summary>

        /// <param name="container"></param>

        /// <param name="objectName"></param>

        /// <param name="fileName"></param>

        /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>

        /// <remarks>>This method should have no timeout or a very long one</remarks>

        public Task PutObject(string container, string objectName, string fileName, string hash = null)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            if (String.IsNullOrWhiteSpace(fileName))

                throw new ArgumentNullException("fileName", "The fileName property can't be empty");

            if (!File.Exists(fileName))

                throw new FileNotFoundException("The file does not exist",fileName);

            try

            {

                var builder= GetAddressBuilder(container,objectName);

                var uri = builder.Uri;

                var client = new RestClient(_baseClient){Timeout=0};           

                string etag = hash ?? CalculateHash(fileName);

                client.Headers.Add("Content-Type", "application/octet-stream");

                client.Headers.Add("ETag", etag);

                Trace.TraceInformation("[PUT] START {0}", objectName);

                client.UploadProgressChanged += (sender, args) =>

                {

                    Trace.TraceInformation("[PUT PROGRESS] {0} {1}% {2} of {3}", fileName, args.ProgressPercentage, args.BytesSent, args.TotalBytesToSend);

                };

                return client.UploadFileTask(uri, "PUT", fileName)

                    .ContinueWith(upload=>

                                      {

                                          client.Dispose();

                                          if (upload.IsFaulted)

                                          {                                              

                                              Trace.TraceError("[PUT] FAIL for {0} with \r{1}",objectName,upload.Exception);

                                          }

                                          else

                                            Trace.TraceInformation("[PUT] END {0}", objectName);

                                      });

            }

            catch (Exception exc)

            {

                Trace.TraceError("[PUT] END {0} with {1}", objectName, exc);

                throw;

            }                

        }

        private static string CalculateHash(string fileName)

        {

            string hash;

            using (var hasher = MD5.Create())

            using(var stream=File.OpenRead(fileName))

            {

                var hashBuilder=new StringBuilder();

                foreach (byte b in hasher.ComputeHash(stream))

                    hashBuilder.Append(b.ToString("x2").ToLower());

                hash = hashBuilder.ToString();                

            }

            return hash;

        }

        public void DeleteObject(string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            using (var client = new RestClient(_baseClient))

            {

                client.DeleteWithRetry(container + "/" + objectName, 3);

                var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};

                if (!expectedCodes.Contains(client.StatusCode))

                    throw CreateWebException("DeleteObject", client.StatusCode);

            }

        }

        public void MoveObject(string sourceContainer, string oldObjectName, string targetContainer,string newObjectName)

        {

            if (String.IsNullOrWhiteSpace(sourceContainer))

                throw new ArgumentNullException("sourceContainer", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(oldObjectName))

                throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");

            if (String.IsNullOrWhiteSpace(targetContainer))

                throw new ArgumentNullException("targetContainer", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(newObjectName))

                throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");

            var targetUrl = targetContainer + "/" + newObjectName;

            var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);

            using (var client = new RestClient(_baseClient))

            {

                client.Headers.Add("X-Copy-From", sourceUrl);

                client.PutWithRetry(targetUrl, 3);

                var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};

                if (expectedCodes.Contains(client.StatusCode))

                {

                    this.DeleteObject(sourceContainer, oldObjectName);

                }

                else

                    throw CreateWebException("MoveObject", client.StatusCode);

            }

        }

        private static WebException CreateWebException(string operation, HttpStatusCode statusCode)

        {

            return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));

        }

    }

}