Synnefo » Pithos MS Client

using System;

using System.Collections.Generic;

using System.ComponentModel.Composition;

using System.Diagnostics;

using System.Diagnostics.Contracts;

using System.IO;

using System.Linq;

using System.Net;

using System.Security.Cryptography;

using System.Text;

using System.Threading.Tasks;

using Hammock;

using Hammock.Caching;

using Hammock.Retries;

using Hammock.Serialization;

using Hammock.Tasks;

using Hammock.Web;

using Newtonsoft.Json;

using Pithos.Interfaces;

namespace Pithos.Network

{

    [Export(typeof(ICloudClient))]

    public class CloudFilesClient:ICloudClient

    {

        string _rackSpaceAuthUrl = "https://auth.api.rackspacecloud.com";

        private string _pithosAuthUrl = "https://pithos.dev.grnet.gr";

        private RestClient _client;

        private readonly TimeSpan _shortTimeout = TimeSpan.FromSeconds(10);

        private readonly int _retries = 5;

        private RetryPolicy _retryPolicy;

        public string ApiKey { get; set; }

        public string UserName { get; set; }

        public Uri StorageUrl { get; set; }

        public string Token { get; set; }

        public Uri Proxy { get; set; }

        public double DownloadPercentLimit { get; set; }

        public double UploadPercentLimit { get; set; }

        public string AuthUrl

        {

            get { return UsePithos ? _pithosAuthUrl : _rackSpaceAuthUrl; }

        }

        public string VersionPath

        {

            get { return UsePithos ? "v1" : "v1.0"; }

        }

        public bool UsePithos { get; set; }

        private bool _authenticated = false;

        public void Authenticate(string userName,string apiKey)

        {

            Trace.TraceInformation("[AUTHENTICATE] Start for {0}", userName);

            if (String.IsNullOrWhiteSpace(userName))

                throw new ArgumentNullException("userName", "The userName property can't be empty");

            if (String.IsNullOrWhiteSpace(apiKey))

                throw new ArgumentNullException("apiKey", "The apiKey property can't be empty");

            if (_authenticated)

                return;

            UserName = userName;

            ApiKey = apiKey;

            var proxy = Proxy != null ? Proxy.ToString() : null;

            if (UsePithos)

            {

                Token = ApiKey;

                string storageUrl = String.Format("{0}/{1}/{2}", AuthUrl, VersionPath, UserName);

                StorageUrl = new Uri(storageUrl);

            }

            else

            {

                string authUrl = String.Format("{0}/{1}", AuthUrl, VersionPath);

                var authClient = new RestClient {Path = authUrl, Proxy = proxy};                

                authClient.AddHeader("X-Auth-User", UserName);

                authClient.AddHeader("X-Auth-Key", ApiKey);

                var response = authClient.Request();

                ThrowIfNotStatusOK(response, "Authentication failed");

                var keys = response.Headers.AllKeys.AsQueryable();

                string storageUrl = GetHeaderValue("X-Storage-Url", response, keys);

                if (String.IsNullOrWhiteSpace(storageUrl))

                    throw new InvalidOperationException("Failed to obtain storage url");

                StorageUrl = new Uri(storageUrl);

                var token = GetHeaderValue("X-Auth-Token", response, keys);

                if (String.IsNullOrWhiteSpace(token))

                    throw new InvalidOperationException("Failed to obtain token url");

                Token = token;

            }

            _retryPolicy = new RetryPolicy { RetryCount = _retries };

            _retryPolicy.RetryConditions.Add(new TimeoutRetryCondition());

            _client = new RestClient { Authority = StorageUrl.AbsoluteUri, Path = UserName, Proxy = proxy };

            _client.FileProgress += OnFileProgress;

            _client.AddHeader("X-Auth-Token", Token);

            /*if (UsePithos)

            {

                _client.AddHeader("X-Auth-User", UserName);

                _client.AddHeader("X-Auth-Key",ApiKey);                

            }*/

            Trace.TraceInformation("[AUTHENTICATE] End for {0}", userName);

        }

        private void OnFileProgress(object sender, FileProgressEventArgs e)

        {

            Trace.TraceInformation("[PROGRESS] {0} {1:p} {2} of {3}",e.FileName,(double)e.BytesWritten/e.TotalBytes, e.BytesWritten,e.TotalBytes);            

        }

        public IList<ContainerInfo> ListContainers()

        {                        

            //Workaround for Hammock quirk: Hammock always

            //appends a / unless a Path is specified.

            //Create a request with a complete path

            var request = new RestRequest { Path = StorageUrl.ToString(), RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            request.AddParameter("format","json");

            //Create a client clone

            var client = new RestClient{Proxy=Proxy.ToString()};

            foreach (var header in _client.GetAllHeaders())

            {

                client.AddHeader(header.Name,header.Value);

            }            

            var response = client.Request(request);

            if (response.StatusCode == HttpStatusCode.NoContent)

                return new List<ContainerInfo>();

            ThrowIfNotStatusOK(response, "List Containers failed");

            var infos=JsonConvert.DeserializeObject<IList<ContainerInfo>>(response.Content);

            return infos;

        }

        public IList<ObjectInfo> ListObjects(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            Trace.TraceInformation("[START] ListObjects");

            var request = new RestRequest { Path = container, RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) };

            request.AddParameter("format", "json");

            var response = _client.Request(request);

            var infos = InfosFromContent(response);

            Trace.TraceInformation("[END] ListObjects");

            return infos;

        }

        public IList<ObjectInfo> ListObjects(string container,string folder)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            Trace.TraceInformation("[START] ListObjects");

            var request = new RestRequest { Path = container,RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) };

            request.AddParameter("format", "json");

            request.AddParameter("path", folder);

            var response = _client.Request(request);

            var infos = InfosFromContent(response);

            Trace.TraceInformation("[END] ListObjects");

            return infos;

        }

        private static IList<ObjectInfo> InfosFromContent(RestResponse response)

        {

            if (response.TimedOut)

                return new List<ObjectInfo>();

            if (response.StatusCode == 0)

                return new List<ObjectInfo>();

            if (response.StatusCode == HttpStatusCode.NoContent)

                return new List<ObjectInfo>();

            var statusCode = (int)response.StatusCode;

            if (statusCode < 200 || statusCode >= 300)

            {

                Trace.TraceWarning("ListObjects failed with code {0} - {1}", response.StatusCode, response.StatusDescription);

                return new List<ObjectInfo>();

            }

            var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(response.Content);

            return infos;

        }

        public bool ContainerExists(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            switch(response.StatusCode)

            {

                case HttpStatusCode.OK:

                case HttpStatusCode.NoContent:

                    return true;

                case HttpStatusCode.NotFound:

                    return false;          

                default:

                    throw CreateWebException("ContainerExists",response.StatusCode);

            }

        }

        public bool ObjectExists(string container,string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head,RetryPolicy = _retryPolicy, Timeout = _shortTimeout };

            var response = _client.Request(request);

            switch (response.StatusCode)

            {

                case HttpStatusCode.OK:

                case HttpStatusCode.NoContent:

                    return true;

                case HttpStatusCode.NotFound:

                    return false;

                default:

                    throw CreateWebException("ObjectExists",response.StatusCode);

            }

        }

        public ObjectInfo GetObjectInfo(string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            if (response.TimedOut)

                return ObjectInfo.Empty;

            switch (response.StatusCode)

            {

                case HttpStatusCode.OK:

                case HttpStatusCode.NoContent:

                    var keys = response.Headers.AllKeys.AsQueryable();

                    var tags=(from key in keys

                             where key.StartsWith("X-Object-Meta-")

                             let name=key.Substring(14)

                             select new {Name=name,Value=response.Headers[name]})

                             .ToDictionary(t=>t.Name,t=>t.Value);

                    var extensions = (from key in keys

                                      where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")

                                      let name = key.Substring(9)

                                      select new { Name = name, Value = response.Headers[name] })

                             .ToDictionary(t => t.Name, t => t.Value);

                    return new ObjectInfo

                               {

                                   Name = objectName,

                                   Bytes = long.Parse(GetHeaderValue("Content-Length", response, keys)),

                                   Hash = GetHeaderValue("ETag", response, keys),

                                   Content_Type = GetHeaderValue("Content-Type", response, keys),

                                   Tags=tags,

                                   Extensions=extensions

                               };

                case HttpStatusCode.NotFound:

                    return ObjectInfo.Empty;

                default:

                    if (request.RetryState.RepeatCount > 0)

                    {

                        Trace.TraceWarning("[RETRY FAIL] GetObjectInfo for {0} failed after {1} retries",

                                                      objectName, request.RetryState.RepeatCount);

                        return ObjectInfo.Empty;

                    }

                    if (response.InnerException != null)

                        throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode), response.InnerException);

                    throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode));

            }

        }

        public void CreateFolder(string container, string folder)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(folder))

                throw new ArgumentNullException("folder", "The folder property can't be empty");

            var folderUrl=String.Format("{0}/{1}",container,folder);

            var request = new RestRequest { Path = folderUrl, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            request.AddHeader("Content-Type", @"application/directory");

            request.AddHeader("Content-Length", "0");

            var response = _client.Request(request);

            if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)

                throw CreateWebException("CreateFolder", response.StatusCode);

        }

        public ContainerInfo GetContainerInfo(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            switch(response.StatusCode)

            {

                case HttpStatusCode.NoContent:

                    var keys = response.Headers.AllKeys.AsQueryable();

                    var containerInfo = new ContainerInfo

                                            {

                                                Name = container,

                                                Count =long.Parse(GetHeaderValue("X-Container-Object-Count", response, keys)),

                                                Bytes =long.Parse(GetHeaderValue("X-Container-Bytes-Used", response, keys))

                                            };

                    return containerInfo;

                case HttpStatusCode.NotFound:

                    return ContainerInfo.Empty;                    

                default:

                    throw CreateWebException("GetContainerInfo", response.StatusCode);

            }

        }

        public void CreateContainer(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            var request = new RestRequest { Path = container, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            var expectedCodes = new[]{HttpStatusCode.Created ,HttpStatusCode.Accepted , HttpStatusCode.OK};

            if (!expectedCodes.Contains(response.StatusCode))

                throw CreateWebException("CreateContainer", response.StatusCode);

        }

        public void DeleteContainer(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            var request = new RestRequest { Path = container, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent};

            if (!expectedCodes.Contains(response.StatusCode))

                throw CreateWebException("DeleteContainer", response.StatusCode);

        }

        /// <summary>

        /// 

        /// </summary>

        /// <param name="container"></param>

        /// <param name="objectName"></param>

        /// <param name="fileName"></param>

        /// <returns></returns>

        /// <remarks>>This method should have no timeout or a very long one</remarks>

        public Task GetObject(string container, string objectName, string fileName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            var request = new RestRequest {Path = container + "/" + objectName, Method = WebMethod.Get};

            /*

                        if (DownloadPercentLimit > 0)

                            request.TaskOptions = new TaskOptions<int> { RateLimitPercent = DownloadPercentLimit };

            */

            try

            {

                var url = String.Join("/", new[] {_client.Authority, container, objectName});

                var uri = new Uri(url);

                var client = new WebClient();

                if (!String.IsNullOrWhiteSpace(_client.Proxy))

                    client.Proxy = new WebProxy(_client.Proxy);

                CopyHeaders(_client, client);

                Trace.TraceInformation("[GET] START {0}", objectName);

                client.DownloadProgressChanged += (sender, args) => 

                    Trace.TraceInformation("[GET PROGRESS] {0} {1}% {2} of {3}",

                                    fileName, args.ProgressPercentage,

                                    args.BytesReceived,

                                    args.TotalBytesToReceive);

                return client.DownloadFileTask(uri, fileName)

                    .ContinueWith(download =>

                                      {

                                          client.Dispose();

                                          if (download.IsFaulted)

                                          {

                                              Trace.TraceError("[GET] FAIL for {0} with \r{1}", objectName,

                                                               download.Exception);

                                          }

                                          else

                                          {

                                              Trace.TraceInformation("[GET] END {0}", objectName);                                             

                                          }

                                      });

            }

            catch (Exception exc)

            {

                Trace.TraceError("[GET] END {0} with {1}", objectName, exc);

                throw;

            }

        }

        /// <summary>

        /// 

        /// </summary>

        /// <param name="container"></param>

        /// <param name="objectName"></param>

        /// <param name="fileName"></param>

        /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>

        /// <remarks>>This method should have no timeout or a very long one</remarks>

        public Task PutObject(string container, string objectName, string fileName, string hash = null)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            if (String.IsNullOrWhiteSpace(fileName))

                throw new ArgumentNullException("fileName", "The fileName property can't be empty");

            if (!File.Exists(fileName))

                throw new FileNotFoundException("The file does not exist",fileName);

            try

            {

                var url = String.Join("/",new[]{_client.Authority,container,objectName});

                var uri = new Uri(url);

                var client = new WebClient();                

                string etag = hash ?? CalculateHash(fileName);

                client.Headers.Add("Content-Type", "application/octet-stream");

                client.Headers.Add("ETag", etag);

                if(!String.IsNullOrWhiteSpace(_client.Proxy))

                    client.Proxy = new WebProxy(_client.Proxy);

                CopyHeaders(_client, client);

                Trace.TraceInformation("[PUT] START {0}", objectName);

                client.UploadProgressChanged += (sender, args) =>

                {

                    Trace.TraceInformation("[PUT PROGRESS] {0} {1}% {2} of {3}", fileName, args.ProgressPercentage, args.BytesSent, args.TotalBytesToSend);

                };

                return client.UploadFileTask(uri, "PUT", fileName)

                    .ContinueWith(upload=>

                                      {

                                          client.Dispose();

                                          if (upload.IsFaulted)

                                          {                                              

                                              Trace.TraceError("[PUT] FAIL for {0} with \r{1}",objectName,upload.Exception);

                                          }

                                          else

                                            Trace.TraceInformation("[PUT] END {0}", objectName);

                                      });

            }

            catch (Exception exc)

            {

                Trace.TraceError("[PUT] END {0} with {1}", objectName, exc);

                throw;

            }                

        }

        /// <summary>

        /// Copies headers from a Hammock RestClient to a WebClient

        /// </summary>

        /// <param name="source">The RestClient from which the headers are copied</param>

        /// <param name="target">The WebClient to which the headers are copied</param>

        private static void CopyHeaders(RestClient source, WebClient target)

        {

            Contract.Requires(source!=null,"source can't be null");

            Contract.Requires(target != null, "target can't be null");

            if (source == null)

                throw new ArgumentNullException("source", "source can't be null");

            if (source == null)

                throw new ArgumentNullException("target", "target can't be null");

            foreach (var header in source.GetAllHeaders())

            {

                target.Headers.Add(header.Name, header.Value);

            }

        }

        private static string CalculateHash(string fileName)

        {

            string hash;

            using (var hasher = MD5.Create())

            using(var stream=File.OpenRead(fileName))

            {

                var hashBuilder=new StringBuilder();

                foreach (byte b in hasher.ComputeHash(stream))

                    hashBuilder.Append(b.ToString("x2").ToLower());

                hash = hashBuilder.ToString();                

            }

            return hash;

        }

        public void DeleteObject(string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent };

            if (!expectedCodes.Contains(response.StatusCode))

                throw CreateWebException("DeleteObject", response.StatusCode);

        }

        public void MoveObject(string sourceContainer, string oldObjectName, string targetContainer,string newObjectName)

        {

            if (String.IsNullOrWhiteSpace(sourceContainer))

                throw new ArgumentNullException("sourceContainer", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(oldObjectName))

                throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");

            if (String.IsNullOrWhiteSpace(targetContainer))

                throw new ArgumentNullException("targetContainer", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(newObjectName))

                throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");

            var targetUrl = targetContainer + "/" + newObjectName;

            var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);

            var request = new RestRequest { Path = targetUrl, Method = WebMethod.Put };

            request.AddHeader("X-Copy-From",sourceUrl);

            request.AddPostContent(new byte[]{});

            var response = _client.Request(request);

            var expectedCodes = new[] { HttpStatusCode.OK ,HttpStatusCode.NoContent ,HttpStatusCode.Created };

            if (expectedCodes.Contains(response.StatusCode))

            {

                this.DeleteObject(sourceContainer,oldObjectName);

            }                

            else

                throw CreateWebException("MoveObject", response.StatusCode);

        }

        private string GetHeaderValue(string headerName, RestResponse response, IQueryable<string> keys)

        {

            if (keys.Any(key => key == headerName))

                return response.Headers[headerName];

            else

                throw new WebException(String.Format("The {0}  header is missing",headerName));

        }

        private static void ThrowIfNotStatusOK(RestResponse response, string message)

        {

            int status = (int)response.StatusCode;

            if (status < 200 || status >= 300)

                throw new WebException(String.Format("{0} with code {1}",message, status));

        }

        private static WebException CreateWebException(string operation, HttpStatusCode statusCode)

        {

            return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));

        }

    }

}