Synnefo » Pithos MS Client

using System;

using System.Collections.Generic;

using System.ComponentModel.Composition;

using System.Diagnostics;

using System.Diagnostics.Contracts;

using System.IO;

using System.Linq;

using System.Net;

using System.Security.Cryptography;

using System.Text;

using System.Threading.Tasks;

using Hammock;

using Hammock.Caching;

using Hammock.Retries;

using Hammock.Serialization;

using Hammock.Tasks;

using Hammock.Web;

using Newtonsoft.Json;

using Pithos.Interfaces;

namespace Pithos.Network

{

    [Export(typeof(ICloudClient))]

    public class CloudFilesClient:ICloudClient

    {

        string _rackSpaceAuthUrl = "https://auth.api.rackspacecloud.com";

        private string _pithosAuthUrl = "http://pithos.dev.grnet.gr";

        private RestClient _client;

        private readonly TimeSpan _shortTimeout = TimeSpan.FromSeconds(10);

        private readonly int _retries = 5;

        private RetryPolicy _retryPolicy;

        public string ApiKey { get; set; }

        public string UserName { get; set; }

        public Uri StorageUrl { get; set; }

        public string Token { get; set; }

        public Uri Proxy { get; set; }

        public double DownloadPercentLimit { get; set; }

        public double UploadPercentLimit { get; set; }

        public string AuthUrl

        {

            get { return UsePithos ? _pithosAuthUrl : _rackSpaceAuthUrl; }

        }

        public string VersionPath

        {

            get { return UsePithos ? "v1" : "v1.0"; }

        }

        public bool UsePithos { get; set; }

        public void Authenticate(string userName,string apiKey)

        {

            Trace.TraceInformation("[AUTHENTICATE] Start for {0}",userName);

            if (String.IsNullOrWhiteSpace(userName))

                throw new ArgumentNullException("userName","The userName property can't be empty");

            if (String.IsNullOrWhiteSpace(apiKey))

                throw new ArgumentNullException("apiKey", "The apiKey property can't be empty");

            UserName = userName;

            ApiKey = apiKey;

            string authUrl = UsePithos ? String.Format("{0}/{1}/{2}", AuthUrl, VersionPath,UserName) 

                                    : String.Format("{0}/{1}", AuthUrl, VersionPath);

            var proxy = Proxy != null ? Proxy.ToString():null;

            var authClient = new RestClient{Path=authUrl,Proxy=proxy};            

            authClient.AddHeader("X-Auth-User", UserName);

            authClient.AddHeader("X-Auth-Key", ApiKey);            

            var response=authClient.Request();

            ThrowIfNotStatusOK(response, "Authentication failed");

            var keys = response.Headers.AllKeys.AsQueryable();

            string storageUrl =UsePithos? 

                String.Format("{0}/{1}/{2}",AuthUrl,VersionPath,UserName)

                :GetHeaderValue("X-Storage-Url", response, keys);

            if (String.IsNullOrWhiteSpace(storageUrl))

                throw new InvalidOperationException("Failed to obtain storage url");

            StorageUrl = new Uri(storageUrl);

            if (!UsePithos)

            {

                var token = GetHeaderValue("X-Auth-Token", response, keys);

                if (String.IsNullOrWhiteSpace(token))

                    throw new InvalidOperationException("Failed to obtain token url");

                Token = token;

            }

            else

                Token = "0000";

            _retryPolicy = new RetryPolicy { RetryCount = _retries };

            _retryPolicy.RetryConditions.Add(new TimeoutRetryCondition());

            _client = new RestClient { Authority = StorageUrl.AbsoluteUri, Path = UserName, Proxy = proxy };

            _client.FileProgress += OnFileProgress;

            _client.AddHeader("X-Auth-Token", Token);

            if (UsePithos)

            {

                _client.AddHeader("X-Auth-User", UserName);

                _client.AddHeader("X-Auth-Key",ApiKey);                

            }

            Trace.TraceInformation("[AUTHENTICATE] End for {0}", userName);

        }

        private void OnFileProgress(object sender, FileProgressEventArgs e)

        {

            Trace.TraceInformation("[PROGRESS] {0} {1:p} {2} of {3}",e.FileName,(double)e.BytesWritten/e.TotalBytes, e.BytesWritten,e.TotalBytes);            

        }

        public IList<ContainerInfo> ListContainers()

        {                        

            //Workaround for Hammock quirk: Hammock always

            //appends a / unless a Path is specified.

            //Create a request with a complete path

            var request = new RestRequest { Path = StorageUrl.ToString(), RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            request.AddParameter("format","json");

            //Create a client clone

            var client = new RestClient{Proxy=Proxy.ToString()};

            foreach (var header in _client.GetAllHeaders())

            {

                client.AddHeader(header.Name,header.Value);

            }            

            var response = client.Request(request);

            if (response.StatusCode == HttpStatusCode.NoContent)

                return new List<ContainerInfo>();

            ThrowIfNotStatusOK(response, "List Containers failed");

            var infos=JsonConvert.DeserializeObject<IList<ContainerInfo>>(response.Content);

            return infos;

        }

        public IList<ObjectInfo> ListObjects(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            Trace.TraceInformation("[START] ListObjects");

            var request = new RestRequest { Path = container, RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) };

            request.AddParameter("format", "json");

            var response = _client.Request(request);

            var infos = InfosFromContent(response);

            Trace.TraceInformation("[END] ListObjects");

            return infos;

        }

        public IList<ObjectInfo> ListObjects(string container,string folder)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            Trace.TraceInformation("[START] ListObjects");

            var request = new RestRequest { Path = container,RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) };

            request.AddParameter("format", "json");

            request.AddParameter("path", folder);

            var response = _client.Request(request);

            var infos = InfosFromContent(response);

            Trace.TraceInformation("[END] ListObjects");

            return infos;

        }

        private static IList<ObjectInfo> InfosFromContent(RestResponse response)

        {

            if (response.TimedOut)

                return new List<ObjectInfo>();

            if (response.StatusCode == 0)

                return new List<ObjectInfo>();

            if (response.StatusCode == HttpStatusCode.NoContent)

                return new List<ObjectInfo>();

            var statusCode = (int)response.StatusCode;

            if (statusCode < 200 || statusCode >= 300)

            {

                Trace.TraceWarning("ListObjects failed with code {1} - {2}", response.StatusCode, response.StatusDescription);

                return new List<ObjectInfo>();

            }

            var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(response.Content);

            return infos;

        }

        public bool ContainerExists(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            switch(response.StatusCode)

            {

                case HttpStatusCode.NoContent:

                    return true;

                case HttpStatusCode.NotFound:

                    return false;                    

                default:

                    throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode));

            }

        }

        public bool ObjectExists(string container,string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head,RetryPolicy = _retryPolicy, Timeout = _shortTimeout };

            var response = _client.Request(request);

            switch (response.StatusCode)

            {

                case HttpStatusCode.OK:

                case HttpStatusCode.NoContent:

                    return true;

                case HttpStatusCode.NotFound:

                    return false;

                default:

                    throw new WebException(String.Format("ObjectExists failed with unexpected status code {0}", response.StatusCode));

            }

        }

        public ObjectInfo GetObjectInfo(string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            if (response.TimedOut)

                return ObjectInfo.Empty;

            switch (response.StatusCode)

            {

                case HttpStatusCode.OK:

                case HttpStatusCode.NoContent:

                    var keys = response.Headers.AllKeys.AsQueryable();

                    return new ObjectInfo

                               {

                                   Name = objectName,

                                   Bytes = long.Parse(GetHeaderValue("Content-Length", response, keys)),

                                   Hash = GetHeaderValue("ETag", response, keys),

                                   Content_Type = GetHeaderValue("Content-Type", response, keys)

                               };

                case HttpStatusCode.NotFound:

                    return ObjectInfo.Empty;

                default:

                    if (request.RetryState.RepeatCount > 0)

                    {

                        Trace.TraceWarning("[RETRY FAIL] GetObjectInfo for {0} failed after {1} retries",

                                                      objectName, request.RetryState.RepeatCount);

                        return ObjectInfo.Empty;

                    }

                    if (response.InnerException != null)

                        throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode), response.InnerException);

                    throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode));

            }

        }

        public void CreateFolder(string container, string folder)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(folder))

                throw new ArgumentNullException("folder", "The folder property can't be empty");

            var folderUrl=String.Format("{0}/{1}",container,folder);

            var request = new RestRequest { Path = folderUrl, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            request.AddHeader("Content-Type", @"application/directory");

            request.AddHeader("Content-Length", "0");

            var response = _client.Request(request);

            if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)

                throw new WebException(String.Format("CreateFolder failed with unexpected status code {0}", response.StatusCode));

        }

        public ContainerInfo GetContainerInfo(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            switch(response.StatusCode)

            {

                case HttpStatusCode.NoContent:

                    var keys = response.Headers.AllKeys.AsQueryable();

                    var containerInfo = new ContainerInfo

                                            {

                                                Name = container,

                                                Count =long.Parse(GetHeaderValue("X-Container-Object-Count", response, keys)),

                                                Bytes =long.Parse(GetHeaderValue("X-Container-Bytes-Used", response, keys))

                                            };

                    return containerInfo;

                case HttpStatusCode.NotFound:

                    return ContainerInfo.Empty;                    

                default:

                    throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode));

            }

        }

        public void CreateContainer(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            var request = new RestRequest { Path = container, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            if (response.StatusCode!=HttpStatusCode.Created && response.StatusCode!=HttpStatusCode.Accepted )

                    throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}", response.StatusCode));

        }

        public void DeleteContainer(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            var request = new RestRequest { Path = container, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent)

                return;

            else

                throw new WebException(String.Format("DeleteContainer failed with unexpected status code {0}", response.StatusCode));

        }

        /// <summary>

        /// 

        /// </summary>

        /// <param name="container"></param>

        /// <param name="objectName"></param>

        /// <returns></returns>

        /// <remarks>>This method should have no timeout or a very long one</remarks>

        public Stream GetObject(string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Get };

/*

            if (DownloadPercentLimit > 0)

                request.TaskOptions = new TaskOptions<int> { RateLimitPercent = DownloadPercentLimit };

*/

            var response = _client.Request(request);

            if (response.StatusCode == HttpStatusCode.NotFound)

                throw new FileNotFoundException();

            if (response.StatusCode == HttpStatusCode.OK)

            {

                return response.ContentStream;

            }

            else

                throw new WebException(String.Format("GetObject failed with unexpected status code {0}", response.StatusCode));

        }

        /// <summary>

        /// 

        /// </summary>

        /// <param name="container"></param>

        /// <param name="objectName"></param>

        /// <param name="fileName"></param>

        /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>

        /// <remarks>>This method should have no timeout or a very long one</remarks>

        public Task PutObject(string container, string objectName, string fileName, string hash = null)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            if (String.IsNullOrWhiteSpace(fileName))

                throw new ArgumentNullException("fileName", "The fileName property can't be empty");

            if (!File.Exists(fileName))

                throw new FileNotFoundException("The file does not exist",fileName);

            string url = container + "/" + objectName;

            var request = new RestRequest {Path=url,Method=WebMethod.Put};           

/*

            if(UploadPercentLimit>0)

                request.TaskOptions=new TaskOptions<int>{RateLimitPercent=UploadPercentLimit};

*/

            Trace.TraceInformation("[PUT] START {0}",objectName);

            string etag = hash??CalculateHash(fileName);

            request.AddFile(fileName, fileName, fileName);            

            //request.AddPostContent(File.ReadAllBytes(fileName));

            request.AddHeader("Content-Type","application/octet-stream");

            request.AddHeader("ETag", etag);

            //_client.TaskOptions = new TaskOptions<int> {RateLimitPercent = 0.5};

            try

            {

                var response=_client.Request(request);

                Trace.TraceInformation("[PUT] END {0}", objectName);

                if (response.StatusCode == HttpStatusCode.Created)

                    return Task.Factory.StartNew(()=>{});

                if (response.StatusCode == HttpStatusCode.LengthRequired)

                    throw new InvalidOperationException();

                else

                    throw new WebException(String.Format("GetObject failed with unexpected status code {0}",

                            response.StatusCode));

                /*            return Task.Factory.FromAsync(_client.BeginRequest(request),ar=>_client.EndRequest(ar))

                                .ContinueWith(t=>

                                {*/

            }

            catch (Exception exc)

            {

                Trace.TraceError("[PUT] END {0} with {1}", objectName, exc);

                throw;

            }                

/*

            var response = t.Result;

                    if (t.IsFaulted)

                        Trace.TraceError("[PUT] END {0} with {1}", objectName, t.Exception);

                    else

                    {

                        Trace.TraceInformation("[PUT] END {0}",objectName);

                    }

*/

                  /*  if (response.StatusCode == HttpStatusCode.Created)

                        return;

                    if (response.StatusCode == HttpStatusCode.LengthRequired)

                        throw new InvalidOperationException();

                    else

                        throw new WebException(String.Format("GetObject failed with unexpected status code {0}",

                                response.StatusCode));*/

                /*});*/

        }

        private static string CalculateHash(string fileName)

        {

            string hash;

            using (var hasher = MD5.Create())

            using(var stream=File.OpenRead(fileName))

            {

                var hashBuilder=new StringBuilder();

                foreach (byte b in hasher.ComputeHash(stream))

                    hashBuilder.Append(b.ToString("x2").ToLower());

                hash = hashBuilder.ToString();                

            }

            return hash;

        }

        public void DeleteObject(string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };

            var response = _client.Request(request);

            if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent)

                return;

            else

                throw new WebException(String.Format("DeleteObject failed with unexpected status code {0}", response.StatusCode));

        }

        public void MoveObject(string container, string oldObjectName, string newObjectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(oldObjectName))

                throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");

            if (String.IsNullOrWhiteSpace(newObjectName))

                throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");

            var request = new RestRequest { Path = container + "/" + newObjectName, Method = WebMethod.Put };

            request.AddHeader("X-Copy-From",String.Format("/{0}/{1}",container,oldObjectName));

            request.AddPostContent(new byte[]{});

            var response = _client.Request(request);

            if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.NoContent || response.StatusCode==HttpStatusCode.Created)

            {

                this.DeleteObject(container,oldObjectName);

            }                

            else

                throw new WebException(String.Format("MoveObject failed with unexpected status code {0}", response.StatusCode));

        }

        private string GetHeaderValue(string headerName, RestResponse response, IQueryable<string> keys)

        {

            if (keys.Any(key => key == headerName))

                return response.Headers[headerName];

            else

                throw new WebException(String.Format("The {0}  header is missing",headerName));

        }

        private static void ThrowIfNotStatusOK(RestResponse response, string message)

        {

            int status = (int)response.StatusCode;

            if (status < 200 || status >= 300)

                throw new WebException(String.Format("{0} with code {1}",message, status));

        }

    }

}