Synnefo » Pithos MS Client

using System;

using System.Collections.Generic;

using System.ComponentModel.Composition;

using System.Diagnostics;

using System.Diagnostics.Contracts;

using System.IO;

using System.Linq;

using System.Net;

using System.Security.Cryptography;

using System.Text;

using System.Threading.Algorithms;

using System.Threading.Tasks;

using Newtonsoft.Json;

using Pithos.Interfaces;

using WebHeaderCollection = System.Net.WebHeaderCollection;

namespace Pithos.Network

{

    [Export(typeof(ICloudClient))]

    public class CloudFilesClient:ICloudClient

    {

        private PithosClient _client;

        private readonly TimeSpan _shortTimeout = TimeSpan.FromSeconds(10);

        private readonly int _retries = 5;        

        public string ApiKey { get; set; }

        public string UserName { get; set; }

        public Uri StorageUrl { get; set; }

        public string Token { get; set; }

        public Uri Proxy { get; set; }

        public double DownloadPercentLimit { get; set; }

        public double UploadPercentLimit { get; set; }

        public string AuthenticationUrl { get; set; }

        public string VersionPath

        {

            get { return UsePithos ? "v1" : "v1.0"; }

        }

        public bool UsePithos { get; set; }

        private bool _authenticated = false;

        public void Authenticate(string userName,string apiKey)

        {

            Trace.TraceInformation("[AUTHENTICATE] Start for {0}", userName);

            if (String.IsNullOrWhiteSpace(userName))

                throw new ArgumentNullException("userName", "The userName property can't be empty");

            if (String.IsNullOrWhiteSpace(apiKey))

                throw new ArgumentNullException("apiKey", "The apiKey property can't be empty");

            if (_authenticated)

                return;

            UserName = userName;

            ApiKey = apiKey;

            if (UsePithos)

            {

                Token = ApiKey;

                string storageUrl = String.Format("{0}/{1}/{2}", AuthenticationUrl, VersionPath, UserName);

                StorageUrl = new Uri(storageUrl);

            }

            else

            {

                string authUrl = String.Format("{0}/{1}", AuthenticationUrl, VersionPath);

                var authClient = new PithosClient{BaseAddress= authUrl};

                if (Proxy != null)

                    authClient.Proxy = new WebProxy(Proxy);

                authClient.Headers.Add("X-Auth-User", UserName);

                authClient.Headers.Add("X-Auth-Key", ApiKey);

                var response = authClient.DownloadStringWithRetry("",3);

                authClient.AssertStatusOK("Authentication failed");

                string storageUrl = authClient.GetHeaderValue("X-Storage-Url");

                if (String.IsNullOrWhiteSpace(storageUrl))

                    throw new InvalidOperationException("Failed to obtain storage url");

                StorageUrl = new Uri(storageUrl);

                var token = authClient.GetHeaderValue("X-Auth-Token");

                if (String.IsNullOrWhiteSpace(token))

                    throw new InvalidOperationException("Failed to obtain token url");

                Token = token;

            }

            _client = new PithosClient{

                BaseAddress  = StorageUrl.AbsoluteUri,                

                Timeout=10000,

                Retries=3};

            if (Proxy!=null)

                _client.Proxy = new WebProxy(Proxy);

            _client.Headers.Add("X-Auth-Token", Token);

            Trace.TraceInformation("[AUTHENTICATE] End for {0}", userName);

        }

        public IList<ContainerInfo> ListContainers()

        {                        

            var content=_client.DownloadStringWithRetry("",3);

            _client.Parameters.Clear();

            _client.Parameters.Add("format", "json");

            _client.AssertStatusOK("List Containers failed");

            if (_client.StatusCode==HttpStatusCode.NoContent)

                return new List<ContainerInfo>();

            var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);

            return infos;

        }

        public IList<ObjectInfo> ListObjects(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            Trace.TraceInformation("[START] ListObjects");

            _client.Parameters.Clear();

            _client.Parameters.Add("format", "json");

            var content = _client.DownloadStringWithRetry(container, 3);

            _client.AssertStatusOK("ListObjects failed");

            var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);

            Trace.TraceInformation("[END] ListObjects");

            return infos;

        }

        public IList<ObjectInfo> ListObjects(string container,string folder)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            Trace.TraceInformation("[START] ListObjects");

            _client.Parameters.Clear();

            _client.Parameters.Add("format", "json");

            _client.Parameters.Add("path", folder);

            var content = _client.DownloadStringWithRetry(container, 3);

            _client.AssertStatusOK("ListObjects failed");

            var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);

            Trace.TraceInformation("[END] ListObjects");

            return infos;

        }

        public bool ContainerExists(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            _client.Parameters.Clear();

            _client.Head(container,3);

            switch (_client.StatusCode)

            {

                case HttpStatusCode.OK:

                case HttpStatusCode.NoContent:

                    return true;

                case HttpStatusCode.NotFound:

                    return false;          

                default:

                    throw CreateWebException("ContainerExists", _client.StatusCode);

            }

        }

        public bool ObjectExists(string container,string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            _client.Parameters.Clear();

            _client.Head(container + "/" + objectName, 3);

            switch (_client.StatusCode)

            {

                case HttpStatusCode.OK:

                case HttpStatusCode.NoContent:

                    return true;

                case HttpStatusCode.NotFound:

                    return false;

                default:

                    throw CreateWebException("ObjectExists", _client.StatusCode);

            }

        }

        public ObjectInfo GetObjectInfo(string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            try

            {

                _client.Parameters.Clear();

                _client.Head(container + "/" + objectName, 3);

                if (_client.TimedOut)

                    return ObjectInfo.Empty;

                switch (_client.StatusCode)

                {

                    case HttpStatusCode.OK:

                    case HttpStatusCode.NoContent:

                        var keys = _client.ResponseHeaders.AllKeys.AsQueryable();

                        var tags = (from key in keys

                                    where key.StartsWith("X-Object-Meta-")

                                    let name = key.Substring(14)

                                    select new { Name = name, Value = _client.ResponseHeaders[name] })

                            .ToDictionary(t => t.Name, t => t.Value);

                        var extensions = (from key in keys

                                          where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")

                                          let name = key.Substring(9)

                                          select new { Name = name, Value = _client.ResponseHeaders[name] })

                            .ToDictionary(t => t.Name, t => t.Value);

                        return new ObjectInfo

                                   {

                                       Name = objectName,

                                       Bytes =

                                           long.Parse(_client.GetHeaderValue("Content-Length")),

                                       Hash = _client.GetHeaderValue("ETag"),

                                       Content_Type = _client.GetHeaderValue("Content-Type"),

                                       Tags = tags,

                                       Extensions = extensions

                                   };

                    case HttpStatusCode.NotFound:

                        return ObjectInfo.Empty;

                    default:

                        throw new WebException(

                            String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",

                                          objectName, _client.StatusCode));

                }

            }

            catch (RetryException e)

            {

                Trace.TraceWarning("[RETRY FAIL] GetObjectInfo for {0} failed.");

                return ObjectInfo.Empty;

            }

            catch (WebException e)

            {                

                Trace.TraceError(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",

                                              objectName, _client.StatusCode), e);

                throw;

            }

        }

        public void CreateFolder(string container, string folder)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(folder))

                throw new ArgumentNullException("folder", "The folder property can't be empty");

            var folderUrl=String.Format("{0}/{1}",container,folder);

            _client.Parameters.Clear();

            _client.Headers.Add("Content-Type", @"application/directory");

            _client.Headers.Add("Content-Length", "0");

            _client.PutWithRetry(folderUrl,3);

            if (_client.StatusCode != HttpStatusCode.Created && _client.StatusCode != HttpStatusCode.Accepted)

                throw CreateWebException("CreateFolder", _client.StatusCode);

        }

        public ContainerInfo GetContainerInfo(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            _client.Head(container);

            switch (_client.StatusCode)

            {

                case HttpStatusCode.NoContent:

                    var containerInfo = new ContainerInfo

                                            {

                                                Name = container,

                                                Count =long.Parse(_client.GetHeaderValue("X-Container-Object-Count")),

                                                Bytes = long.Parse(_client.GetHeaderValue("X-Container-Bytes-Used"))

                                            };

                    return containerInfo;

                case HttpStatusCode.NotFound:

                    return ContainerInfo.Empty;                    

                default:

                    throw CreateWebException("GetContainerInfo", _client.StatusCode);

            }

        }

        public void CreateContainer(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            _client.PutWithRetry(container,3);

            var expectedCodes = new[]{HttpStatusCode.Created ,HttpStatusCode.Accepted , HttpStatusCode.OK};

            if (!expectedCodes.Contains(_client.StatusCode))

                throw CreateWebException("CreateContainer", _client.StatusCode);

        }

        public void DeleteContainer(string container)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            _client.DeleteWithRetry(container,3);

            var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent};

            if (!expectedCodes.Contains(_client.StatusCode))

                throw CreateWebException("DeleteContainer", _client.StatusCode);

        }

        /// <summary>

        /// 

        /// </summary>

        /// <param name="container"></param>

        /// <param name="objectName"></param>

        /// <param name="fileName"></param>

        /// <returns></returns>

        /// <remarks>>This method should have no timeout or a very long one</remarks>

        public Task GetObject(string container, string objectName, string fileName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            try

            {

                var url = String.Join("/", _client.BaseAddress, container, objectName);

                var uri = new Uri(url);

                var client = new PithosClient(_client){Timeout=0};

                Trace.TraceInformation("[GET] START {0}", objectName);

                client.DownloadProgressChanged += (sender, args) => 

                    Trace.TraceInformation("[GET PROGRESS] {0} {1}% {2} of {3}",

                                    fileName, args.ProgressPercentage,

                                    args.BytesReceived,

                                    args.TotalBytesToReceive);

                return _client.DownloadFileTask(uri, fileName)

                    .ContinueWith(download =>

                                      {

                                          client.Dispose();

                                          if (download.IsFaulted)

                                          {

                                              Trace.TraceError("[GET] FAIL for {0} with \r{1}", objectName,

                                                               download.Exception);

                                          }

                                          else

                                          {

                                              Trace.TraceInformation("[GET] END {0}", objectName);                                             

                                          }

                                      });

            }

            catch (Exception exc)

            {

                Trace.TraceError("[GET] END {0} with {1}", objectName, exc);

                throw;

            }

        }

        /// <summary>

        /// 

        /// </summary>

        /// <param name="container"></param>

        /// <param name="objectName"></param>

        /// <param name="fileName"></param>

        /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>

        /// <remarks>>This method should have no timeout or a very long one</remarks>

        public Task PutObject(string container, string objectName, string fileName, string hash = null)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            if (String.IsNullOrWhiteSpace(fileName))

                throw new ArgumentNullException("fileName", "The fileName property can't be empty");

            if (!File.Exists(fileName))

                throw new FileNotFoundException("The file does not exist",fileName);

            try

            {

                var url = String.Join("/",_client.BaseAddress,container,objectName);

                var uri = new Uri(url);

                var client = new PithosClient(_client){Timeout=0};           

                string etag = hash ?? CalculateHash(fileName);

                client.Headers.Add("Content-Type", "application/octet-stream");

                client.Headers.Add("ETag", etag);

                Trace.TraceInformation("[PUT] START {0}", objectName);

                client.UploadProgressChanged += (sender, args) =>

                {

                    Trace.TraceInformation("[PUT PROGRESS] {0} {1}% {2} of {3}", fileName, args.ProgressPercentage, args.BytesSent, args.TotalBytesToSend);

                };

                return client.UploadFileTask(uri, "PUT", fileName)

                    .ContinueWith(upload=>

                                      {

                                          client.Dispose();

                                          if (upload.IsFaulted)

                                          {                                              

                                              Trace.TraceError("[PUT] FAIL for {0} with \r{1}",objectName,upload.Exception);

                                          }

                                          else

                                            Trace.TraceInformation("[PUT] END {0}", objectName);

                                      });

            }

            catch (Exception exc)

            {

                Trace.TraceError("[PUT] END {0} with {1}", objectName, exc);

                throw;

            }                

        }

        private static string CalculateHash(string fileName)

        {

            string hash;

            using (var hasher = MD5.Create())

            using(var stream=File.OpenRead(fileName))

            {

                var hashBuilder=new StringBuilder();

                foreach (byte b in hasher.ComputeHash(stream))

                    hashBuilder.Append(b.ToString("x2").ToLower());

                hash = hashBuilder.ToString();                

            }

            return hash;

        }

        public void DeleteObject(string container, string objectName)

        {

            if (String.IsNullOrWhiteSpace(container))

                throw new ArgumentNullException("container", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(objectName))

                throw new ArgumentNullException("objectName", "The objectName property can't be empty");

            _client.DeleteWithRetry(container + "/" + objectName,3);

            var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent };

            if (!expectedCodes.Contains(_client.StatusCode))

                throw CreateWebException("DeleteObject", _client.StatusCode);

        }

        public void MoveObject(string sourceContainer, string oldObjectName, string targetContainer,string newObjectName)

        {

            if (String.IsNullOrWhiteSpace(sourceContainer))

                throw new ArgumentNullException("sourceContainer", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(oldObjectName))

                throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");

            if (String.IsNullOrWhiteSpace(targetContainer))

                throw new ArgumentNullException("targetContainer", "The container property can't be empty");

            if (String.IsNullOrWhiteSpace(newObjectName))

                throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");

            var targetUrl = targetContainer + "/" + newObjectName;

            var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);

            var client = new PithosClient(_client);

            client.Headers.Add("X-Copy-From", sourceUrl);

            client.PutWithRetry(targetUrl,3);

            var expectedCodes = new[] { HttpStatusCode.OK ,HttpStatusCode.NoContent ,HttpStatusCode.Created };

            if (expectedCodes.Contains(client.StatusCode))

            {

                this.DeleteObject(sourceContainer,oldObjectName);

            }                

            else

                throw CreateWebException("MoveObject", client.StatusCode);

        }

        private static WebException CreateWebException(string operation, HttpStatusCode statusCode)

        {

            return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));

        }

    }

}