Synnefo » Pithos MS Client

using System;

using System.Collections.Concurrent;

using System.Collections.Generic;

using System.ComponentModel.Composition;

using System.Diagnostics;

using System.Diagnostics.Contracts;

using System.IO;

using System.Linq;

using System.Net.NetworkInformation;

using System.Security.Cryptography;

using System.ServiceModel.Description;

using System.Text;

using System.Threading;

using System.Threading.Tasks;

using Castle.ActiveRecord.Queries;

using Microsoft.WindowsAPICodePack.Net;

using Pithos.Core.Agents;

using Pithos.Interfaces;

using System.ServiceModel;

using Pithos.Network;

using log4net;

namespace Pithos.Core

{

    [Export(typeof(PithosMonitor))]

    public class PithosMonitor:IDisposable

    {

        private int _blockSize;

        private string _blockHash;

        [Import]

        public IPithosSettings Settings{get;set;}

        [Import]

        public IStatusKeeper StatusKeeper { get; set; }

        [Import]

        public IPithosWorkflow Workflow { get; set; }

        public ICloudClient CloudClient { get; set; }

        public IStatusNotification StatusNotification { get; set; }

        [Import]

        public FileAgent FileAgent { get; set; }

        [Import]

        public WorkflowAgent WorkflowAgent { get; set; }

        [Import]

        public NetworkAgent NetworkAgent { get; set; }        

        public string UserName { get; set; }

        private string _apiKey;

        public string ApiKey

        {

            get { return _apiKey; }

            set

            {

                _apiKey = value;

                if (_accountInfo != null)

                    _accountInfo.Token = value;

            }

        }

        private AccountInfo _accountInfo;

        private static readonly ILog Log = LogManager.GetLogger(typeof(PithosMonitor));

        public bool Pause

        {

            get { return FileAgent.Pause; }

            set

            {

                FileAgent.Pause = value;

                if (value)

                {

                    StatusKeeper.SetPithosStatus(PithosStatus.SyncPaused);

                    StatusNotification.NotifyChange("Paused");

                }

                else

                {

                    StatusKeeper.SetPithosStatus(PithosStatus.InSynch);

                    StatusNotification.NotifyChange("Synchronizing");

                }

            }

        }

        private string _rootPath;

        public string RootPath

        {

            get { return _rootPath; }

            set 

            {

                _rootPath = String.IsNullOrWhiteSpace(value) 

                    ? String.Empty 

                    : value.ToLower();

            }

        }

        CancellationTokenSource _cancellationSource;

        private bool _started;

        public void Start()

        {            

            if (String.IsNullOrWhiteSpace(ApiKey))

                throw new InvalidOperationException("The ApiKey is empty");

            if (String.IsNullOrWhiteSpace(UserName))

                throw new InvalidOperationException("The UserName is empty");

            if (String.IsNullOrWhiteSpace(AuthenticationUrl))

                throw new InvalidOperationException("The Authentication url is empty");

            Contract.EndContractBlock();

            //If the account doesn't have a valid path, don't start monitoring but don't throw either

            if (String.IsNullOrWhiteSpace(RootPath))

                //TODO; Warn user?

                return;

            StatusNotification.NotifyChange("Starting");

            if (_started)

            {

                if (!_cancellationSource.IsCancellationRequested)

                    return;

            }

            _cancellationSource = new CancellationTokenSource();

            CloudClient=new CloudFilesClient(UserName,ApiKey);

            var proxyUri = ProxyFromSettings();            

            CloudClient.Proxy = proxyUri;

            CloudClient.UsePithos = this.UsePithos;

            CloudClient.AuthenticationUrl = this.AuthenticationUrl;            

            _accountInfo = CloudClient.Authenticate();

            _accountInfo.AccountPath = RootPath;

            var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);

            if (!Directory.Exists(pithosFolder))

                Directory.CreateDirectory(pithosFolder);

            //Create the cache folder and ensure it is hidden

            CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);

            var policy=CloudClient.GetAccountPolicies(_accountInfo);

            StatusNotification.NotifyAccount(policy);

            EnsurePithosContainers();

            StatusKeeper.BlockHash = _blockHash;

            StatusKeeper.BlockSize = _blockSize;

            StatusKeeper.StartProcessing(_cancellationSource.Token);

            IndexLocalFiles();

            StartWatcherAgent();

            StartNetworkAgent();

            StartWorkflowAgent();

            WorkflowAgent.RestartInterruptedFiles(_accountInfo);

            _started = true;

        }

        private void EnsurePithosContainers()

        {

            //Create the two default containers if they are missing

            var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};

            foreach (var container in pithosContainers)

            {                

                var info=CloudClient.GetContainerInfo(this.UserName, container);

                if (info == ContainerInfo.Empty)

                {

                    CloudClient.CreateContainer(this.UserName, container);

                    info = CloudClient.GetContainerInfo(this.UserName, container);

                }

                _blockSize = info.BlockSize;

                _blockHash = info.BlockHash;

                _accountInfo.BlockSize = _blockSize;

                _accountInfo.BlockHash = _blockHash;

            }

        }

        public string AuthenticationUrl { get; set; }

        private Uri ProxyFromSettings()

        {            

            if (Settings.UseManualProxy)

            {

                var proxyUri = new UriBuilder

                                   {

                                       Host = Settings.ProxyServer, 

                                       Port = Settings.ProxyPort

                                   };

                if (Settings.ProxyAuthentication)

                {

                    proxyUri.UserName = Settings.ProxyUsername;

                    proxyUri.Password = Settings.ProxyPassword;

                }

                return proxyUri.Uri;

            }

            return null;

        }

        private void IndexLocalFiles()

        {

            StatusNotification.NotifyChange("Indexing Local Files",TraceLevel.Info);

            using (log4net.ThreadContext.Stacks["Monitor"].Push("Indexing local files"))

            {

                Log.Info("START");

                try

                {

                    var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);

                    var directory = new DirectoryInfo(RootPath);

                    var files =

                        from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)

                        where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&

                              !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)

                        select file;

                    StatusKeeper.ProcessExistingFiles(files);

                }

                catch (Exception exc)

                {

                    Log.Error("[ERROR]", exc);

                }

                finally

                {

                    Log.Info("[END]");

                }

            }

        }

        private void StartWorkflowAgent()

        {

            bool connected = NetworkListManager.IsConnectedToInternet;

            //If we are not connected retry later

            if (!connected)

            {

                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);

                return;

            }

            try

            {

                WorkflowAgent.StatusNotification = StatusNotification;

                WorkflowAgent.Start();                

            }

            catch (Exception)

            {

                //Faild to authenticate due to network or account error

                //Retry after a while

                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);

            }

        }

        public bool UsePithos { get; set; }

        internal class LocalFileComparer:EqualityComparer<CloudAction>

        {

            public override bool Equals(CloudAction x, CloudAction y)

            {

                if (x.Action != y.Action)

                    return false;

                if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName))

                    return false;

                if (x.CloudFile != null && y.CloudFile != null )

                {

                    if (x.CloudFile.Hash == null & y.CloudFile.Hash != null)

                        return false;

                    if (x.CloudFile.Hash != null & y.CloudFile.Hash == null)

                        return false;

                    if (x.CloudFile.Hash == null & y.CloudFile.Hash == null)

                        return (x.CloudFile.Name == y.CloudFile.Name);

                    if (!x.CloudFile.Hash.Equals(y.CloudFile.Hash))

                        return false;

                }

                if (x.CloudFile == null ^ y.CloudFile == null ||

                    x.LocalFile == null ^ y.LocalFile == null)

                    return false;

                return true;

            }

            public override int GetHashCode(CloudAction obj)

            {

                var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode();

                var hash2 = (obj.CloudFile == null) ? int.MaxValue : (obj.CloudFile.Hash ?? obj.CloudFile.Name??"").GetHashCode();

                var hash3 = obj.Action.GetHashCode();

                return hash1 ^ hash2 & hash3;

            }

        }        

        private Timer timer;

        private void StartNetworkAgent()

        {

            NetworkAgent.AddAccount(_accountInfo);

            NetworkAgent.StatusNotification = StatusNotification;

            NetworkAgent.Start();

            NetworkAgent.ProcessRemoteFiles();

        }

        //Make sure a hidden cache folder exists to store partial downloads

        private static string CreateHiddenFolder(string rootPath, string folderName)

        {

            if (String.IsNullOrWhiteSpace(rootPath))

                throw new ArgumentNullException("rootPath");

            if (!Path.IsPathRooted(rootPath))

                throw new ArgumentException("rootPath");

            if (String.IsNullOrWhiteSpace(folderName))

                throw new ArgumentNullException("folderName");

            Contract.EndContractBlock();

            var folder = Path.Combine(rootPath, folderName);

            if (!Directory.Exists(folder))

            {

                var info = Directory.CreateDirectory(folder);

                info.Attributes |= FileAttributes.Hidden;

                Log.InfoFormat("Created cache Folder: {0}", folder);

            }

            else

            {

                var info = new DirectoryInfo(folder);

                if ((info.Attributes & FileAttributes.Hidden) == 0)

                {

                    info.Attributes |= FileAttributes.Hidden;

                    Log.InfoFormat("Reset cache folder to hidden: {0}", folder);

                }                                

            }

            return folder;

        }

        private void StartWatcherAgent()

        {

            AgentLocator<FileAgent>.Register(FileAgent,RootPath);

            FileAgent.StatusKeeper = StatusKeeper;

            FileAgent.Workflow = Workflow;

            FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);

            FileAgent.Start(_accountInfo, RootPath);

        }

        public void Stop()

        {

            AgentLocator<FileAgent>.Remove(RootPath);

            if (FileAgent!=null)

                FileAgent.Stop();

            FileAgent = null;

            if (timer != null)

                timer.Dispose();

            timer = null;            

        }

        ~PithosMonitor()

        {

            Dispose(false);

        }

        public void Dispose()

        {

            Dispose(true);

            GC.SuppressFinalize(this);

        }

        protected virtual void Dispose(bool disposing)

        {

            if (disposing)

            {

                Stop();

            }

        }

        public void MoveFileStates(string oldPath, string newPath)

        {

            if (String.IsNullOrWhiteSpace(oldPath))

                throw new ArgumentNullException("oldPath");

            if (!Path.IsPathRooted(oldPath))

                throw new ArgumentException("oldPath must be an absolute path","oldPath");

            if (string.IsNullOrWhiteSpace(newPath))

                throw new ArgumentNullException("newPath");

            if (!Path.IsPathRooted(newPath))

                throw new ArgumentException("newPath must be an absolute path","newPath");

            Contract.EndContractBlock();

            StatusKeeper.ChangeRoots(oldPath, newPath);

        }

        public void AddSelectivePaths(string[] added)

        {

           /* FileAgent.SelectivePaths.AddRange(added);

            NetworkAgent.SyncPaths(added);*/

        }

        public void RemoveSelectivePaths(string[] removed)

        {

            FileAgent.SelectivePaths.RemoveAll(removed.Contains);

            foreach (var removedPath in removed.Where(Directory.Exists))

            {

                Directory.Delete(removedPath,true);

            }

        }

        public IEnumerable<string> GetRootFolders()

        {

            var dirs = from container in CloudClient.ListContainers(UserName)

                       from dir in CloudClient.ListObjects(UserName, container.Name, "")

                       select dir.Name;

            return dirs;

        }

        public ObjectInfo GetObjectInfo(string filePath)

        {

            if (String.IsNullOrWhiteSpace(filePath))

                throw new ArgumentNullException("filePath");

            Contract.EndContractBlock();

            var file=new FileInfo(filePath);

            string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);

            var relativePath = file.AsRelativeTo(RootPath);

            string accountName,container;

            var parts=relativePath.Split('\\');

            var accountInfo = _accountInfo;

            if (relativePath.StartsWith(FolderConstants.OthersFolder))

            {                

                accountName = parts[1];

                container = parts[2];

                relativeUrl = String.Join("/", parts.Splice(3));

                //Create the root URL for the target account

                var oldName = UserName;

                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;

                var nameIndex=absoluteUri.IndexOf(oldName);

                var root=absoluteUri.Substring(0, nameIndex);

                accountInfo = new AccountInfo

                {

                    UserName = accountName,

                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),

                    StorageUri = new Uri(root + accountName),

                    BlockHash=accountInfo.BlockHash,

                    BlockSize=accountInfo.BlockSize,

                    Token=accountInfo.Token

                };

            }

            else

            {

                accountName = this.UserName;

                container = parts[0];

                relativeUrl = String.Join("/", parts.Splice(1));

            }

            var client = new CloudFilesClient(accountInfo);

            var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);

            return objectInfo;

        }

        public Task<ContainerInfo> GetContainerInfo(string filePath)

        {

            if (String.IsNullOrWhiteSpace(filePath))

                throw new ArgumentNullException("filePath");

            Contract.EndContractBlock();

            var file=new FileInfo(filePath);

            var relativePath = file.AsRelativeTo(RootPath);

            string accountName,container;

            var parts=relativePath.Split('\\');

            var accountInfo = _accountInfo;

            if (relativePath.StartsWith(FolderConstants.OthersFolder))

            {                

                accountName = parts[1];

                container = parts[2];                

                //Create the root URL for the target account

                var oldName = UserName;

                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;

                var nameIndex=absoluteUri.IndexOf(oldName);

                var root=absoluteUri.Substring(0, nameIndex);

                accountInfo = new AccountInfo

                {

                    UserName = accountName,

                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),

                    StorageUri = new Uri(root + accountName),

                    BlockHash=accountInfo.BlockHash,

                    BlockSize=accountInfo.BlockSize,

                    Token=accountInfo.Token

                };

            }

            else

            {

                accountName = UserName;

                container = parts[0];                

            }

            return Task.Factory.StartNew(() =>

            {

                var client = new CloudFilesClient(accountInfo);

                var containerInfo = client.GetContainerInfo(accountName, container);

                return containerInfo;

            });

        }

    }

    public interface IStatusNotification

    {        

        void NotifyChange(string status,TraceLevel level=TraceLevel.Info);

        void NotifyChangedFile(string filePath);

        void NotifyAccount(AccountInfo policy);

    }

}