Synnefo » Pithos MS Client

#region

/* -----------------------------------------------------------------------

 * <copyright file="PithosMonitor.cs" company="GRNet">

 * 

 * Copyright 2011-2012 GRNET S.A. All rights reserved.

 *

 * Redistribution and use in source and binary forms, with or

 * without modification, are permitted provided that the following

 * conditions are met:

 *

 *   1. Redistributions of source code must retain the above

 *      copyright notice, this list of conditions and the following

 *      disclaimer.

 *

 *   2. Redistributions in binary form must reproduce the above

 *      copyright notice, this list of conditions and the following

 *      disclaimer in the documentation and/or other materials

 *      provided with the distribution.

 *

 *

 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

 * POSSIBILITY OF SUCH DAMAGE.

 *

 * The views and conclusions contained in the software and

 * documentation are those of the authors and should not be

 * interpreted as representing official policies, either expressed

 * or implied, of GRNET S.A.

 * </copyright>

 * -----------------------------------------------------------------------

 */

#endregion

using System;

using System.Collections.Generic;

using System.ComponentModel.Composition;

using System.Diagnostics.Contracts;

using System.IO;

using System.Linq;

using System.Reflection;

using System.Threading;

using System.Threading.Tasks;

using Pithos.Core.Agents;

using Pithos.Interfaces;

using Pithos.Network;

using log4net;

namespace Pithos.Core

{

    [Export(typeof(PithosMonitor))]

    public class PithosMonitor:IDisposable

    {

        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);

        private int _blockSize;

        private string _blockHash;

        [Import]

        public IPithosSettings Settings{get;set;}

        private IStatusKeeper _statusKeeper;

        [Import]

        public IStatusKeeper StatusKeeper

        {

            get { return _statusKeeper; }

            set

            {

                _statusKeeper = value;

                FileAgent.StatusKeeper = value;

            }

        }

        private IPithosWorkflow _workflow;

        [Import]

        public IPithosWorkflow Workflow

        {

            get { return _workflow; }

            set

            {

                _workflow = value;

                FileAgent.Workflow = value;

            }

        }

        public ICloudClient CloudClient { get; set; }

        public IStatusNotification StatusNotification { get; set; }

        //[Import]

        public FileAgent FileAgent { get; private set; }

/*

        private WorkflowAgent _workflowAgent;

        [Import]

        public WorkflowAgent WorkflowAgent

        {

            get { return _workflowAgent; }

            set

            {

                _workflowAgent = value;

                //FileAgent.WorkflowAgent = value;

            }

        }

*/

        [Import]

        public NetworkAgent NetworkAgent { get; set; }

        private PollAgent _pollAgent;

        [Import]

        public PollAgent PollAgent

        {

            get { return _pollAgent; }

            set

            {

                _pollAgent = value;

                FileAgent.PollAgent = value;

            }

        }

        private Selectives _selectives;

        [Import]

        public Selectives Selectives

        {

            get { return _selectives; }

            set

            {

                _selectives = value;

                FileAgent.Selectives = value;

            }

        }

        public string UserName { get; set; }

        private string _apiKey;

        public string ApiKey

        {

            get { return _apiKey; }

            set

            {

                _apiKey = value;

                if (_accountInfo != null)

                    _accountInfo.Token = value;

            }

        }

        private AccountInfo _accountInfo;

        public AccountInfo Account

        {

            get { return _accountInfo; }

        }

        public bool Pause { get; set; }       

        /*public bool Pause

        {

            get { return FileAgent.Pause; }

            set

            {

                FileAgent.Pause = value;

            }

        }*/

        private string _rootPath;

        public string RootPath

        {

            get { return _rootPath; }

            set 

            {

                _rootPath = String.IsNullOrWhiteSpace(value) 

                    ? String.Empty 

                    : value.ToLower();

            }

        }

        CancellationTokenSource _cancellationSource;

        public PithosMonitor()

        {

            FileAgent = new FileAgent();            

        }

        private bool _started;

        public async Task Start()

        {            

            if (String.IsNullOrWhiteSpace(ApiKey))

                throw new InvalidOperationException("The ApiKey is empty");

            if (String.IsNullOrWhiteSpace(UserName))

                throw new InvalidOperationException("The UserName is empty");

            if (String.IsNullOrWhiteSpace(AuthenticationUrl))

                throw new InvalidOperationException("The Authentication url is empty");

            Contract.EndContractBlock();

            //If the account doesn't have a valid path, don't start monitoring but don't throw either

            if (String.IsNullOrWhiteSpace(RootPath))

                //TODO; Warn user?

                return;

            //WorkflowAgent.StatusNotification = StatusNotification;

            StatusNotification.NotifyChange("Starting");

            if (_started)

            {

                if (!_cancellationSource.IsCancellationRequested)

                    return;

            }

            _cancellationSource = new CancellationTokenSource();

            lock (this)

            {

                CloudClient = new CloudFilesClient(UserName, ApiKey)

                                  {UsePithos = true, AuthenticationUrl = AuthenticationUrl};

                _accountInfo = CloudClient.Authenticate();

            }

            _accountInfo.SiteUri = AuthenticationUrl;

            _accountInfo.AccountPath = RootPath;

            var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);

            if (!Directory.Exists(pithosFolder))

                Directory.CreateDirectory(pithosFolder);

            //Create the cache folder and ensure it is hidden

            CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);

            var policy=CloudClient.GetAccountPolicies(_accountInfo);

            StatusNotification.NotifyAccount(policy);

            EnsurePithosContainers();

            StatusKeeper.BlockHash = _blockHash;

            StatusKeeper.BlockSize = _blockSize;

            StatusKeeper.StartProcessing(_cancellationSource.Token);

            LoadSelectivePaths();

            await IndexLocalFiles();

            StartWatcherAgent();

            StartNetworkAgent();

            //WorkflowAgent.RestartInterruptedFiles(_accountInfo);

            _started = true;

        }

        private void LoadSelectivePaths()

        {

            var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey);

            var selectiveUrls = settings.SelectiveFolders.Cast<string>()

                .Select(url => new Uri(url, UriKind.RelativeOrAbsolute))

                .Where(uri => uri.IsAbsoluteUri).ToArray();

            SetSelectivePaths(selectiveUrls, null, null);

            CleanupUnselectedStates();

        }

        private void EnsurePithosContainers()

        {

            //Create the two default containers if they are missing

            var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};

            foreach (var container in pithosContainers)

            {                

                var info=CloudClient.GetContainerInfo(UserName, container);

                if (info == ContainerInfo.Empty)

                {

                    CloudClient.CreateContainer(UserName, container);

                    info = CloudClient.GetContainerInfo(UserName, container);

                }

                _blockSize = info.BlockSize;

                _blockHash = info.BlockHash;

                _accountInfo.BlockSize = _blockSize;

                _accountInfo.BlockHash = _blockHash;

            }

        }

        public string AuthenticationUrl { get; set; }

        private Task IndexLocalFiles()

        {            

            return TaskEx.Run(() =>

                           {

                               using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))

                               {

                                   try

                                   {

                                       //StatusNotification.NotifyChange("Indexing Local Files");

                                       Log.Info("Start local indexing");

                                       StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,

                                                                          "Indexing Local Files");

                                       var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);

                                       var directory = new DirectoryInfo(RootPath);

                                       var files =

                                           from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)

                                           where

                                               !file.FullName.StartsWith(cachePath,

                                                                         StringComparison.InvariantCultureIgnoreCase) &&

                                               !file.Extension.Equals("ignore",

                                                                      StringComparison.InvariantCultureIgnoreCase)

                                           select file;

                                       StatusKeeper.ProcessExistingFiles(files);

                                   }

                                   catch (Exception exc)

                                   {

                                       Log.Error("[ERROR]", exc);

                                   }

                                   finally

                                   {

                                       Log.Info("[END]");

                                   }

                                   StatusNotification.SetPithosStatus(PithosStatus.LocalComplete, "Indexing Completed");

                               }

                           });

        }

       /* private void StartWorkflowAgent()

        {

            WorkflowAgent.StatusNotification = StatusNotification;

/*            //On Vista and up we can check for a network connection

            bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;

            //If we are not connected retry later

            if (!connected)

            {

                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);

                return;

            }#1#

            try

            {

                WorkflowAgent.Start();                

            }

            catch (Exception)

            {

                //Faild to authenticate due to network or account error

                //Retry after a while

                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);

            }

        }*/

        private void StartNetworkAgent()

        {

            NetworkAgent.StatusNotification = StatusNotification;

            //TODO: The Network and Poll agents are not account specific

            //They should be moved outside PithosMonitor

/*

            NetworkAgent.Start();

*/

            PollAgent.AddAccount(_accountInfo);

            PollAgent.StatusNotification = StatusNotification;

            TaskEx.Run(()=> PollAgent.PollRemoteFiles());

        }

        //Make sure a hidden cache folder exists to store partial downloads

        private static void CreateHiddenFolder(string rootPath, string folderName)

        {

            if (String.IsNullOrWhiteSpace(rootPath))

                throw new ArgumentNullException("rootPath");

            if (!Path.IsPathRooted(rootPath))

                throw new ArgumentException("rootPath");

            if (String.IsNullOrWhiteSpace(folderName))

                throw new ArgumentNullException("folderName");

            Contract.EndContractBlock();

            var folder = Path.Combine(rootPath, folderName);

            if (!Directory.Exists(folder))

            {

                var info = Directory.CreateDirectory(folder);

                info.Attributes |= FileAttributes.Hidden;

                Log.InfoFormat("Created cache Folder: {0}", folder);

            }

            else

            {

                var info = new DirectoryInfo(folder);

                if ((info.Attributes & FileAttributes.Hidden) == 0)

                {

                    info.Attributes |= FileAttributes.Hidden;

                    Log.InfoFormat("Reset cache folder to hidden: {0}", folder);

                }                                

            }

        }

        private void StartWatcherAgent()

        {

            if (Log.IsDebugEnabled)

                Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);

            AgentLocator<FileAgent>.Register(FileAgent,RootPath);

            FileAgent.IdleTimeout = Settings.FileIdleTimeout;

            FileAgent.StatusKeeper = StatusKeeper;

            FileAgent.StatusNotification = StatusNotification;

            FileAgent.Workflow = Workflow;

            FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);

            //Ensure this starts in a different thread than the initiating UI thread

            TaskEx.Run(()=>FileAgent.Start(_accountInfo, RootPath));

        }

        public void Stop()

        {

/*

            AgentLocator<FileAgent>.Remove(RootPath);

            if (FileAgent!=null)

                FileAgent.Stop();

            FileAgent = null;

*/

        }

        ~PithosMonitor()

        {

            Dispose(false);

        }

        public void Dispose()

        {

            Dispose(true);

            GC.SuppressFinalize(this);

        }

        protected virtual void Dispose(bool disposing)

        {

            if (disposing)

            {

                Stop();

            }

        }

        public void MoveFileStates(string oldPath, string newPath)

        {

            if (String.IsNullOrWhiteSpace(oldPath))

                throw new ArgumentNullException("oldPath");

            if (!Path.IsPathRooted(oldPath))

                throw new ArgumentException("oldPath must be an absolute path","oldPath");

            if (string.IsNullOrWhiteSpace(newPath))

                throw new ArgumentNullException("newPath");

            if (!Path.IsPathRooted(newPath))

                throw new ArgumentException("newPath must be an absolute path","newPath");

            Contract.EndContractBlock();

            StatusKeeper.ChangeRoots(oldPath, newPath);

        }

        private void CleanupUnselectedStates()

        {

            //var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey);

            if (!Selectives.IsSelectiveEnabled(_accountInfo.AccountKey)) return;

            List<string> selectivePaths;

            if (Selectives.SelectivePaths.TryGetValue(_accountInfo.AccountKey, out selectivePaths))

            {

                var statePaths= FileState.Queryable.Select(state => state.FilePath).ToList();

                var removedPaths = statePaths

                    .Where(sp => !selectivePaths.Any(sp.IsAtOrBelow))

                    .ToList();

                UnversionSelectivePaths(removedPaths);

            }

            else

            {

                StatusKeeper.ClearFolderStatus(Account.AccountPath);

            }

        }

        public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)

        {

            //Convert the uris to paths

            //var selectivePaths = UrisToFilePaths(uris);

            var selectiveUri = uris.ToList();

            this.Selectives.SetSelectedUris(_accountInfo,selectiveUri);

            var removedPaths = UrisToFilePaths(removed);

            UnversionSelectivePaths(removedPaths);

        }

        /// <summary>

        /// Mark all unselected paths as Unversioned

        /// </summary>

        /// <param name="removed"></param>

        private void UnversionSelectivePaths(List<string> removed)

        {

            if (removed == null)

                return;

            //Ensure we remove any file state below the deleted folders

            FileState.UnversionPaths(removed);

        }

        /// <summary>

        /// Return a list of absolute filepaths from a list of Uris

        /// </summary>

        /// <param name="uris"></param>

        /// <returns></returns>

        public List<string> UrisToFilePaths(IEnumerable<Uri> uris)

        {

            if (uris == null)

                return new List<string>();

            var own = (from uri in uris

                       where uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())

                                   let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()

                                   //Trim the account name

                                   select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();

            var others= (from uri in uris

                         where !uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())

                                   let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()

                                   //Trim the account name

                                   select Path.Combine(RootPath,"others-shared", relativePath)).ToList();

            return own.Union(others).ToList();            

        }

        public ObjectInfo GetObjectInfo(string filePath)

        {

            if (String.IsNullOrWhiteSpace(filePath))

                throw new ArgumentNullException("filePath");

            Contract.EndContractBlock();

            var file=new FileInfo(filePath);

            string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);

            var relativePath = file.AsRelativeTo(RootPath);

            string accountName,container;

            var parts=relativePath.Split('\\');

            var accountInfo = _accountInfo;

            if (relativePath.StartsWith(FolderConstants.OthersFolder))

            {                

                accountName = parts[1];

                container = parts[2];

                relativeUrl = String.Join("/", parts.Splice(3));

                //Create the root URL for the target account

                var oldName = UserName;

                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;

                var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);

                var root=absoluteUri.Substring(0, nameIndex);

                accountInfo = new AccountInfo

                {

                    UserName = accountName,

                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),

                    StorageUri = new Uri(root + accountName),

                    BlockHash=accountInfo.BlockHash,

                    BlockSize=accountInfo.BlockSize,

                    Token=accountInfo.Token

                };

            }

            else

            {

                accountName = UserName;

                container = parts[0];

                relativeUrl = String.Join("/", parts.Splice(1));

            }

            var client = new CloudFilesClient(accountInfo);

            var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);

            return objectInfo;

        }

        public Task<ContainerInfo> GetContainerInfo(string filePath)

        {

            if (String.IsNullOrWhiteSpace(filePath))

                throw new ArgumentNullException("filePath");

            Contract.EndContractBlock();

            var file=new FileInfo(filePath);

            var relativePath = file.AsRelativeTo(RootPath);

            string accountName,container;

            var parts=relativePath.Split('\\');

            var accountInfo = _accountInfo;

            if (relativePath.StartsWith(FolderConstants.OthersFolder))

            {                

                accountName = parts[1];

                container = parts[2];                

                //Create the root URL for the target account

                var oldName = UserName;

                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;

                var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);

                var root=absoluteUri.Substring(0, nameIndex);

                accountInfo = new AccountInfo

                {

                    UserName = accountName,

                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),

                    StorageUri = new Uri(root + accountName),

                    BlockHash=accountInfo.BlockHash,

                    BlockSize=accountInfo.BlockSize,

                    Token=accountInfo.Token

                };

            }

            else

            {

                accountName = UserName;

                container = parts[0];                

            }

            return Task.Factory.StartNew(() =>

            {

                var client = new CloudFilesClient(accountInfo);

                var containerInfo = client.GetContainerInfo(accountName, container);

                return containerInfo;

            });

        }

    }

}