Selective filtering modifications to allow uploading of new root folders

[pithos-ms-client] / trunk / Pithos.Core / PithosMonitor.cs

#region
/* -----------------------------------------------------------------------
 * <copyright file="PithosMonitor.cs" company="GRNet">
 * 
 * Copyright 2011-2012 GRNET S.A. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 *   1. Redistributions of source code must retain the above
 *      copyright notice, this list of conditions and the following
 *      disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above
 *      copyright notice, this list of conditions and the following
 *      disclaimer in the documentation and/or other materials
 *      provided with the distribution.
 *
 *
 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and
 * documentation are those of the authors and should not be
 * interpreted as representing official policies, either expressed
 * or implied, of GRNET S.A.
 * </copyright>
 * -----------------------------------------------------------------------
 */
#endregion
using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Diagnostics.Contracts;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Pithos.Core.Agents;
using Pithos.Interfaces;
using Pithos.Network;
using log4net;

namespace Pithos.Core
{
    [Export(typeof(PithosMonitor))]
    public class PithosMonitor:IDisposable
    {
        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);

        private int _blockSize;
        private string _blockHash;

        [Import]
        public IPithosSettings Settings{get;set;}

        private IStatusKeeper _statusKeeper;

        [Import]
        public IStatusKeeper StatusKeeper
        {
            get { return _statusKeeper; }
            set
            {
                _statusKeeper = value;
                FileAgent.StatusKeeper = value;
            }
        }




        private IPithosWorkflow _workflow;

        [Import]
        public IPithosWorkflow Workflow
        {
            get { return _workflow; }
            set
            {
                _workflow = value;
                FileAgent.Workflow = value;
            }
        }

        public ICloudClient CloudClient { get; set; }

        public IStatusNotification StatusNotification { get; set; }

        //[Import]
        public FileAgent FileAgent { get; private set; }

/*
        private WorkflowAgent _workflowAgent;

        [Import]
        public WorkflowAgent WorkflowAgent
        {
            get { return _workflowAgent; }
            set
            {
                _workflowAgent = value;
                //FileAgent.WorkflowAgent = value;
            }
        }
*/
        
        [Import]
        public NetworkAgent NetworkAgent { get; set; }

        private PollAgent _pollAgent;

        [Import]
        public PollAgent PollAgent
        {
            get { return _pollAgent; }
            set
            {
                _pollAgent = value;
                FileAgent.PollAgent = value;
            }
        }

        private Selectives _selectives;

        [Import]
        public Selectives Selectives
        {
            get { return _selectives; }
            set
            {
                _selectives = value;
                FileAgent.Selectives = value;
            }
        }

        public string UserName { get; set; }
        private string _apiKey;
        public string ApiKey
        {
            get { return _apiKey; }
            set
            {
                _apiKey = value;
                if (_accountInfo != null)
                    _accountInfo.Token = value;
            }
        }

        private AccountInfo _accountInfo;

        public AccountInfo Account
        {
            get { return _accountInfo; }
        }





        public bool Pause { get; set; }       
        /*public bool Pause
        {
            get { return FileAgent.Pause; }
            set
            {
                FileAgent.Pause = value;
            }
        }*/

        private string _rootPath;
        public string RootPath
        {
            get { return _rootPath; }
            set 
            {
                _rootPath = String.IsNullOrWhiteSpace(value) 
                    ? String.Empty 
                    : value.ToLower();
            }
        }


        CancellationTokenSource _cancellationSource;

        public PithosMonitor()
        {
            FileAgent = new FileAgent();            
        }
        private bool _started;

        public void Start()
        {            
            if (String.IsNullOrWhiteSpace(ApiKey))
                throw new InvalidOperationException("The ApiKey is empty");
            if (String.IsNullOrWhiteSpace(UserName))
                throw new InvalidOperationException("The UserName is empty");
            if (String.IsNullOrWhiteSpace(AuthenticationUrl))
                throw new InvalidOperationException("The Authentication url is empty");
            Contract.EndContractBlock();

            //If the account doesn't have a valid path, don't start monitoring but don't throw either
            if (String.IsNullOrWhiteSpace(RootPath))
                //TODO; Warn user?
                return;

            //WorkflowAgent.StatusNotification = StatusNotification;

            StatusNotification.NotifyChange("Starting");
            if (_started)
            {
                if (!_cancellationSource.IsCancellationRequested)
                    return;
            }
            _cancellationSource = new CancellationTokenSource();

            lock (this)
            {
                CloudClient = new CloudFilesClient(UserName, ApiKey)
                                  {UsePithos = true, AuthenticationUrl = AuthenticationUrl};
                _accountInfo = CloudClient.Authenticate();
            }
            _accountInfo.SiteUri = AuthenticationUrl;
            _accountInfo.AccountPath = RootPath;


            var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
            if (!Directory.Exists(pithosFolder))
                Directory.CreateDirectory(pithosFolder);
            //Create the cache folder and ensure it is hidden
            CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);

            var policy=CloudClient.GetAccountPolicies(_accountInfo);

            StatusNotification.NotifyAccount(policy);
            EnsurePithosContainers();
            
            StatusKeeper.BlockHash = _blockHash;
            StatusKeeper.BlockSize = _blockSize;
            
            StatusKeeper.StartProcessing(_cancellationSource.Token);
            CleanupUnselectedStates();
            IndexLocalFiles();
            //Extract the URIs from the string collection
            var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey );
                            
            var selectiveUrls=settings.SelectiveFolders.Cast<string>().Select(url => new Uri(url,UriKind.RelativeOrAbsolute))
                .Where(uri=>uri.IsAbsoluteUri).ToArray();

            SetSelectivePaths(selectiveUrls,null,null);
            
            StartWatcherAgent();

            StartNetworkAgent();
            
            //WorkflowAgent.RestartInterruptedFiles(_accountInfo);
            _started = true;
        }

        private void EnsurePithosContainers()
        {

            //Create the two default containers if they are missing
            var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
            foreach (var container in pithosContainers)
            {                
                var info=CloudClient.GetContainerInfo(UserName, container);
                if (info == ContainerInfo.Empty)
                {
                    CloudClient.CreateContainer(UserName, container);
                    info = CloudClient.GetContainerInfo(UserName, container);
                }
                _blockSize = info.BlockSize;
                _blockHash = info.BlockHash;
                _accountInfo.BlockSize = _blockSize;
                _accountInfo.BlockHash = _blockHash;
            }
        }

        public string AuthenticationUrl { get; set; }

        private void IndexLocalFiles()
        {
            using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))
            {
                
                try
                {
                    //StatusNotification.NotifyChange("Indexing Local Files");
                    Log.Info("Start local indexing");
                    StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Indexing Local Files");                    

                    var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
                    var directory = new DirectoryInfo(RootPath);
                    var files =
                        from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
                        where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
                              !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
                        select file;
                    StatusKeeper.ProcessExistingFiles(files);

                }
                catch (Exception exc)
                {
                    Log.Error("[ERROR]", exc);
                }
                finally
                {
                    Log.Info("[END]");
                }
                StatusNotification.SetPithosStatus(PithosStatus.LocalComplete,"Indexing Completed");
            }
        }

        
  


       /* private void StartWorkflowAgent()
        {
            WorkflowAgent.StatusNotification = StatusNotification;

/*            //On Vista and up we can check for a network connection
            bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
            //If we are not connected retry later
            if (!connected)
            {
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
                return;
            }#1#

            try
            {
                WorkflowAgent.Start();                
            }
            catch (Exception)
            {
                //Faild to authenticate due to network or account error
                //Retry after a while
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
            }
        }*/


        private void StartNetworkAgent()
        {
            NetworkAgent.StatusNotification = StatusNotification;

            //TODO: The Network and Poll agents are not account specific
            //They should be moved outside PithosMonitor
/*
            NetworkAgent.Start();
*/

            PollAgent.AddAccount(_accountInfo);

            PollAgent.StatusNotification = StatusNotification;

            PollAgent.PollRemoteFiles();
        }

        //Make sure a hidden cache folder exists to store partial downloads
        private static void CreateHiddenFolder(string rootPath, string folderName)
        {
            if (String.IsNullOrWhiteSpace(rootPath))
                throw new ArgumentNullException("rootPath");
            if (!Path.IsPathRooted(rootPath))
                throw new ArgumentException("rootPath");
            if (String.IsNullOrWhiteSpace(folderName))
                throw new ArgumentNullException("folderName");
            Contract.EndContractBlock();

            var folder = Path.Combine(rootPath, folderName);
            if (!Directory.Exists(folder))
            {
                var info = Directory.CreateDirectory(folder);
                info.Attributes |= FileAttributes.Hidden;

                Log.InfoFormat("Created cache Folder: {0}", folder);
            }
            else
            {
                var info = new DirectoryInfo(folder);
                if ((info.Attributes & FileAttributes.Hidden) == 0)
                {
                    info.Attributes |= FileAttributes.Hidden;
                    Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
                }                                
            }
        }

       


        private void StartWatcherAgent()
        {
            if (Log.IsDebugEnabled)
                Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);

            AgentLocator<FileAgent>.Register(FileAgent,RootPath);
            
            FileAgent.IdleTimeout = Settings.FileIdleTimeout;
            FileAgent.StatusKeeper = StatusKeeper;
            FileAgent.StatusNotification = StatusNotification;
            FileAgent.Workflow = Workflow;
            FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
            FileAgent.Start(_accountInfo, RootPath);
        }

        public void Stop()
        {
/*
            AgentLocator<FileAgent>.Remove(RootPath);

            if (FileAgent!=null)
                FileAgent.Stop();
            FileAgent = null;
*/
        }


        ~PithosMonitor()
        {
            Dispose(false);
        }

        public void Dispose()
        {
            Dispose(true);
            GC.SuppressFinalize(this);
        }

        protected virtual void Dispose(bool disposing)
        {
            if (disposing)
            {
                Stop();
            }
        }


        public void MoveFileStates(string oldPath, string newPath)
        {
            if (String.IsNullOrWhiteSpace(oldPath))
                throw new ArgumentNullException("oldPath");
            if (!Path.IsPathRooted(oldPath))
                throw new ArgumentException("oldPath must be an absolute path","oldPath");
            if (string.IsNullOrWhiteSpace(newPath))
                throw new ArgumentNullException("newPath");
            if (!Path.IsPathRooted(newPath))
                throw new ArgumentException("newPath must be an absolute path","newPath");
            Contract.EndContractBlock();

            StatusKeeper.ChangeRoots(oldPath, newPath);
        }

        private void CleanupUnselectedStates()
        {
            //var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey);
            if (!Selectives.IsSelectiveEnabled(_accountInfo.AccountKey)) return;

            List<string> selectivePaths;
            if (Selectives.SelectivePaths.TryGetValue(_accountInfo.AccountKey, out selectivePaths))
            {

                var removedPaths = from state in FileState.Queryable
                                   from path in selectivePaths
                                   where !state.FilePath.IsAtOrBelow(path)
                                   select state.FilePath;

                UnversionSelectivePaths(removedPaths.ToList());
            }
            else
            {
                StatusKeeper.ClearFolderStatus(Account.AccountPath);
            }
        }

        public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)
        {
            //Convert the uris to paths
            //var selectivePaths = UrisToFilePaths(uris);
            
            var selectiveUri = uris.ToList();
            this.Selectives.SetSelectedUris(_accountInfo,selectiveUri);

            var removedPaths = UrisToFilePaths(removed);
            UnversionSelectivePaths(removedPaths);

        }

        /// <summary>
        /// Mark all unselected paths as Unversioned
        /// </summary>
        /// <param name="removed"></param>
        private void UnversionSelectivePaths(List<string> removed)
        {
            if (removed == null)
                return;

            //Ensure we remove any file state below the deleted folders
            FileState.UnversionPaths(removed);
        }


        /// <summary>
        /// Return a list of absolute filepaths from a list of Uris
        /// </summary>
        /// <param name="uris"></param>
        /// <returns></returns>
        public List<string> UrisToFilePaths(IEnumerable<Uri> uris)
        {
            if (uris == null)
                return new List<string>();

            var own = (from uri in uris
                       where uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
                                   let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
                                   //Trim the account name
                                   select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();
            var others= (from uri in uris
                         where !uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
                                   let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
                                   //Trim the account name
                                   select Path.Combine(RootPath,"others-shared", relativePath)).ToList();
            return own.Union(others).ToList();            
        }


        public ObjectInfo GetObjectInfo(string filePath)
        {
            if (String.IsNullOrWhiteSpace(filePath))
                throw new ArgumentNullException("filePath");
            Contract.EndContractBlock();

            var file=new FileInfo(filePath);
            string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
            var relativePath = file.AsRelativeTo(RootPath);
            
            string accountName,container;
            
            var parts=relativePath.Split('\\');

            var accountInfo = _accountInfo;
            if (relativePath.StartsWith(FolderConstants.OthersFolder))
            {                
                accountName = parts[1];
                container = parts[2];
                relativeUrl = String.Join("/", parts.Splice(3));
                //Create the root URL for the target account
                var oldName = UserName;
                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
                var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
                var root=absoluteUri.Substring(0, nameIndex);

                accountInfo = new AccountInfo
                {
                    UserName = accountName,
                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
                    StorageUri = new Uri(root + accountName),
                    BlockHash=accountInfo.BlockHash,
                    BlockSize=accountInfo.BlockSize,
                    Token=accountInfo.Token
                };
            }
            else
            {
                accountName = UserName;
                container = parts[0];
                relativeUrl = String.Join("/", parts.Splice(1));
            }
            
            var client = new CloudFilesClient(accountInfo);
            var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
            return objectInfo;
        }
        
        public Task<ContainerInfo> GetContainerInfo(string filePath)
        {
            if (String.IsNullOrWhiteSpace(filePath))
                throw new ArgumentNullException("filePath");
            Contract.EndContractBlock();

            var file=new FileInfo(filePath);
            var relativePath = file.AsRelativeTo(RootPath);
            
            string accountName,container;
            
            var parts=relativePath.Split('\\');

            var accountInfo = _accountInfo;
            if (relativePath.StartsWith(FolderConstants.OthersFolder))
            {                
                accountName = parts[1];
                container = parts[2];                
                //Create the root URL for the target account
                var oldName = UserName;
                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
                var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
                var root=absoluteUri.Substring(0, nameIndex);

                accountInfo = new AccountInfo
                {
                    UserName = accountName,
                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
                    StorageUri = new Uri(root + accountName),
                    BlockHash=accountInfo.BlockHash,
                    BlockSize=accountInfo.BlockSize,
                    Token=accountInfo.Token
                };
            }
            else
            {
                accountName = UserName;
                container = parts[0];                
            }

            return Task.Factory.StartNew(() =>
            {
                var client = new CloudFilesClient(accountInfo);
                var containerInfo = client.GetContainerInfo(accountName, container);
                return containerInfo;
            });
        }
    }
}