Merge branch 'master' of https://code.grnet.gr/git/pithos-ms-client
[pithos-ms-client] / trunk / Pithos.Core / PithosMonitor.cs
index c58c264..7ca5a30 100644 (file)
+#region
+/* -----------------------------------------------------------------------
+ * <copyright file="PithosMonitor.cs" company="GRNet">
+ * 
+ * Copyright 2011-2012 GRNET S.A. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ *   1. Redistributions of source code must retain the above
+ *      copyright notice, this list of conditions and the following
+ *      disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above
+ *      copyright notice, this list of conditions and the following
+ *      disclaimer in the documentation and/or other materials
+ *      provided with the distribution.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and
+ * documentation are those of the authors and should not be
+ * interpreted as representing official policies, either expressed
+ * or implied, of GRNET S.A.
+ * </copyright>
+ * -----------------------------------------------------------------------
+ */
+#endregion
 using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.ComponentModel.Composition;
-using System.Diagnostics;
 using System.Diagnostics.Contracts;
 using System.IO;
 using System.Linq;
-using System.Security.Cryptography;
-using System.Text;
+using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
+using Pithos.Core.Agents;
 using Pithos.Interfaces;
+using Pithos.Network;
+using log4net;
 
 namespace Pithos.Core
 {
     [Export(typeof(PithosMonitor))]
     public class PithosMonitor:IDisposable
     {
+        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
+        private int _blockSize;
+        private string _blockHash;
 
         [Import]
         public IPithosSettings Settings{get;set;}
 
-        [Import]
-        public IStatusKeeper StatusKeeper { get; set; }
+        private IStatusKeeper _statusKeeper;
 
         [Import]
-        public IPithosWorkflow Workflow { get; set; }
+        public IStatusKeeper StatusKeeper
+        {
+            get { return _statusKeeper; }
+            set
+            {
+                _statusKeeper = value;
+                FileAgent.StatusKeeper = value;
+            }
+        }
 
-        [Import]
-        public ICloudClient CloudClient { get; set; }
 
-        [Import]
-        public ICloudClient CloudListeningClient { get; set; }
+        private IPithosWorkflow _workflow;
 
-        private FileSystemWatcher _watcher;
-
-        public bool Pause
+        [Import]
+        public IPithosWorkflow Workflow
         {
-            get { return _watcher == null || !_watcher.EnableRaisingEvents; }
+            get { return _workflow; }
             set
             {
-                if (_watcher!=null)
-                    _watcher.EnableRaisingEvents = !value;                
+                _workflow = value;
+                FileAgent.Workflow = value;
             }
         }
 
+        public ICloudClient CloudClient { get; set; }
 
-        CancellationTokenSource _cancellationSource;
+        public IStatusNotification StatusNotification { get; set; }
 
-        BlockingCollection<WorkflowState> _fileEvents = new BlockingCollection<WorkflowState>();
+        //[Import]
+        public FileAgent FileAgent { get; private set; }
 
-        
+        private WorkflowAgent _workflowAgent;
 
-        public void Start()
+        [Import]
+        public WorkflowAgent WorkflowAgent
         {
-            string path = Settings.PithosPath;
-
-            CloudClient.Authenticate(Settings.UserName,Settings.ApiKey);
-
-            if (_cancellationSource != null)
+            get { return _workflowAgent; }
+            set
             {
-                if (!_cancellationSource.IsCancellationRequested)
-                    return;
+                _workflowAgent = value;
+                FileAgent.WorkflowAgent = value;
             }
-            _cancellationSource=new CancellationTokenSource();
-            StartListening();
-            StartSending();
-
-
-            _watcher = new FileSystemWatcher(path);            
-            _watcher.Changed += OnFileEvent;
-            _watcher.Created += OnFileEvent;
-            _watcher.Deleted += OnFileEvent;
-            _watcher.Renamed += OnRenameEvent;
-            _watcher.EnableRaisingEvents = true;            
         }
+        
+        [Import]
+        public NetworkAgent NetworkAgent { get; set; }
+        [Import]
+        public PollAgent PollAgent { get; set; }       
 
-        internal enum CloudActionType
+        public string UserName { get; set; }
+        private string _apiKey;
+        public string ApiKey
         {
-            Upload=0,
-            Download,
-            UploadUnconditional,
-            DownloadUnconditional,
-            DeleteLocal,
-            DeleteCloud
+            get { return _apiKey; }
+            set
+            {
+                _apiKey = value;
+                if (_accountInfo != null)
+                    _accountInfo.Token = value;
+            }
         }
 
-        internal class ListenerAction
-        {
-            public CloudActionType Action { get; set; }
-            public FileInfo LocalFile { get; set; }
-            public ObjectInfo CloudFile { get; set; }
+        private AccountInfo _accountInfo;
+
 
-            public Lazy<string> LocalHash { get; set; }
 
-            public ListenerAction(CloudActionType action, FileInfo localFile, ObjectInfo cloudFile)
-            {
-                Action = action;
-                LocalFile = localFile;
-                CloudFile = cloudFile;
-                LocalHash=new Lazy<string>(()=>CalculateHash(LocalFile.FullName),LazyThreadSafetyMode.ExecutionAndPublication);
-            }
-            
-        }
 
-        internal class LocalFileComparer:EqualityComparer<ListenerAction>
+
+
+        public bool Pause
         {
-            public override bool Equals(ListenerAction x, ListenerAction y)
+            get { return FileAgent.Pause; }
+            set
             {
-                if (x.Action != y.Action)
-                    return false;
-                if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName))
-                    return false;
-                if (x.CloudFile != null && y.CloudFile != null && !x.CloudFile.Hash.Equals(y.CloudFile.Hash))
-                    return false;
-                if (x.CloudFile == null ^ y.CloudFile == null ||
-                    x.LocalFile == null ^ y.LocalFile == null)
-                    return false;
-                return true;
+                FileAgent.Pause = value;
             }
+        }
 
-            public override int GetHashCode(ListenerAction obj)
+        private string _rootPath;
+        public string RootPath
+        {
+            get { return _rootPath; }
+            set 
             {
-                var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode();
-                var hash2 = (obj.CloudFile == null) ? int.MaxValue : obj.CloudFile.Hash.GetHashCode();
-                var hash3 = obj.Action.GetHashCode();
-                return hash1 ^ hash2 & hash3;
+                _rootPath = String.IsNullOrWhiteSpace(value) 
+                    ? String.Empty 
+                    : value.ToLower();
             }
         }
 
-        private BlockingCollection<ListenerAction> _listenerActions=new BlockingCollection<ListenerAction>();
 
-        private Timer timer;
+        CancellationTokenSource _cancellationSource;
 
-        private void StartListening()
+        public PithosMonitor()
         {
-            
-            Func<Task> listener = ()=>Task.Factory.StartNew(()=>CloudClient.ListObjects("PITHOS"))
-                .ContinueWith(task =>
-                                  {
-                                      
-                                      var objects = task.Result;
-                                      if (objects.Count == 0)
-                                          return;
-
-                                      var pithosDir = new DirectoryInfo(Settings.PithosPath);
-                                      
-                                      var upFiles = from info in objects
-                                                    select info.Name;
-                                      
-                                      var onlyLocal = from localFile in pithosDir.EnumerateFiles()
-                                                      where !upFiles.Contains(localFile.Name) 
-                                                      select new ListenerAction(CloudActionType.UploadUnconditional, localFile,null);
-                                      
-                                      
-                                    
-
-                                      var localNames =pithosDir.EnumerateFiles().Select(info => info.Name);
-                                      var onlyRemote = from upFile in objects
-                                                       where !localNames.Contains(upFile.Name)
-                                                       select new ListenerAction(CloudActionType.DownloadUnconditional,null,upFile);
-
-
-                                      var existingObjects = from  upFile in objects
-                                                            join  localFile in pithosDir.EnumerateFiles()
-                                                            on upFile.Name equals localFile.Name 
-                                                       select new ListenerAction(CloudActionType.Download, localFile, upFile);
-
-                                      var uniques =
-                                          onlyLocal.Union(onlyRemote).Union(existingObjects)
-                                          .Except(_listenerActions,new LocalFileComparer());
-
-                                      _listenerActions.AddFromEnumerable(uniques, false);
-                                     
-                                 }
-                );
-
-            Task.Factory.StartNew(() =>
-                                      {
-                                          foreach (var action in _listenerActions.GetConsumingEnumerable())
-                                          {
-                                              var localFile = action.LocalFile;
-                                              var cloudFile = action.CloudFile;
-                                              var downloadPath = (cloudFile==null)? String.Empty:Path.Combine(Settings.PithosPath,cloudFile.Name);
-
-                                              switch(action.Action)
-                                              {
-                                                  case CloudActionType.UploadUnconditional:
-                                                                                                        
-                                                    UploadCloudFile(localFile.Name, localFile.Length, localFile.FullName, action.LocalHash.Value);
-                                                    break;
-                                                  case CloudActionType.DownloadUnconditional:                                                      
-                                                    DownloadCloudFile("PITHOS", cloudFile.Name, downloadPath);
-                                                    break;
-                                                  case CloudActionType.Download:                                                    
-                                                    if (File.Exists(downloadPath))
-                                                    {                                                                                                     
-                                                        if (cloudFile.Hash != action.LocalHash.Value)
-                                                        {
-                                                            var lastLocalTime=localFile.LastWriteTime;
-                                                            var lastUpTime=cloudFile.Last_Modified;
-                                                            if(lastUpTime<=lastLocalTime)
-                                                            {
-                                                                //Files in conflict
-                                                                StatusKeeper.SetFileOverlayStatus(downloadPath,FileOverlayStatus.Conflict);
-                                                            }
-                                                            else
-                                                                DownloadCloudFile("PITHOS", action.CloudFile.Name, downloadPath);
-                                                        }
-                                                    }
-                                                    else
-                                                        DownloadCloudFile("PITHOS", action.CloudFile.Name, downloadPath);
-                                                    break;
-                                              }
-                                          }
-                                      }
-                );
-            
-            timer = new Timer(o => listener(), null, TimeSpan.Zero, TimeSpan.FromSeconds(10));
-            
+            FileAgent = new FileAgent();
         }
+        private bool _started;
 
-        private void DownloadCloudFile(string container, string fileName, string localPath)
-        {
-            using (var upstream = CloudClient.GetObject(container, fileName))
-            using (var fileStream = File.OpenWrite(localPath))
+        public void Start()
+        {            
+            if (String.IsNullOrWhiteSpace(ApiKey))
+                throw new InvalidOperationException("The ApiKey is empty");
+            if (String.IsNullOrWhiteSpace(UserName))
+                throw new InvalidOperationException("The UserName is empty");
+            if (String.IsNullOrWhiteSpace(AuthenticationUrl))
+                throw new InvalidOperationException("The Authentication url is empty");
+            Contract.EndContractBlock();
+
+            //If the account doesn't have a valid path, don't start monitoring but don't throw either
+            if (String.IsNullOrWhiteSpace(RootPath))
+                //TODO; Warn user?
+                return;
+
+            WorkflowAgent.StatusNotification = StatusNotification;
+
+            StatusNotification.NotifyChange("Starting");
+            if (_started)
             {
-                upstream.CopyTo(fileStream);
+                if (!_cancellationSource.IsCancellationRequested)
+                    return;
             }
-        }
-
-        private void StartSending()
-        {
-            Task.Factory.StartNew(() =>
-                                      {
-                                          foreach (var state in _fileEvents.GetConsumingEnumerable())
-                                          {
-                                              try
-                                              {
-                                                  UpdateFileStatus(state);
-                                                  UpdateOverlayStatus(state);
-                                                  UpdateFileChecksum(state);
-                                                  SynchToCloud(state);
-                                              }
-                                              catch (OperationCanceledException)
-                                              {
-                                                  throw;
-                                              }
-                                              catch(Exception ex)
-                                              {}
-                                          }
-                                          
-                                      },_cancellationSource.Token);
-        }
-
+            _cancellationSource = new CancellationTokenSource();
 
-        private WorkflowState SynchToCloud(WorkflowState state)
-        {
-            if (state.Skip)
-                return state;
-            string path = state.Path;
-            string fileName = Path.GetFileName(path);
-
-            switch(state.Status)
+            lock (this)
             {
-                case FileStatus.Created:
-                case FileStatus.Modified:
-                    var info = new FileInfo(path);
-                    long fileSize = info.Length;
-                    UploadCloudFile(fileName, fileSize, path,state.Hash);
-                    break;
-                case FileStatus.Deleted:
-                    DeleteCloudFile(fileName);
-                    break;
-                case FileStatus.Renamed:
-                    RenameCloudFile(state);
-                    break;
+                CloudClient = new CloudFilesClient(UserName, ApiKey)
+                                  {UsePithos = true, AuthenticationUrl = AuthenticationUrl};
+                _accountInfo = CloudClient.Authenticate();
             }
-            return state;
-        }
+            _accountInfo.SiteUri = AuthenticationUrl;
+            _accountInfo.AccountPath = RootPath;
 
-        private void RenameCloudFile(WorkflowState state)
-        {
-            this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Synch);
 
+            var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
+            if (!Directory.Exists(pithosFolder))
+                Directory.CreateDirectory(pithosFolder);
+            //Create the cache folder and ensure it is hidden
+            CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
+
+            var policy=CloudClient.GetAccountPolicies(_accountInfo);
 
+            StatusNotification.NotifyAccount(policy);
+            EnsurePithosContainers();
+            
+            StatusKeeper.BlockHash = _blockHash;
+            StatusKeeper.BlockSize = _blockSize;
+            
+            StatusKeeper.StartProcessing(_cancellationSource.Token);
+            IndexLocalFiles();
+            //Extract the URIs from the string collection
+            var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey );
+            var selectiveUrls=settings.SelectiveFolders.Cast<string>().Select(url => new Uri(url)).ToArray();
 
-            CloudClient.MoveObject("PITHOS", state.OldFileName, state.FileName);
+            SetSelectivePaths(selectiveUrls,null,null);
+            
+            StartWatcherAgent();
 
-            this.StatusKeeper.SetFileStatus(state.Path, FileStatus.Unchanged);
-            this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
-            Workflow.RaiseChangeNotification(state.Path);
+            StartNetworkAgent();
+            
+            WorkflowAgent.RestartInterruptedFiles(_accountInfo);
+            _started = true;
         }
 
-        private void DeleteCloudFile(string fileName)
+        private void EnsurePithosContainers()
         {
-            Contract.Requires(!Path.IsPathRooted(fileName));
 
-            this.StatusKeeper.SetFileOverlayStatus(fileName, FileOverlayStatus.Synch);
-            CloudClient.DeleteObject("PITHOS", fileName);
-            this.StatusKeeper.ClearFileStatus(fileName);
-            this.StatusKeeper.RemoveFileOverlayStatus(fileName);            
+            //Create the two default containers if they are missing
+            var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
+            foreach (var container in pithosContainers)
+            {                
+                var info=CloudClient.GetContainerInfo(UserName, container);
+                if (info == ContainerInfo.Empty)
+                {
+                    CloudClient.CreateContainer(UserName, container);
+                    info = CloudClient.GetContainerInfo(UserName, container);
+                }
+                _blockSize = info.BlockSize;
+                _blockHash = info.BlockHash;
+                _accountInfo.BlockSize = _blockSize;
+                _accountInfo.BlockHash = _blockHash;
+            }
         }
 
-        private void UploadCloudFile(string fileName, long fileSize, string path,string hash)
+        public string AuthenticationUrl { get; set; }
+
+        private void IndexLocalFiles()
         {
-            Contract.Requires(!Path.IsPathRooted(fileName));
-            //Even if GetObjectInfo times out, we can proceed with the upload
-            var info=CloudClient.GetObjectInfo("PITHOS", fileName);
-            if ( hash != info.Hash)
+            using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))
             {
-                this.StatusKeeper.SetFileOverlayStatus(path, FileOverlayStatus.Synch);
-                using (var stream = File.OpenRead(path))
+                
+                try
+                {
+                    //StatusNotification.NotifyChange("Indexing Local Files");
+                    Log.Info("Start local indexing");
+                    StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Indexing Local Files");                    
+
+                    var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
+                    var directory = new DirectoryInfo(RootPath);
+                    var files =
+                        from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
+                        where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
+                              !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
+                        select file;
+                    StatusKeeper.ProcessExistingFiles(files);
+
+                }
+                catch (Exception exc)
                 {
-                    CloudClient.PutObject("PITHOS", fileName, stream, fileSize);
+                    Log.Error("[ERROR]", exc);
                 }
+                finally
+                {
+                    Log.Info("[END]");
+                }
+                StatusNotification.SetPithosStatus(PithosStatus.LocalComplete,"Indexing Completed");
             }
-            this.StatusKeeper.SetFileStatus(path,FileStatus.Unchanged);
-            this.StatusKeeper.SetFileOverlayStatus(path,FileOverlayStatus.Normal);
-            Workflow.RaiseChangeNotification(path);
         }
 
-        private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
-        {
-            {WatcherChangeTypes.Created,FileStatus.Created},
-            {WatcherChangeTypes.Changed,FileStatus.Modified},
-            {WatcherChangeTypes.Deleted,FileStatus.Deleted},
-            {WatcherChangeTypes.Renamed,FileStatus.Renamed}
-        };
+        
+  
 
-        private WorkflowState UpdateFileStatus(WorkflowState  state)
-        {
-            string path = state.Path;
-            FileStatus status = _statusDict[state.TriggeringChange];
-            var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
-            if (status == oldStatus)
-            {
-                state.Status = status;
-                state.Skip = true;
-                return state;
-            }
-            if (state.Status == FileStatus.Renamed)
-                Workflow.ClearFileStatus(path);                
 
-            state.Status = Workflow.SetFileStatus(path, status);                
-            return state;
-        }
+       /* private void StartWorkflowAgent()
+        {
+            WorkflowAgent.StatusNotification = StatusNotification;
 
-        private WorkflowState UpdateOverlayStatus(WorkflowState state)
-        {            
-            if (state.Skip)
-                return state;
+/*            //On Vista and up we can check for a network connection
+            bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
+            //If we are not connected retry later
+            if (!connected)
+            {
+                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
+                return;
+            }#1#
 
-            switch (state.Status)
+            try
             {
-                case FileStatus.Created:
-                case FileStatus.Modified:
-                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
-                    break;
-                case FileStatus.Deleted:
-                    this.StatusKeeper.RemoveFileOverlayStatus(state.Path);
-                    break;
-                case FileStatus.Renamed:
-                    this.StatusKeeper.RemoveFileOverlayStatus(state.OldPath);
-                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
-                    break;
-                case FileStatus.Unchanged:
-                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
-                    break;
+                WorkflowAgent.Start();                
             }
-
-            if (state.Status==FileStatus.Deleted)
-                Workflow.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
-            else
-                Workflow.RaiseChangeNotification(state.Path);
-            return state;
-        }
+            catch (Exception)
+            {
+                //Faild to authenticate due to network or account error
+                //Retry after a while
+                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
+            }
+        }*/
 
 
-        private WorkflowState UpdateFileChecksum(WorkflowState state)
+        private void StartNetworkAgent()
         {
-            if (state.Skip)
-                return state;
+            NetworkAgent.StatusNotification = StatusNotification;
 
-            if (state.Status == FileStatus.Deleted)
-                return state;
+            //TODO: The Network and Poll agents are not account specific
+            //They should be moved outside PithosMonitor
+            NetworkAgent.Start();
 
-            string path = state.Path;
-            string hash = CalculateHash(path);
+            PollAgent.AddAccount(_accountInfo);
 
-            StatusKeeper.UpdateFileChecksum(path, hash);
+            PollAgent.StatusNotification = StatusNotification;
 
-            state.Hash = hash;
-            return state;
+            PollAgent.PollRemoteFiles();
         }
 
-        private static string CalculateHash(string path)
+        //Make sure a hidden cache folder exists to store partial downloads
+        private static void CreateHiddenFolder(string rootPath, string folderName)
         {
-            string hash;
-            using (var hasher = MD5.Create())
-            using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096, true))
+            if (String.IsNullOrWhiteSpace(rootPath))
+                throw new ArgumentNullException("rootPath");
+            if (!Path.IsPathRooted(rootPath))
+                throw new ArgumentException("rootPath");
+            if (String.IsNullOrWhiteSpace(folderName))
+                throw new ArgumentNullException("folderName");
+            Contract.EndContractBlock();
+
+            var folder = Path.Combine(rootPath, folderName);
+            if (!Directory.Exists(folder))
             {
-                var hashBytes = hasher.ComputeHash(stream);
-                var hashBuilder = new StringBuilder();
-                foreach (byte b in hasher.ComputeHash(stream))
-                    hashBuilder.Append(b.ToString("x2").ToLower());
-                hash = hashBuilder.ToString();
+                var info = Directory.CreateDirectory(folder);
+                info.Attributes |= FileAttributes.Hidden;
 
+                Log.InfoFormat("Created cache Folder: {0}", folder);
+            }
+            else
+            {
+                var info = new DirectoryInfo(folder);
+                if ((info.Attributes & FileAttributes.Hidden) == 0)
+                {
+                    info.Attributes |= FileAttributes.Hidden;
+                    Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
+                }                                
             }
-            return hash;
         }
 
-        private FileSystemEventArgs CalculateSignature(FileSystemEventArgs arg)
-        {
-            Debug.WriteLine(String.Format("{0} {1} {2}", arg.ChangeType, arg.Name, arg.FullPath), "INFO");
-            return arg;
-        }
+       
 
-        void OnFileEvent(object sender, FileSystemEventArgs e)
-        {
-            _fileEvents.Add(new WorkflowState{Path=e.FullPath,FileName = e.Name,TriggeringChange=e.ChangeType});            
-        }
 
-        void OnRenameEvent(object sender, RenamedEventArgs e)
+        private void StartWatcherAgent()
         {
-            _fileEvents.Add(new WorkflowState { OldPath=e.OldFullPath,OldFileName=e.OldName,
-                Path = e.FullPath, FileName = e.Name, TriggeringChange = e.ChangeType });
+            if (Log.IsDebugEnabled)
+                Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);
+
+            AgentLocator<FileAgent>.Register(FileAgent,RootPath);
+            
+            FileAgent.IdleTimeout = Settings.FileIdleTimeout;
+            FileAgent.StatusKeeper = StatusKeeper;
+            FileAgent.StatusNotification = StatusNotification;
+            FileAgent.Workflow = Workflow;
+            FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
+            FileAgent.Start(_accountInfo, RootPath);
         }
 
         public void Stop()
         {
-            if (_watcher != null)
-            {
-                _watcher.Changed -= OnFileEvent;
-                _watcher.Created -= OnFileEvent;
-                _watcher.Deleted -= OnFileEvent;
-                _watcher.Renamed -= OnRenameEvent;
-                _watcher.Dispose();
-            }
-            _watcher = null;
-            _fileEvents.CompleteAdding();
-            if (timer != null)
-                timer.Dispose();
-            timer = null;
+            AgentLocator<FileAgent>.Remove(RootPath);
+
+            if (FileAgent!=null)
+                FileAgent.Stop();
+            FileAgent = null;
         }
 
+
         ~PithosMonitor()
         {
             Dispose(false);
@@ -465,5 +411,167 @@ namespace Pithos.Core
         }
 
 
+        public void MoveFileStates(string oldPath, string newPath)
+        {
+            if (String.IsNullOrWhiteSpace(oldPath))
+                throw new ArgumentNullException("oldPath");
+            if (!Path.IsPathRooted(oldPath))
+                throw new ArgumentException("oldPath must be an absolute path","oldPath");
+            if (string.IsNullOrWhiteSpace(newPath))
+                throw new ArgumentNullException("newPath");
+            if (!Path.IsPathRooted(newPath))
+                throw new ArgumentException("newPath must be an absolute path","newPath");
+            Contract.EndContractBlock();
+
+            StatusKeeper.ChangeRoots(oldPath, newPath);
+        }
+
+        public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)
+        {
+            //Convert the uris to paths
+            var selectivePaths = UrisToFilePaths(uris);
+            
+            FileAgent.SelectivePaths=selectivePaths;
+            WorkflowAgent.SelectivePaths = selectivePaths;
+            PollAgent.SetSyncUris(_accountInfo.AccountKey,uris);
+            
+            var removedPaths = UrisToFilePaths(removed);
+            UnversionSelectivePaths(removedPaths);
+
+        }
+
+        /// <summary>
+        /// Mark all unselected paths as Unversioned
+        /// </summary>
+        /// <param name="removed"></param>
+        private void UnversionSelectivePaths(List<string> removed)
+        {
+            if (removed == null)
+                return;
+
+            //Ensure we remove any file state below the deleted folders
+            FileState.UnversionPaths(removed);
+        }
+
+
+        /// <summary>
+        /// Return a list of absolute filepaths from a list of Uris
+        /// </summary>
+        /// <param name="uris"></param>
+        /// <returns></returns>
+        private List<string> UrisToFilePaths(IEnumerable<Uri> uris)
+        {
+            if (uris == null)
+                return new List<string>();
+
+            var own = (from uri in uris
+                       where uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
+                                   let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
+                                   //Trim the account name
+                                   select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();
+            var others= (from uri in uris
+                         where !uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
+                                   let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
+                                   //Trim the account name
+                                   select Path.Combine(RootPath,"others-shared", relativePath)).ToList();
+            return own.Union(others).ToList();            
+        }
+
+
+        public ObjectInfo GetObjectInfo(string filePath)
+        {
+            if (String.IsNullOrWhiteSpace(filePath))
+                throw new ArgumentNullException("filePath");
+            Contract.EndContractBlock();
+
+            var file=new FileInfo(filePath);
+            string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
+            var relativePath = file.AsRelativeTo(RootPath);
+            
+            string accountName,container;
+            
+            var parts=relativePath.Split('\\');
+
+            var accountInfo = _accountInfo;
+            if (relativePath.StartsWith(FolderConstants.OthersFolder))
+            {                
+                accountName = parts[1];
+                container = parts[2];
+                relativeUrl = String.Join("/", parts.Splice(3));
+                //Create the root URL for the target account
+                var oldName = UserName;
+                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
+                var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
+                var root=absoluteUri.Substring(0, nameIndex);
+
+                accountInfo = new AccountInfo
+                {
+                    UserName = accountName,
+                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
+                    StorageUri = new Uri(root + accountName),
+                    BlockHash=accountInfo.BlockHash,
+                    BlockSize=accountInfo.BlockSize,
+                    Token=accountInfo.Token
+                };
+            }
+            else
+            {
+                accountName = UserName;
+                container = parts[0];
+                relativeUrl = String.Join("/", parts.Splice(1));
+            }
+            
+            var client = new CloudFilesClient(accountInfo);
+            var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
+            return objectInfo;
+        }
+        
+        public Task<ContainerInfo> GetContainerInfo(string filePath)
+        {
+            if (String.IsNullOrWhiteSpace(filePath))
+                throw new ArgumentNullException("filePath");
+            Contract.EndContractBlock();
+
+            var file=new FileInfo(filePath);
+            var relativePath = file.AsRelativeTo(RootPath);
+            
+            string accountName,container;
+            
+            var parts=relativePath.Split('\\');
+
+            var accountInfo = _accountInfo;
+            if (relativePath.StartsWith(FolderConstants.OthersFolder))
+            {                
+                accountName = parts[1];
+                container = parts[2];                
+                //Create the root URL for the target account
+                var oldName = UserName;
+                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
+                var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
+                var root=absoluteUri.Substring(0, nameIndex);
+
+                accountInfo = new AccountInfo
+                {
+                    UserName = accountName,
+                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
+                    StorageUri = new Uri(root + accountName),
+                    BlockHash=accountInfo.BlockHash,
+                    BlockSize=accountInfo.BlockSize,
+                    Token=accountInfo.Token
+                };
+            }
+            else
+            {
+                accountName = UserName;
+                container = parts[0];                
+            }
+
+            return Task.Factory.StartNew(() =>
+            {
+                var client = new CloudFilesClient(accountInfo);
+                var containerInfo = client.GetContainerInfo(accountName, container);
+                return containerInfo;
+            });
+        }
     }
 }