using System; using System.Collections.Generic; using System.ComponentModel.Composition; using System.Diagnostics; using System.Diagnostics.Contracts; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using Pithos.Interfaces; using Pithos.Network; using log4net; using log4net.Core; namespace Pithos.Core.Agents { // [Export] public class FileAgent { Agent _agent; private FileSystemWatcher _watcher; private FileSystemWatcherAdapter _adapter; //[Import] public IStatusKeeper StatusKeeper { get; set; } //[Import] public IPithosWorkflow Workflow { get; set; } //[Import] public WorkflowAgent WorkflowAgent { get; set; } private AccountInfo AccountInfo { get; set; } private string RootPath { get; set; } private static readonly ILog Log = LogManager.GetLogger("FileAgent"); public void Start(AccountInfo accountInfo,string rootPath) { if (accountInfo==null) throw new ArgumentNullException("accountInfo"); if (String.IsNullOrWhiteSpace(rootPath)) throw new ArgumentNullException("rootPath"); if (!Path.IsPathRooted(rootPath)) throw new ArgumentException("rootPath must be an absolute path","rootPath"); Contract.EndContractBlock(); AccountInfo = accountInfo; RootPath = rootPath; _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true}; _adapter = new FileSystemWatcherAdapter(_watcher); _adapter.Changed += OnFileEvent; _adapter.Created += OnFileEvent; _adapter.Deleted += OnFileEvent; _adapter.Renamed += OnRenameEvent; _adapter.Moved += OnMoveEvent; _watcher.EnableRaisingEvents = true; _agent = Agent.Start(inbox => { Action loop = null; loop = () => { var message = inbox.Receive(); var process=message.Then(Process,inbox.CancellationToken); inbox.LoopAsync(process,loop,ex=> Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex)); }; loop(); }); } private Task Process(WorkflowState state) { if (state==null) throw new ArgumentNullException("state"); Contract.EndContractBlock(); if (Ignore(state.Path)) return CompletedTask.Default; var networkState = NetworkGate.GetNetworkState(state.Path); //Skip if the file is already being downloaded or uploaded and //the change is create or modify if (networkState != NetworkOperation.None && ( state.TriggeringChange == WatcherChangeTypes.Created || state.TriggeringChange == WatcherChangeTypes.Changed )) return CompletedTask.Default; try { UpdateFileStatus(state); UpdateOverlayStatus(state); UpdateFileChecksum(state); WorkflowAgent.Post(state); } catch (IOException exc) { if (File.Exists(state.Path)) { Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc); _agent.Post(state); } else { Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc); } } catch (Exception exc) { Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}", state.Path, exc); } return CompletedTask.Default; } public bool Pause { get { return _watcher == null || !_watcher.EnableRaisingEvents; } set { if (_watcher != null) _watcher.EnableRaisingEvents = !value; } } public string CachePath { get; set; } private List _selectivePaths = new List(); public List SelectivePaths { get { return _selectivePaths; } set { _selectivePaths = value; } } public void Post(WorkflowState workflowState) { if (workflowState == null) throw new ArgumentNullException("workflowState"); Contract.EndContractBlock(); _agent.Post(workflowState); } public void Stop() { if (_watcher != null) { _watcher.Changed -= OnFileEvent; _watcher.Created -= OnFileEvent; _watcher.Deleted -= OnFileEvent; _watcher.Renamed -= OnRenameEvent; _watcher.Dispose(); } _watcher = null; if (_agent!=null) _agent.Stop(); } // Enumerate all files in the Pithos directory except those in the Fragment folder // and files with a .ignore extension public IEnumerable EnumerateFiles(string searchPattern="*") { var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories) where !Ignore(filePath) select filePath; return monitoredFiles; } public IEnumerable EnumerateFileInfos(string searchPattern="*") { var rootDir = new DirectoryInfo(RootPath); var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories) where !Ignore(file.FullName) select file; return monitoredFiles; } public IEnumerable EnumerateFilesAsRelativeUrls(string searchPattern="*") { var rootDir = new DirectoryInfo(RootPath); var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories) where !Ignore(file.FullName) select file.AsRelativeUrlTo(RootPath); return monitoredFiles; } public IEnumerable EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*") { var rootDir = new DirectoryInfo(RootPath); var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories) where !Ignore(file.FullName) select file.AsRelativeUrlTo(RootPath); return monitoredFiles; } private bool Ignore(string filePath) { var pithosPath = Path.Combine(RootPath, "pithos"); if (pithosPath.Equals(filePath, StringComparison.InvariantCultureIgnoreCase)) return true; if (filePath.StartsWith(CachePath)) return true; if (_ignoreFiles.ContainsKey(filePath.ToLower())) return true; return false; } //Post a Change message for all events except rename void OnFileEvent(object sender, FileSystemEventArgs e) { //Ignore events that affect the cache folder var filePath = e.FullPath; if (Ignore(filePath)) return; _agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType }); } //Post a Change message for renames containing the old and new names void OnRenameEvent(object sender, RenamedEventArgs e) { var oldFullPath = e.OldFullPath; var fullPath = e.FullPath; if (Ignore(oldFullPath) || Ignore(fullPath)) return; _agent.Post(new WorkflowState { AccountInfo=AccountInfo, OldPath = oldFullPath, OldFileName = e.OldName, Path = fullPath, FileName = e.Name, TriggeringChange = e.ChangeType }); } //Post a Change message for renames containing the old and new names void OnMoveEvent(object sender, MovedEventArgs e) { var oldFullPath = e.OldFullPath; var fullPath = e.FullPath; if (Ignore(oldFullPath) || Ignore(fullPath)) return; _agent.Post(new WorkflowState { AccountInfo=AccountInfo, OldPath = oldFullPath, OldFileName = e.OldName, Path = fullPath, FileName = e.Name, TriggeringChange = e.ChangeType }); } private Dictionary _statusDict = new Dictionary { {WatcherChangeTypes.Created,FileStatus.Created}, {WatcherChangeTypes.Changed,FileStatus.Modified}, {WatcherChangeTypes.Deleted,FileStatus.Deleted}, {WatcherChangeTypes.Renamed,FileStatus.Renamed} }; private Dictionary _ignoreFiles=new Dictionary(); private WorkflowState UpdateFileStatus(WorkflowState state) { if (state==null) throw new ArgumentNullException("state"); if (String.IsNullOrWhiteSpace(state.Path)) throw new ArgumentException("The state's Path can't be empty","state"); Contract.EndContractBlock(); var path = state.Path; var status = _statusDict[state.TriggeringChange]; var oldStatus = Workflow.StatusKeeper.GetFileStatus(path); if (status == oldStatus) { state.Status = status; state.Skip = true; return state; } if (state.Status == FileStatus.Renamed) Workflow.ClearFileStatus(path); state.Status = Workflow.SetFileStatus(path, status); return state; } private WorkflowState UpdateOverlayStatus(WorkflowState state) { if (state==null) throw new ArgumentNullException("state"); Contract.EndContractBlock(); if (state.Skip) return state; switch (state.Status) { case FileStatus.Created: case FileStatus.Modified: this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified); break; case FileStatus.Deleted: //this.StatusAgent.RemoveFileOverlayStatus(state.Path); break; case FileStatus.Renamed: this.StatusKeeper.ClearFileStatus(state.OldPath); this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified); break; case FileStatus.Unchanged: this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal); break; } if (state.Status == FileStatus.Deleted) NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path)); else NativeMethods.RaiseChangeNotification(state.Path); return state; } private WorkflowState UpdateFileChecksum(WorkflowState state) { if (state.Skip) return state; if (state.Status == FileStatus.Deleted) return state; var path = state.Path; //Skip calculation for folders if (Directory.Exists(path)) return state; var info = new FileInfo(path); string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash); StatusKeeper.UpdateFileChecksum(path, hash); state.Hash = hash; return state; } //Does the file exist in the container's local folder? public bool Exists(string relativePath) { if (String.IsNullOrWhiteSpace(relativePath)) throw new ArgumentNullException("relativePath"); //A RootPath must be set before calling this method if (String.IsNullOrWhiteSpace(RootPath)) throw new InvalidOperationException("RootPath was not set"); Contract.EndContractBlock(); //Create the absolute path by combining the RootPath with the relativePath var absolutePath=Path.Combine(RootPath, relativePath); //Is this a valid file? if (File.Exists(absolutePath)) return true; //Or a directory? if (Directory.Exists(absolutePath)) return true; //Fail if it is neither return false; } public FileSystemInfo GetFileSystemInfo(string relativePath) { if (String.IsNullOrWhiteSpace(relativePath)) throw new ArgumentNullException("relativePath"); //A RootPath must be set before calling this method if (String.IsNullOrWhiteSpace(RootPath)) throw new InvalidOperationException("RootPath was not set"); Contract.EndContractBlock(); var absolutePath = Path.Combine(RootPath, relativePath); if (Directory.Exists(absolutePath)) return new DirectoryInfo(absolutePath).WithProperCapitalization(); else return new FileInfo(absolutePath).WithProperCapitalization(); } public void Delete(string relativePath) { var absolutePath = Path.Combine(RootPath, relativePath).ToLower(); if (File.Exists(absolutePath)) { try { File.Delete(absolutePath); } //The file may have been deleted by another thread. Just ignore the relevant exception catch (FileNotFoundException) { } } else if (Directory.Exists(absolutePath)) { try { Directory.Delete(absolutePath, true); } //The directory may have been deleted by another thread. Just ignore the relevant exception catch (DirectoryNotFoundException){} } //_ignoreFiles[absolutePath] = absolutePath; StatusKeeper.ClearFileStatus(absolutePath); } } }