#region
/* -----------------------------------------------------------------------
*
*
* Copyright 2011-2012 GRNET S.A. All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* 1. Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* 2. Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
*
* THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and
* documentation are those of the authors and should not be
* interpreted as representing official policies, either expressed
* or implied, of GRNET S.A.
*
* -----------------------------------------------------------------------
*/
#endregion
using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.Diagnostics.Contracts;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Pithos.Interfaces;
using Pithos.Network;
using log4net;
using log4net.Core;
namespace Pithos.Core.Agents
{
// [Export]
public class FileAgent
{
Agent _agent;
private FileSystemWatcher _watcher;
private FileSystemWatcherAdapter _adapter;
//[Import]
public IStatusKeeper StatusKeeper { get; set; }
//[Import]
public IPithosWorkflow Workflow { get; set; }
//[Import]
public WorkflowAgent WorkflowAgent { get; set; }
private AccountInfo AccountInfo { get; set; }
private string RootPath { get; set; }
private static readonly ILog Log = LogManager.GetLogger("FileAgent");
public void Start(AccountInfo accountInfo,string rootPath)
{
if (accountInfo==null)
throw new ArgumentNullException("accountInfo");
if (String.IsNullOrWhiteSpace(rootPath))
throw new ArgumentNullException("rootPath");
if (!Path.IsPathRooted(rootPath))
throw new ArgumentException("rootPath must be an absolute path","rootPath");
Contract.EndContractBlock();
AccountInfo = accountInfo;
RootPath = rootPath;
_watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true};
_adapter = new FileSystemWatcherAdapter(_watcher);
_adapter.Changed += OnFileEvent;
_adapter.Created += OnFileEvent;
_adapter.Deleted += OnFileEvent;
_adapter.Renamed += OnRenameEvent;
_adapter.Moved += OnMoveEvent;
_watcher.EnableRaisingEvents = true;
_agent = Agent.Start(inbox =>
{
Action loop = null;
loop = () =>
{
var message = inbox.Receive();
var process=message.Then(Process,inbox.CancellationToken);
inbox.LoopAsync(process,loop,ex=>
Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
};
loop();
});
}
private Task Process(WorkflowState state)
{
if (state==null)
throw new ArgumentNullException("state");
Contract.EndContractBlock();
if (Ignore(state.Path))
return CompletedTask.Default;
var networkState = NetworkGate.GetNetworkState(state.Path);
//Skip if the file is already being downloaded or uploaded and
//the change is create or modify
if (networkState != NetworkOperation.None &&
(
state.TriggeringChange == WatcherChangeTypes.Created ||
state.TriggeringChange == WatcherChangeTypes.Changed
))
return CompletedTask.Default;
try
{
UpdateFileStatus(state);
UpdateOverlayStatus(state);
UpdateFileChecksum(state);
WorkflowAgent.Post(state);
}
catch (IOException exc)
{
if (File.Exists(state.Path))
{
Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
_agent.Post(state);
}
else
{
Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
}
}
catch (Exception exc)
{
Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
state.Path, exc);
}
return CompletedTask.Default;
}
public bool Pause
{
get { return _watcher == null || !_watcher.EnableRaisingEvents; }
set
{
if (_watcher != null)
_watcher.EnableRaisingEvents = !value;
}
}
public string CachePath { get; set; }
private List _selectivePaths = new List();
public List SelectivePaths
{
get { return _selectivePaths; }
set { _selectivePaths = value; }
}
public void Post(WorkflowState workflowState)
{
if (workflowState == null)
throw new ArgumentNullException("workflowState");
Contract.EndContractBlock();
_agent.Post(workflowState);
}
public void Stop()
{
if (_watcher != null)
{
_watcher.Changed -= OnFileEvent;
_watcher.Created -= OnFileEvent;
_watcher.Deleted -= OnFileEvent;
_watcher.Renamed -= OnRenameEvent;
_watcher.Dispose();
}
_watcher = null;
if (_agent!=null)
_agent.Stop();
}
// Enumerate all files in the Pithos directory except those in the Fragment folder
// and files with a .ignore extension
public IEnumerable EnumerateFiles(string searchPattern="*")
{
var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
where !Ignore(filePath)
select filePath;
return monitoredFiles;
}
public IEnumerable EnumerateFileInfos(string searchPattern="*")
{
var rootDir = new DirectoryInfo(RootPath);
var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
where !Ignore(file.FullName)
select file;
return monitoredFiles;
}
public IEnumerable EnumerateFilesAsRelativeUrls(string searchPattern="*")
{
var rootDir = new DirectoryInfo(RootPath);
var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
where !Ignore(file.FullName)
select file.AsRelativeUrlTo(RootPath);
return monitoredFiles;
}
public IEnumerable EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*")
{
var rootDir = new DirectoryInfo(RootPath);
var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories)
where !Ignore(file.FullName)
select file.AsRelativeUrlTo(RootPath);
return monitoredFiles;
}
private bool Ignore(string filePath)
{
//Ignore all first-level directories and files (ie at the container folders level)
if (FoundBelowRoot(filePath, RootPath,1))
return true;
//Ignore first-level items under the "others" folder (ie at the accounts folders level).
var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder);
if (FoundBelowRoot(filePath, othersPath,1))
return true;
//Ignore second-level (container) folders under the "others" folder (ie at the container folders level).
if (FoundBelowRoot(filePath, othersPath,2))
return true;
//Ignore anything happening in the cache path
if (filePath.StartsWith(CachePath))
return true;
if (_ignoreFiles.ContainsKey(filePath.ToLower()))
return true;
//Ignore if selective synchronization is defined,
return SelectivePaths.Count > 0
//And the target file is not below any of the selective paths
&& !SelectivePaths.Any(filePath.IsAtOrDirectlyBelow);
}
/* private static bool FoundInRoot(string filePath, string rootPath)
{
//var rootDirectory = new DirectoryInfo(rootPath);
//If the paths are equal, return true
if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
return true;
//If the filepath is below the root path
if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
{
//Get the relative path
var relativePath = filePath.Substring(rootPath.Length + 1);
//If the relativePath does NOT contains a path separator, we found a match
return (!relativePath.Contains(@"\"));
}
//If the filepath is not under the root path, return false
return false;
}*/
private static bool FoundBelowRoot(string filePath, string rootPath,int level)
{
//var rootDirectory = new DirectoryInfo(rootPath);
//If the paths are equal, return true
if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
return true;
//If the filepath is below the root path
if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
{
//Get the relative path
var relativePath = filePath.Substring(rootPath.Length + 1);
//If the relativePath does NOT contains a path separator, we found a match
var levels=relativePath.ToCharArray().Count(c=>c=='\\')+1;
return levels==level;
}
//If the filepath is not under the root path, return false
return false;
}
//Post a Change message for all events except rename
void OnFileEvent(object sender, FileSystemEventArgs e)
{
//Ignore events that affect the cache folder
var filePath = e.FullPath;
if (Ignore(filePath))
return;
_agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
}
//Post a Change message for renames containing the old and new names
void OnRenameEvent(object sender, RenamedEventArgs e)
{
var oldFullPath = e.OldFullPath;
var fullPath = e.FullPath;
if (Ignore(oldFullPath) || Ignore(fullPath))
return;
_agent.Post(new WorkflowState
{
AccountInfo=AccountInfo,
OldPath = oldFullPath,
OldFileName = e.OldName,
Path = fullPath,
FileName = e.Name,
TriggeringChange = e.ChangeType
});
}
//Post a Change message for renames containing the old and new names
void OnMoveEvent(object sender, MovedEventArgs e)
{
var oldFullPath = e.OldFullPath;
var fullPath = e.FullPath;
if (Ignore(oldFullPath) || Ignore(fullPath))
return;
_agent.Post(new WorkflowState
{
AccountInfo=AccountInfo,
OldPath = oldFullPath,
OldFileName = e.OldName,
Path = fullPath,
FileName = e.Name,
TriggeringChange = e.ChangeType
});
}
private Dictionary _statusDict = new Dictionary
{
{WatcherChangeTypes.Created,FileStatus.Created},
{WatcherChangeTypes.Changed,FileStatus.Modified},
{WatcherChangeTypes.Deleted,FileStatus.Deleted},
{WatcherChangeTypes.Renamed,FileStatus.Renamed}
};
private Dictionary _ignoreFiles=new Dictionary();
private WorkflowState UpdateFileStatus(WorkflowState state)
{
if (state==null)
throw new ArgumentNullException("state");
if (String.IsNullOrWhiteSpace(state.Path))
throw new ArgumentException("The state's Path can't be empty","state");
Contract.EndContractBlock();
var path = state.Path;
var status = _statusDict[state.TriggeringChange];
var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
if (status == oldStatus)
{
state.Status = status;
state.Skip = true;
return state;
}
if (state.Status == FileStatus.Renamed)
Workflow.ClearFileStatus(path);
state.Status = Workflow.SetFileStatus(path, status);
return state;
}
private WorkflowState UpdateOverlayStatus(WorkflowState state)
{
if (state==null)
throw new ArgumentNullException("state");
Contract.EndContractBlock();
if (state.Skip)
return state;
switch (state.Status)
{
case FileStatus.Created:
case FileStatus.Modified:
this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
break;
case FileStatus.Deleted:
//this.StatusAgent.RemoveFileOverlayStatus(state.Path);
break;
case FileStatus.Renamed:
this.StatusKeeper.ClearFileStatus(state.OldPath);
this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
break;
case FileStatus.Unchanged:
this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
break;
}
if (state.Status == FileStatus.Deleted)
NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
else
NativeMethods.RaiseChangeNotification(state.Path);
return state;
}
private WorkflowState UpdateFileChecksum(WorkflowState state)
{
if (state.Skip)
return state;
if (state.Status == FileStatus.Deleted)
return state;
var path = state.Path;
//Skip calculation for folders
if (Directory.Exists(path))
return state;
var info = new FileInfo(path);
string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
StatusKeeper.UpdateFileChecksum(path, hash);
state.Hash = hash;
return state;
}
//Does the file exist in the container's local folder?
public bool Exists(string relativePath)
{
if (String.IsNullOrWhiteSpace(relativePath))
throw new ArgumentNullException("relativePath");
//A RootPath must be set before calling this method
if (String.IsNullOrWhiteSpace(RootPath))
throw new InvalidOperationException("RootPath was not set");
Contract.EndContractBlock();
//Create the absolute path by combining the RootPath with the relativePath
var absolutePath=Path.Combine(RootPath, relativePath);
//Is this a valid file?
if (File.Exists(absolutePath))
return true;
//Or a directory?
if (Directory.Exists(absolutePath))
return true;
//Fail if it is neither
return false;
}
public static FileAgent GetFileAgent(AccountInfo accountInfo)
{
return GetFileAgent(accountInfo.AccountPath);
}
public static FileAgent GetFileAgent(string rootPath)
{
return AgentLocator.Get(rootPath.ToLower());
}
public FileSystemInfo GetFileSystemInfo(string relativePath)
{
if (String.IsNullOrWhiteSpace(relativePath))
throw new ArgumentNullException("relativePath");
//A RootPath must be set before calling this method
if (String.IsNullOrWhiteSpace(RootPath))
throw new InvalidOperationException("RootPath was not set");
Contract.EndContractBlock();
var absolutePath = Path.Combine(RootPath, relativePath);
if (Directory.Exists(absolutePath))
return new DirectoryInfo(absolutePath).WithProperCapitalization();
else
return new FileInfo(absolutePath).WithProperCapitalization();
}
public void Delete(string relativePath)
{
var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
if (File.Exists(absolutePath))
{
try
{
File.Delete(absolutePath);
}
//The file may have been deleted by another thread. Just ignore the relevant exception
catch (FileNotFoundException) { }
}
else if (Directory.Exists(absolutePath))
{
try
{
Directory.Delete(absolutePath, true);
}
//The directory may have been deleted by another thread. Just ignore the relevant exception
catch (DirectoryNotFoundException){}
}
//_ignoreFiles[absolutePath] = absolutePath;
StatusKeeper.ClearFileStatus(absolutePath);
}
}
}