{
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
- Agent<WorkflowState> _agent;
+ /*
+ Agent<WorkflowState> _agent;
+ */
private FileSystemWatcher _watcher;
private FileSystemWatcherAdapter _adapter;
+ private FileEventIdleBatch _eventIdleBatch;
//[Import]
public IStatusKeeper StatusKeeper { get; set; }
//[Import]
public IPithosWorkflow Workflow { get; set; }
//[Import]
- public WorkflowAgent WorkflowAgent { get; set; }
+ //public WorkflowAgent WorkflowAgent { get; set; }
private AccountInfo AccountInfo { get; set; }
internal string RootPath { get; set; }
- private FileEventIdleBatch _eventIdleBatch;
+ public TimeSpan IdleTimeout { get; set; }
+
+ public PollAgent PollAgent { get; set; }
+
+ private void ProcessBatchedEvents(Dictionary<string, FileSystemEventArgs[]> fileEvents)
+ {
+ var paths = fileEvents.Keys;
- public FileAgent(int idleTimeout)
- {
- _eventIdleBatch=new FileEventIdleBatch(idleTimeout,ProcessBatchedEvents);
+ PollAgent.SynchNow(paths);
}
+/*
private void ProcessBatchedEvents(Dictionary<string, FileSystemEventArgs[]> fileEvents)
{
StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Uploading {0} files",fileEvents.Count));
- foreach (var fileEvent in fileEvents)
+ //Start with events that do not originate in one of the ignored folders
+ var initialEvents = from evt in fileEvents
+ where !IgnorePaths(evt.Key)
+ select evt;
+
+ IEnumerable<KeyValuePair<string, FileSystemEventArgs[]>> cleanEvents;
+
+
+ var selectiveEnabled = Selectives.IsSelectiveEnabled(AccountInfo.AccountKey);
+ //When selective sync is enabled,
+ if (selectiveEnabled)
{
- var filePath = fileEvent.Key;
+ //Include all selected items
+ var selectedEvents = from evt in initialEvents
+ where Selectives.IsSelected(AccountInfo, evt.Key)
+ select evt;
+ //And all folder creations in the unselected folders
+ var folderCreations = from evt in initialEvents
+ let folderPath=evt.Key
+ //The original folder may not exist due to renames. Just make sure that the path is not a file
+ where !File.Exists(folderPath)
+ //We only want unselected items
+ && !Selectives.IsSelected(AccountInfo, folderPath)
+ //Is there any creation event related to the folder?
+ && evt.Value.Any(arg => arg.ChangeType == WatcherChangeTypes.Created)
+ select evt;
+ cleanEvents = selectedEvents.Union(folderCreations).ToList();
+ }
+ //If selective is disabled, only exclude the shared folders
+ else
+ {
+ cleanEvents = (from evt in initialEvents
+ where !evt.Key.IsSharedTo(AccountInfo)
+ select evt).ToList();
+ }
+
+
+ foreach (var fileEvent in cleanEvents)
+ {
+ //var filePath = fileEvent.Key;
var changes = fileEvent.Value;
-
- if (Ignore(filePath)) continue;
-
+
+ var isNotFile = !File.Exists(fileEvent.Key);
foreach (var change in changes)
{
if (change.ChangeType == WatcherChangeTypes.Renamed)
{
- var rename = (RenamedEventArgs) change;
- _agent.Post(new WorkflowState
+ var rename = (MovedEventArgs) change;
+ _agent.Post(new WorkflowState(change)
{
AccountInfo = AccountInfo,
OldPath = rename.OldFullPath,
- OldFileName = rename.OldName,
+ OldFileName = Path.GetFileName(rename.OldName),
Path = rename.FullPath,
- FileName = rename.Name,
+ FileName = Path.GetFileName(rename.Name),
TriggeringChange = rename.ChangeType
});
}
else
- _agent.Post(new WorkflowState
- {
- AccountInfo = AccountInfo,
- Path = change.FullPath,
- FileName = change.Name,
- TriggeringChange = change.ChangeType
- });
+ {
+ var isCreation = selectiveEnabled && isNotFile && change.ChangeType == WatcherChangeTypes.Created;
+ _agent.Post(new WorkflowState(change)
+ {
+ AccountInfo = AccountInfo,
+ Path = change.FullPath,
+ FileName = Path.GetFileName(change.Name),
+ TriggeringChange = change.ChangeType,
+ IsCreation=isCreation
+ });
+ }
}
}
StatusNotification.SetPithosStatus(PithosStatus.LocalComplete);
}
+*/
public void Start(AccountInfo accountInfo,string rootPath)
{
throw new ArgumentNullException("rootPath");
if (!Path.IsPathRooted(rootPath))
throw new ArgumentException("rootPath must be an absolute path","rootPath");
- Contract.EndContractBlock();
+ if (IdleTimeout == null)
+ throw new InvalidOperationException("IdleTimeout must have a valid value");
+ Contract.EndContractBlock();
AccountInfo = accountInfo;
RootPath = rootPath;
- _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true,InternalBufferSize=8*4096};
+
+ _eventIdleBatch = new FileEventIdleBatch((int)IdleTimeout.TotalMilliseconds, ProcessBatchedEvents);
+
+ _watcher = new FileSystemWatcher(rootPath) { IncludeSubdirectories = true, InternalBufferSize = 8 * 4096 };
_adapter = new FileSystemWatcherAdapter(_watcher);
_adapter.Changed += OnFileEvent;
_adapter.Moved += OnMoveEvent;
_watcher.EnableRaisingEvents = true;
+/*
+
+
_agent = Agent<WorkflowState>.Start(inbox =>
{
Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
};
loop();
- });
+ });*/
}
+/*
private Task<object> Process(WorkflowState state)
{
if (state==null)
_watcher.EnableRaisingEvents = !value;
}
}
+*/
public string CachePath { get; set; }
- private List<string> _selectivePaths = new List<string>();
+ /*private List<string> _selectivePaths = new List<string>();
public List<string> SelectivePaths
{
get { return _selectivePaths; }
set { _selectivePaths = value; }
}
+*/
+ public Selectives Selectives { get; set; }
+/*
public void Post(WorkflowState workflowState)
{
if (workflowState == null)
_agent.Stop();
}
+*/
// Enumerate all files in the Pithos directory except those in the Fragment folder
// and files with a .ignore extension
public IEnumerable<string> EnumerateFiles(string searchPattern="*")
{
var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
where !Ignore(filePath)
+ orderby filePath ascending
select filePath;
return monitoredFiles;
}
var rootDir = new DirectoryInfo(RootPath);
var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
where !Ignore(file.FullName)
+ orderby file.FullName ascending
select file;
return monitoredFiles;
}
+ public IEnumerable<FileSystemInfo> EnumerateFileSystemInfos(string searchPattern="*")
+ {
+ var rootDir = new DirectoryInfo(RootPath);
+ //Ensure folders appear first, to allow folder processing as soon as possilbe
+ var folders = (from file in rootDir.EnumerateDirectories(searchPattern, SearchOption.AllDirectories)
+ where !Ignore(file.FullName)
+ orderby file.FullName ascending
+ select file).ToList();
+ var files = (from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
+ where !Ignore(file.FullName)
+ orderby file.Length ascending
+ select file as FileSystemInfo).ToList();
+ var monitoredFiles = folders
+ //Process small files first, leaving expensive large files for last
+ .Concat(files);
+ return monitoredFiles;
+ }
+
public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
{
var rootDir = new DirectoryInfo(RootPath);
var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
where !Ignore(file.FullName)
+ orderby file.FullName ascending
select file.AsRelativeUrlTo(RootPath);
return monitoredFiles;
}
var rootDir = new DirectoryInfo(RootPath);
var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories)
where !Ignore(file.FullName)
+ orderby file.FullName ascending
select file.AsRelativeUrlTo(RootPath);
return monitoredFiles;
}
- private bool Ignore(string filePath)
+ public bool Ignore(string filePath)
{
- //Ignore all first-level directories and files (ie at the container folders level)
- if (FoundBelowRoot(filePath, RootPath,1))
+ if (IgnorePaths(filePath)) return true;
+
+
+ //If selective sync is enabled,
+ if (IsUnselectedRootFolder(filePath))
+ return false;
+ //Ignore if selective synchronization is defined,
+ //And the target file is not below any of the selective paths
+ var ignore = !Selectives.IsSelected(AccountInfo, filePath);
+ return ignore;
+ }
+
+ public bool IsUnselectedRootFolder(string filePath)
+ {
+ return Selectives.IsSelectiveEnabled(AccountInfo.AccountKey) //propagate folder events
+ && Directory.Exists(filePath) //from the container root folder only. Note, in the first level below the account root path are the containers
+ && FoundBelowRoot(filePath, RootPath, 2);
+ }
+
+ public bool IgnorePaths(string filePath)
+ {
+//Ignore all first-level directories and files (ie at the container folders level)
+ if (FoundBelowRoot(filePath, RootPath, 1))
return true;
//Ignore first-level items under the "others" folder (ie at the accounts folders level).
var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder);
- if (FoundBelowRoot(filePath, othersPath,1))
+ if (FoundBelowRoot(filePath, othersPath, 1))
return true;
//Ignore second-level (container) folders under the "others" folder (ie at the container folders level).
- if (FoundBelowRoot(filePath, othersPath,2))
- return true;
+ if (FoundBelowRoot(filePath, othersPath, 2))
+ return true;
//Ignore anything happening in the cache path
if (filePath.StartsWith(CachePath))
return true;
- if (_ignoreFiles.ContainsKey(filePath.ToLower()))
- return true;
-
- //Ignore if selective synchronization is defined,
- return SelectivePaths.Count > 0
- //And the target file is not below any of the selective paths
- && !SelectivePaths.Any(filePath.IsAtOrDirectlyBelow);
+
+ //Finally, ignore events about one of the ignored files
+ return _ignoreFiles.ContainsKey(filePath.ToLower());
}
/* private static bool FoundInRoot(string filePath, string rootPath)
return false;
}
- //Post a Change message for all events except rename
- void OnFileEvent(object sender, FileSystemEventArgs e)
- {
- //Ignore events that affect the cache folder
- var filePath = e.FullPath;
- if (Ignore(filePath))
- return;
- _eventIdleBatch.Post(e);
- }
-
-
-/*
+ /*
//Post a Change message for renames containing the old and new names
void OnRenameEvent(object sender, RenamedEventArgs e)
{
TriggeringChange = e.ChangeType
});
}
-*/
+ */
+
+ //Post a Change message for all events except rename
+ void OnFileEvent(object sender, FileSystemEventArgs e)
+ {
+ //Ignore events that affect the cache folder
+ var filePath = e.FullPath;
+ if (Ignore(filePath))
+ return;
+ _eventIdleBatch.Post(e);
+ }
//Post a Change message for moves containing the old and new names
void OnMoveEvent(object sender, MovedEventArgs e)
{
var oldFullPath = e.OldFullPath;
var fullPath = e.FullPath;
- if (Ignore(oldFullPath) || Ignore(fullPath))
+
+
+ //If the source path is one of the ignored folders, ignore
+ if (IgnorePaths(oldFullPath))
+ return;
+
+ //TODO: Must prevent move propagation if the source folder is blocked by selective sync
+ //Ignore takes into account Selective Sync
+ if (Ignore(fullPath))
return;
_eventIdleBatch.Post(e);
switch (state.Status)
{
case FileStatus.Created:
- this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
+ this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash).Wait();
break;
case FileStatus.Modified:
- this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
+ this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ShortHash).Wait();
break;
case FileStatus.Deleted:
//this.StatusAgent.RemoveFileOverlayStatus(state.Path);
break;
case FileStatus.Renamed:
this.StatusKeeper.ClearFileStatus(state.OldPath);
- this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
+ this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ShortHash).Wait();
break;
case FileStatus.Unchanged:
- this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal,state.ShortHash);
+ this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal, state.ShortHash).Wait();
break;
}
if (Directory.Exists(path))
return state;
-
var info = new FileInfo(path);
- StatusNotification.Notify(new StatusNotification(String.Format("Hashing [{0}]",info.Name)));
- var shortHash = info.ComputeShortHash();
-
- string merkleHash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
- StatusKeeper.UpdateFileChecksum(path,shortHash, merkleHash);
+ using (StatusNotification.GetNotifier("Hashing {0}", "Finished Hashing {0}", info.Name))
+ {
- state.Hash = merkleHash;
- return state;
+ var shortHash = info.ComputeShortHash();
+
+ string merkleHash = info.CalculateHash(StatusKeeper.BlockSize, StatusKeeper.BlockHash);
+ StatusKeeper.UpdateFileChecksum(path, shortHash, merkleHash);
+
+ state.Hash = merkleHash;
+ return state;
+ }
}
//Does the file exist in the container's local folder?
public void Delete(string relativePath)
{
var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
+ if (Log.IsDebugEnabled)
+ Log.DebugFormat("Deleting {0}", absolutePath);
if (File.Exists(absolutePath))
{
try
projects / pithos-ms-client / blobdiff