X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/blobdiff_plain/6b0de454753cbd7b8d994ccfd9ae72ad1200332f..012c853742d106465c7b7f9163d6b30b0bca7b00:/trunk/Pithos.Core/Agents/FileAgent.cs diff --git a/trunk/Pithos.Core/Agents/FileAgent.cs b/trunk/Pithos.Core/Agents/FileAgent.cs index 0ffe721..978ace2 100644 --- a/trunk/Pithos.Core/Agents/FileAgent.cs +++ b/trunk/Pithos.Core/Agents/FileAgent.cs @@ -57,9 +57,12 @@ namespace Pithos.Core.Agents { private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - Agent _agent; + /* + Agent _agent; + */ private FileSystemWatcher _watcher; private FileSystemWatcherAdapter _adapter; + private FileEventIdleBatch _eventIdleBatch; //[Import] public IStatusKeeper StatusKeeper { get; set; } @@ -68,33 +71,76 @@ namespace Pithos.Core.Agents //[Import] public IPithosWorkflow Workflow { get; set; } //[Import] - public WorkflowAgent WorkflowAgent { get; set; } + //public WorkflowAgent WorkflowAgent { get; set; } private AccountInfo AccountInfo { get; set; } internal string RootPath { get; set; } - private FileEventIdleBatch _eventIdleBatch; - public TimeSpan IdleTimeout { get; set; } + public PollAgent PollAgent { get; set; } + + private void ProcessBatchedEvents(Dictionary fileEvents) + { + var paths = fileEvents.Keys; + + PollAgent.SynchNow(paths); + } +/* private void ProcessBatchedEvents(Dictionary fileEvents) { StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Uploading {0} files",fileEvents.Count)); - foreach (var fileEvent in fileEvents) + //Start with events that do not originate in one of the ignored folders + var initialEvents = from evt in fileEvents + where !IgnorePaths(evt.Key) + select evt; + + IEnumerable> cleanEvents; + + + var selectiveEnabled = Selectives.IsSelectiveEnabled(AccountInfo.AccountKey); + //When selective sync is enabled, + if (selectiveEnabled) { - var filePath = fileEvent.Key; + //Include all selected items + var selectedEvents = from evt in initialEvents + where Selectives.IsSelected(AccountInfo, evt.Key) + select evt; + //And all folder creations in the unselected folders + var folderCreations = from evt in initialEvents + let folderPath=evt.Key + //The original folder may not exist due to renames. Just make sure that the path is not a file + where !File.Exists(folderPath) + //We only want unselected items + && !Selectives.IsSelected(AccountInfo, folderPath) + //Is there any creation event related to the folder? + && evt.Value.Any(arg => arg.ChangeType == WatcherChangeTypes.Created) + select evt; + cleanEvents = selectedEvents.Union(folderCreations).ToList(); + } + //If selective is disabled, only exclude the shared folders + else + { + cleanEvents = (from evt in initialEvents + where !evt.Key.IsSharedTo(AccountInfo) + select evt).ToList(); + } + + + foreach (var fileEvent in cleanEvents) + { + //var filePath = fileEvent.Key; var changes = fileEvent.Value; - - if (Ignore(filePath)) continue; - + + var isNotFile = !File.Exists(fileEvent.Key); foreach (var change in changes) { if (change.ChangeType == WatcherChangeTypes.Renamed) { var rename = (MovedEventArgs) change; - _agent.Post(new WorkflowState + _agent.Post(new WorkflowState(change) { AccountInfo = AccountInfo, OldPath = rename.OldFullPath, @@ -105,17 +151,22 @@ namespace Pithos.Core.Agents }); } else - _agent.Post(new WorkflowState - { - AccountInfo = AccountInfo, - Path = change.FullPath, - FileName = Path.GetFileName(change.Name), - TriggeringChange = change.ChangeType - }); + { + var isCreation = selectiveEnabled && isNotFile && change.ChangeType == WatcherChangeTypes.Created; + _agent.Post(new WorkflowState(change) + { + AccountInfo = AccountInfo, + Path = change.FullPath, + FileName = Path.GetFileName(change.Name), + TriggeringChange = change.ChangeType, + IsCreation=isCreation + }); + } } } StatusNotification.SetPithosStatus(PithosStatus.LocalComplete); } +*/ public void Start(AccountInfo accountInfo,string rootPath) { @@ -127,14 +178,14 @@ namespace Pithos.Core.Agents throw new ArgumentException("rootPath must be an absolute path","rootPath"); if (IdleTimeout == null) throw new InvalidOperationException("IdleTimeout must have a valid value"); - Contract.EndContractBlock(); + Contract.EndContractBlock(); AccountInfo = accountInfo; RootPath = rootPath; - + _eventIdleBatch = new FileEventIdleBatch((int)IdleTimeout.TotalMilliseconds, ProcessBatchedEvents); - _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true,InternalBufferSize=8*4096}; + _watcher = new FileSystemWatcher(rootPath) { IncludeSubdirectories = true, InternalBufferSize = 8 * 4096 }; _adapter = new FileSystemWatcherAdapter(_watcher); _adapter.Changed += OnFileEvent; @@ -144,6 +195,9 @@ namespace Pithos.Core.Agents _adapter.Moved += OnMoveEvent; _watcher.EnableRaisingEvents = true; +/* + + _agent = Agent.Start(inbox => { @@ -156,9 +210,10 @@ namespace Pithos.Core.Agents Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex)); }; loop(); - }); + });*/ } +/* private Task Process(WorkflowState state) { if (state==null) @@ -216,17 +271,21 @@ namespace Pithos.Core.Agents _watcher.EnableRaisingEvents = !value; } } +*/ public string CachePath { get; set; } - private List _selectivePaths = new List(); + /*private List _selectivePaths = new List(); public List SelectivePaths { get { return _selectivePaths; } set { _selectivePaths = value; } } +*/ + public Selectives Selectives { get; set; } +/* public void Post(WorkflowState workflowState) { if (workflowState == null) @@ -248,12 +307,14 @@ namespace Pithos.Core.Agents _agent.Stop(); } +*/ // Enumerate all files in the Pithos directory except those in the Fragment folder // and files with a .ignore extension public IEnumerable EnumerateFiles(string searchPattern="*") { var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories) where !Ignore(filePath) + orderby filePath ascending select filePath; return monitoredFiles; } @@ -263,15 +324,35 @@ namespace Pithos.Core.Agents var rootDir = new DirectoryInfo(RootPath); var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories) where !Ignore(file.FullName) + orderby file.FullName ascending select file; return monitoredFiles; } + public IEnumerable EnumerateFileSystemInfos(string searchPattern="*") + { + var rootDir = new DirectoryInfo(RootPath); + //Ensure folders appear first, to allow folder processing as soon as possilbe + var folders = (from file in rootDir.EnumerateDirectories(searchPattern, SearchOption.AllDirectories) + where !Ignore(file.FullName) + orderby file.FullName ascending + select file).ToList(); + var files = (from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories) + where !Ignore(file.FullName) + orderby file.Length ascending + select file as FileSystemInfo).ToList(); + var monitoredFiles = folders + //Process small files first, leaving expensive large files for last + .Concat(files); + return monitoredFiles; + } + public IEnumerable EnumerateFilesAsRelativeUrls(string searchPattern="*") { var rootDir = new DirectoryInfo(RootPath); var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories) where !Ignore(file.FullName) + orderby file.FullName ascending select file.AsRelativeUrlTo(RootPath); return monitoredFiles; } @@ -281,6 +362,7 @@ namespace Pithos.Core.Agents var rootDir = new DirectoryInfo(RootPath); var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories) where !Ignore(file.FullName) + orderby file.FullName ascending select file.AsRelativeUrlTo(RootPath); return monitoredFiles; } @@ -288,32 +370,49 @@ namespace Pithos.Core.Agents - private bool Ignore(string filePath) + public bool Ignore(string filePath) { - //Ignore all first-level directories and files (ie at the container folders level) - if (FoundBelowRoot(filePath, RootPath,1)) + if (IgnorePaths(filePath)) return true; + + + //If selective sync is enabled, + if (IsUnselectedRootFolder(filePath)) + return false; + //Ignore if selective synchronization is defined, + //And the target file is not below any of the selective paths + var ignore = !Selectives.IsSelected(AccountInfo, filePath); + return ignore; + } + + public bool IsUnselectedRootFolder(string filePath) + { + return Selectives.IsSelectiveEnabled(AccountInfo.AccountKey) //propagate folder events + && Directory.Exists(filePath) //from the container root folder only. Note, in the first level below the account root path are the containers + && FoundBelowRoot(filePath, RootPath, 2); + } + + public bool IgnorePaths(string filePath) + { +//Ignore all first-level directories and files (ie at the container folders level) + if (FoundBelowRoot(filePath, RootPath, 1)) return true; //Ignore first-level items under the "others" folder (ie at the accounts folders level). var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder); - if (FoundBelowRoot(filePath, othersPath,1)) + if (FoundBelowRoot(filePath, othersPath, 1)) return true; //Ignore second-level (container) folders under the "others" folder (ie at the container folders level). - if (FoundBelowRoot(filePath, othersPath,2)) - return true; + if (FoundBelowRoot(filePath, othersPath, 2)) + return true; //Ignore anything happening in the cache path if (filePath.StartsWith(CachePath)) return true; - if (_ignoreFiles.ContainsKey(filePath.ToLower())) - return true; - - //Ignore if selective synchronization is defined, - return SelectivePaths.Count > 0 - //And the target file is not below any of the selective paths - && !SelectivePaths.Any(filePath.IsAtOrDirectlyBelow); + + //Finally, ignore events about one of the ignored files + return _ignoreFiles.ContainsKey(filePath.ToLower()); } /* private static bool FoundInRoot(string filePath, string rootPath) @@ -360,18 +459,7 @@ namespace Pithos.Core.Agents return false; } - //Post a Change message for all events except rename - void OnFileEvent(object sender, FileSystemEventArgs e) - { - //Ignore events that affect the cache folder - var filePath = e.FullPath; - if (Ignore(filePath)) - return; - _eventIdleBatch.Post(e); - } - - -/* + /* //Post a Change message for renames containing the old and new names void OnRenameEvent(object sender, RenamedEventArgs e) { @@ -390,14 +478,32 @@ namespace Pithos.Core.Agents TriggeringChange = e.ChangeType }); } -*/ + */ + + //Post a Change message for all events except rename + void OnFileEvent(object sender, FileSystemEventArgs e) + { + //Ignore events that affect the cache folder + var filePath = e.FullPath; + if (Ignore(filePath)) + return; + _eventIdleBatch.Post(e); + } //Post a Change message for moves containing the old and new names void OnMoveEvent(object sender, MovedEventArgs e) { var oldFullPath = e.OldFullPath; var fullPath = e.FullPath; - if (Ignore(oldFullPath) || Ignore(fullPath)) + + + //If the source path is one of the ignored folders, ignore + if (IgnorePaths(oldFullPath)) + return; + + //TODO: Must prevent move propagation if the source folder is blocked by selective sync + //Ignore takes into account Selective Sync + if (Ignore(fullPath)) return; _eventIdleBatch.Post(e); @@ -451,20 +557,20 @@ namespace Pithos.Core.Agents switch (state.Status) { case FileStatus.Created: - this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash); + this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash).Wait(); break; case FileStatus.Modified: - this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash); + this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ShortHash).Wait(); break; case FileStatus.Deleted: //this.StatusAgent.RemoveFileOverlayStatus(state.Path); break; case FileStatus.Renamed: this.StatusKeeper.ClearFileStatus(state.OldPath); - this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash); + this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ShortHash).Wait(); break; case FileStatus.Unchanged: - this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal,state.ShortHash); + this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal, state.ShortHash).Wait(); break; } @@ -489,17 +595,19 @@ namespace Pithos.Core.Agents if (Directory.Exists(path)) return state; - var info = new FileInfo(path); - StatusNotification.Notify(new StatusNotification(String.Format("Hashing [{0}]",info.Name))); - var shortHash = info.ComputeShortHash(); - - string merkleHash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash); - StatusKeeper.UpdateFileChecksum(path,shortHash, merkleHash); + using (StatusNotification.GetNotifier("Hashing {0}", "Finished Hashing {0}", info.Name)) + { - state.Hash = merkleHash; - return state; + var shortHash = info.ComputeShortHash(); + + string merkleHash = info.CalculateHash(StatusKeeper.BlockSize, StatusKeeper.BlockHash); + StatusKeeper.UpdateFileChecksum(path, shortHash, merkleHash); + + state.Hash = merkleHash; + return state; + } } //Does the file exist in the container's local folder? @@ -555,6 +663,8 @@ namespace Pithos.Core.Agents public void Delete(string relativePath) { var absolutePath = Path.Combine(RootPath, relativePath).ToLower(); + if (Log.IsDebugEnabled) + Log.DebugFormat("Deleting {0}", absolutePath); if (File.Exists(absolutePath)) { try