From fec5da0698201abe1c169691fab663a53113fc2e Mon Sep 17 00:00:00 2001 From: Panagiotis Kanavos Date: Wed, 22 Feb 2012 17:57:18 +0200 Subject: [PATCH] File and polling filtering work properly --- .../SelectiveSynch/SelectiveSynchViewModel.cs | 30 +++---- trunk/Pithos.Core/Agents/CollectionExtensions.cs | 86 +++++++++++++++++++- trunk/Pithos.Core/Agents/FileAgent.cs | 13 +-- trunk/Pithos.Core/Agents/PollAgent.cs | 59 ++++++++++---- trunk/Pithos.Core/FileState.cs | 36 +++++--- trunk/Pithos.Core/PithosMonitor.cs | 39 ++++++--- 6 files changed, 194 insertions(+), 69 deletions(-) diff --git a/trunk/Pithos.Client.WPF/SelectiveSynch/SelectiveSynchViewModel.cs b/trunk/Pithos.Client.WPF/SelectiveSynch/SelectiveSynchViewModel.cs index 2539edb..c354066 100644 --- a/trunk/Pithos.Client.WPF/SelectiveSynch/SelectiveSynchViewModel.cs +++ b/trunk/Pithos.Client.WPF/SelectiveSynch/SelectiveSynchViewModel.cs @@ -117,6 +117,7 @@ namespace Pithos.Client.WPF.SelectiveSynch select new DirectoryRecord { DisplayName = container.Name, + Uri=new Uri(client.StorageUrl,container.Name), Directories = (from dir in client.ListObjects(_monitor.UserName, container.Name, "") where dir.Content_Type == DirectoryType select new DirectoryRecord { DisplayName = dir.Name, ObjectInfo = dir }).ToList() @@ -193,9 +194,12 @@ namespace Pithos.Client.WPF.SelectiveSynch public void SaveChanges() { - var selections = GetSelectedFolderNames(); + var uris = (from root in RootNodes + from record in root + where record.IsChecked == true && record.Uri != null + select record.Uri).ToArray(); - SaveSettings(selections); + SaveSettings(uris); //RootNodes is an ObservableCollection, it can't be enumerated iterativelly @@ -207,10 +211,7 @@ namespace Pithos.Client.WPF.SelectiveSynch from record in root where record.Removed && record.Uri != null select record.Uri).ToArray(); - var uris = (from root in RootNodes - from record in root - where record.IsChecked==true && record.Uri != null - select record.Uri).ToArray(); + //TODO: Include Uris for the containers as well _events.Publish(new SelectiveSynchChanges{Account=Account,Uris=uris,Added=added,Removed=removed}); @@ -220,23 +221,14 @@ namespace Pithos.Client.WPF.SelectiveSynch } - private void SaveSettings(string[] selections) + private void SaveSettings(IEnumerable uris) { + var selections = uris.Select(uri => uri.ToString()).ToArray(); + Account.SelectiveFolders.Clear(); Account.SelectiveFolders.AddRange(selections); Settings.Default.Save(); - } - - private string[] GetSelectedFolderNames() - { - - var selections = from node in RootNodes - from childNode in node - where childNode.ObjectInfo != null - && childNode.IsChecked == true - select node.ObjectInfo.Uri.ToString(); - return selections.ToArray(); - } + } public void RejectChanges() { diff --git a/trunk/Pithos.Core/Agents/CollectionExtensions.cs b/trunk/Pithos.Core/Agents/CollectionExtensions.cs index 4138953..84549fd 100644 --- a/trunk/Pithos.Core/Agents/CollectionExtensions.cs +++ b/trunk/Pithos.Core/Agents/CollectionExtensions.cs @@ -40,6 +40,7 @@ */ #endregion using System.Collections.Concurrent; +using System.Diagnostics.Contracts; using Pithos.Interfaces; namespace Pithos.Core.Agents @@ -90,11 +91,90 @@ namespace Pithos.Core.Agents return infos; if (filterUris.Count == 0) return infos; + //Allow all objects whose Uris start with any of the filters var filteredUris = from info in infos - where !filterUris.Any(s => info.Uri.ToString() - .StartsWith(s.ToString())) - select info; + where filterUris.Any(f => info.Uri.IsAtOrBelow(f)) + select info; return filteredUris; } + + public static IEnumerable FilterDirectlyBelow(this IEnumerable infos,List filterUris ) + { + if (filterUris == null) + return infos; + if (filterUris.Count == 0) + return infos; + //Allow all objects whose Uris start with any of the filters + var filteredUris = from info in infos + where filterUris.Any(f => info.Uri.IsAtOrDirectlyBelow(f)) + select info; + return filteredUris; + } + + public static bool IsAtOrBelow(this Uri target,Uri root) + { + Contract.Requires(root != null); + Contract.Requires(target != null); + + var targetSegments = target.Segments; + var rootSegments = root.Segments; + + return InnerAtOrBelow(targetSegments, rootSegments); + } + + private static bool InnerAtOrBelow(string[] targetSegments, string[] rootSegments) + { + //If the uri is shorter than the root, no point in comparing + if (targetSegments.Length < rootSegments.Length) + return false; + //If the uri is below the root, it should match the root's segments one by one + //If there is any root segment that doesn't match its corresponding target segment, + //the target is not below the root + var mismatch = rootSegments + .Where((t, i) => !String.Equals(targetSegments[i], t)) + .Any(); + return !mismatch; + } + + + public static bool IsAtOrDirectlyBelow(this Uri target,Uri root) + { + Contract.Requires(root!=null); + Contract.Requires(target!=null); + + return + //If the target is directly below the root, it will have exactly + //one segment more than the root + target.Segments.Length == root.Segments.Length + 1 + //Ensure that the candidate target is indeed below the root + && target.IsAtOrBelow(root); + } + + public static bool IsAtOrBelow(this string targetPath, string rootPath) + { + Contract.Requires(!String.IsNullOrWhiteSpace(targetPath)); + Contract.Requires(!String.IsNullOrWhiteSpace(rootPath)); + + var targetSegments = targetPath.Split('\\'); + var rootSegments = rootPath.Split('\\'); + + return InnerAtOrBelow(targetSegments, rootSegments); + } + + public static bool IsAtOrDirectlyBelow(this string targetPath, string rootPath) + { + Contract.Requires(!String.IsNullOrWhiteSpace(targetPath)); + Contract.Requires(!String.IsNullOrWhiteSpace(rootPath)); + + var targetSegments = targetPath.Split('\\'); + var rootSegments = rootPath.Split('\\'); + + return + //If the target is directly below the root, it will have exactly + //one segment more than the root + targetSegments.Length == rootSegments.Length + 1 + //Ensure that the candidate target is indeed below the root + && InnerAtOrBelow(targetSegments, rootSegments); + } } } diff --git a/trunk/Pithos.Core/Agents/FileAgent.cs b/trunk/Pithos.Core/Agents/FileAgent.cs index f7c4b5b..5f001e3 100644 --- a/trunk/Pithos.Core/Agents/FileAgent.cs +++ b/trunk/Pithos.Core/Agents/FileAgent.cs @@ -265,15 +265,10 @@ namespace Pithos.Core.Agents if (_ignoreFiles.ContainsKey(filePath.ToLower())) return true; - //If selective synchronization is defined, - if (SelectivePaths.Count>0) - { - //Abort if the file is not located under any of the selected folders - if (!SelectivePaths.Any(filePath.StartsWith)) - return true; - } - - return false; + //Ignore if selective synchronization is defined, + return SelectivePaths.Count > 0 + //And the target file is not below any of the selective paths + && !SelectivePaths.Any(filePath.IsAtOrDirectlyBelow); } /* private static bool FoundInRoot(string filePath, string rootPath) diff --git a/trunk/Pithos.Core/Agents/PollAgent.cs b/trunk/Pithos.Core/Agents/PollAgent.cs index b1e1d0e..28308f6 100644 --- a/trunk/Pithos.Core/Agents/PollAgent.cs +++ b/trunk/Pithos.Core/Agents/PollAgent.cs @@ -99,7 +99,11 @@ namespace Pithos.Core.Agents _syncEvent.Set(); } - //Remote files are polled periodically. Any changes are processed + /// + /// Remote files are polled periodically. Any changes are processed + /// + /// + /// public async Task PollRemoteFiles(DateTime? since = null) { Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!"); @@ -133,11 +137,19 @@ namespace Pithos.Core.Agents } UpdateStatus(PithosStatus.InSynch); - //Wait for the polling interval to pass or the Sync event to be signalled - nextSince = await WaitForScheduledOrManualPoll(nextSince); - - TaskEx.Run(()=>PollRemoteFiles(nextSince)); - + //The multiple try blocks are required because we can't have an await call + //inside a finally block + //TODO: Find a more elegant solution for reschedulling in the event of an exception + try + { + //Wait for the polling interval to pass or the Sync event to be signalled + nextSince = await WaitForScheduledOrManualPoll(nextSince); + } + finally + { + //Ensure polling is scheduled even in case of error + TaskEx.Run(() => PollRemoteFiles(nextSince)); + } } } @@ -243,16 +255,16 @@ namespace Pithos.Core.Agents var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes); - ProcessDeletedFiles(accountInfo, differencer.Deleted.FilterBelow(SelectiveUris), pollTime); + ProcessDeletedFiles(accountInfo, differencer.Deleted.FilterDirectlyBelow(SelectiveUris), pollTime); // @@@ NEED To add previous state here as well, To compare with previous hash //Create a list of actions from the remote files - var allActions = ChangesToActions(accountInfo, differencer.Changed.FilterBelow(SelectiveUris)) + var allActions = ChangesToActions(accountInfo, differencer.Changed.FilterDirectlyBelow(SelectiveUris)) .Union( - CreatesToActions(accountInfo, differencer.Created.FilterBelow(SelectiveUris))); + CreatesToActions(accountInfo, differencer.Created.FilterDirectlyBelow(SelectiveUris))); //And remove those that are already being processed by the agent var distinctActions = allActions @@ -439,10 +451,22 @@ namespace Pithos.Core.Agents } } + /// + /// Notify the UI to update the visual status + /// + /// private void UpdateStatus(PithosStatus status) { - StatusKeeper.SetPithosStatus(status); - StatusNotification.Notify(new Notification()); + try + { + StatusKeeper.SetPithosStatus(status); + StatusNotification.Notify(new Notification()); + } + catch (Exception exc) + { + //Failure is not critical, just log it + Log.Warn("Error while updating status", exc); + } } private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable containers) @@ -458,10 +482,9 @@ namespace Pithos.Core.Agents } } - public void SetSyncUris(string[] uris) - { - var selectiveUris = uris.Select(uri => new Uri(uri)); - SelectiveUris=selectiveUris.ToList(); + public void SetSyncUris(Uri[] uris) + { + SelectiveUris=uris.ToList(); } protected List SelectiveUris @@ -469,5 +492,11 @@ namespace Pithos.Core.Agents get { return _selectiveUris;} set { _selectiveUris = value; } } + + public void AddAccount(AccountInfo accountInfo) + { + if (!_accounts.Contains(accountInfo)) + _accounts.Add(accountInfo); + } } } diff --git a/trunk/Pithos.Core/FileState.cs b/trunk/Pithos.Core/FileState.cs index 4a764c5..559d584 100644 --- a/trunk/Pithos.Core/FileState.cs +++ b/trunk/Pithos.Core/FileState.cs @@ -54,8 +54,7 @@ using log4net; namespace Pithos.Core { using System; - using System.Collections.Generic; - using System.Linq; + using System.Collections.Generic; /// /// TODO: Update summary. @@ -420,23 +419,36 @@ namespace Pithos.Core } } - public static void RemovePaths(IEnumerable removed) + /// + /// Remove all FileState rows from the database whose path + /// starts with one of the removed paths + /// + /// + public static void RemovePaths(List removed) { - - var disjunction = new Disjunction(); + if (removed == null) + return; + if (removed.Count == 0) + return; + //Create a disjunction (list of OR statements + var disjunction = new Disjunction(); foreach (var path in removed) { - disjunction.Add(Restrictions.On(s => s.FilePath).IsLike(path, MatchMode.Start)); + //with the restriction FileState.FilePath like '@path%' + disjunction.Add(Restrictions.On(s => s.FilePath) + .IsLike(path, MatchMode.Start)); } - - + //Generate a query from the disjunction var query=QueryOver.Of().Where(disjunction); - var aq = new ProjectionQuery(query.DetachedCriteria, - Projections.ProjectionList().Add(Projections.Id())); - var ids=aq.Execute(); - FileState.DeleteAll(ids); + //Adn retrieveonly the IDs + var idQuery = new ProjectionQuery(query.DetachedCriteria, + Projections.ProjectionList() + .Add(Projections.Id())); + + var ids=idQuery.Execute(); + DeleteAll(ids); } } diff --git a/trunk/Pithos.Core/PithosMonitor.cs b/trunk/Pithos.Core/PithosMonitor.cs index 6c18da1..4e8e393 100644 --- a/trunk/Pithos.Core/PithosMonitor.cs +++ b/trunk/Pithos.Core/PithosMonitor.cs @@ -240,8 +240,7 @@ namespace Pithos.Core IndexLocalFiles(); //Extract the URIs from the string collection var settings = Settings.Accounts.First(s => s.AccountName == _accountInfo.UserName); - var selectiveUrls=new string[settings.SelectiveFolders.Count]; - settings.SelectiveFolders.CopyTo(selectiveUrls,0); + var selectiveUrls=settings.SelectiveFolders.Cast().Select(url => new Uri(url)).ToArray(); SetSelectivePaths(selectiveUrls,null,null); @@ -378,6 +377,8 @@ namespace Pithos.Core NetworkAgent.Start(); + PollAgent.AddAccount(_accountInfo); + PollAgent.StatusNotification = StatusNotification; PollAgent.PollRemoteFiles(); @@ -475,25 +476,41 @@ namespace Pithos.Core StatusKeeper.ChangeRoots(oldPath, newPath); } - public void SetSelectivePaths(string[] uris,string[] added, string[] removed) + public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed) { //Convert the uris to paths - var selectivePaths = (from string selectiveUrl in uris - select new Uri(selectiveUrl) - .MakeRelativeUri(_accountInfo.StorageUri) - .RelativeUriToFilePath()); - - FileAgent.SelectivePaths=selectivePaths.ToList(); + var selectivePaths = UrisToFilePaths(uris); + + FileAgent.SelectivePaths=selectivePaths; PollAgent.SetSyncUris(uris); - RemoveSelectivePaths(removed); + var removedPaths = UrisToFilePaths(removed); + RemoveSelectivePaths(removedPaths); + + } + + /// + /// Return a list of absolute filepaths from a list of Uris + /// + /// + /// + private List UrisToFilePaths(IEnumerable uris) + { + if (uris == null) + return new List(); + + return (from uri in uris + let relativePath = _accountInfo.StorageUri + .MakeRelativeUri(uri) + .RelativeUriToFilePath() + select Path.Combine(RootPath, relativePath)).ToList(); } /// /// Delete the folders that were removed from synchronization /// /// - private void RemoveSelectivePaths(IEnumerable removed) + private void RemoveSelectivePaths(List removed) { if (removed == null) return; -- 1.7.10.4