X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/blobdiff_plain/14ecd267215d811ba0b586fd1bd1baa0ab482de5..2341c6036d5af50def10d2696063955ce199360d:/trunk/Pithos.Core/Agents/StatusAgent.cs diff --git a/trunk/Pithos.Core/Agents/StatusAgent.cs b/trunk/Pithos.Core/Agents/StatusAgent.cs index 67dafb2..7090f90 100644 --- a/trunk/Pithos.Core/Agents/StatusAgent.cs +++ b/trunk/Pithos.Core/Agents/StatusAgent.cs @@ -1,3 +1,44 @@ +#region +/* ----------------------------------------------------------------------- + * + * + * Copyright 2011-2012 GRNET S.A. All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and + * documentation are those of the authors and should not be + * interpreted as representing official policies, either expressed + * or implied, of GRNET S.A. + * + * ----------------------------------------------------------------------- + */ +#endregion using System; using System.Collections.Generic; using System.ComponentModel.Composition; @@ -6,42 +47,106 @@ using System.Diagnostics; using System.Diagnostics.Contracts; using System.IO; using System.Linq; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; using System.Threading; using System.Threading.Tasks; using Castle.ActiveRecord; +using Castle.ActiveRecord.Framework; using Castle.ActiveRecord.Framework.Config; +using NHibernate.ByteCode.Castle; +using NHibernate.Cfg; +using NHibernate.Cfg.Loquacious; +using NHibernate.Dialect; using Pithos.Interfaces; using Pithos.Network; using log4net; +using Environment = System.Environment; namespace Pithos.Core.Agents { [Export(typeof(IStatusChecker)),Export(typeof(IStatusKeeper))] public class StatusAgent:IStatusChecker,IStatusKeeper { + private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + [System.ComponentModel.Composition.Import] public IPithosSettings Settings { get; set; } private Agent _persistenceAgent; - private static readonly ILog Log = LogManager.GetLogger("StatusAgent"); public StatusAgent() { var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); - _pithosDataPath = Path.Combine(appDataPath , "Pithos"); + _pithosDataPath = Path.Combine(appDataPath , "GRNET\\PITHOS"); if (!Directory.Exists(_pithosDataPath)) Directory.CreateDirectory(_pithosDataPath); + + var dbPath = Path.Combine(_pithosDataPath, "pithos.db"); + + MigrateOldDb(dbPath, appDataPath); + + var source = GetConfiguration(_pithosDataPath); ActiveRecordStarter.Initialize(source,typeof(FileState),typeof(FileTag)); + ActiveRecordStarter.UpdateSchema(); - if (!File.Exists(Path.Combine(_pithosDataPath ,"pithos.db"))) + + if (!File.Exists(dbPath)) ActiveRecordStarter.CreateSchema(); - } + CreateTrigger(); + + } + + + private static void MigrateOldDb(string dbPath, string appDataPath) + { + if(String.IsNullOrWhiteSpace(dbPath)) + throw new ArgumentNullException("dbPath"); + if(String.IsNullOrWhiteSpace(appDataPath)) + throw new ArgumentNullException("appDataPath"); + Contract.EndContractBlock(); + + var oldDbPath = Path.Combine(appDataPath, "Pithos", "pithos.db"); + var oldDbInfo = new FileInfo(oldDbPath); + if (oldDbInfo.Exists && !File.Exists(dbPath)) + { + Log.InfoFormat("Moving database from {0} to {1}",oldDbInfo.FullName,dbPath); + var oldDirectory = oldDbInfo.Directory; + oldDbInfo.MoveTo(dbPath); + + if (Log.IsDebugEnabled) + Log.DebugFormat("Deleting {0}",oldDirectory.FullName); + + oldDirectory.Delete(true); + } + } + + private void CreateTrigger() + { + using (var connection = GetConnection()) + using (var triggerCommand = connection.CreateCommand()) + { + var cmdText = new StringBuilder() + .AppendLine("CREATE TRIGGER IF NOT EXISTS update_last_modified UPDATE ON FileState FOR EACH ROW") + .AppendLine("BEGIN") + .AppendLine("UPDATE FileState SET Modified=datetime('now') WHERE Id=old.Id;") + .AppendLine("END;") + .AppendLine("CREATE TRIGGER IF NOT EXISTS insert_last_modified INSERT ON FileState FOR EACH ROW") + .AppendLine("BEGIN") + .AppendLine("UPDATE FileState SET Modified=datetime('now') WHERE Id=new.Id;") + .AppendLine("END;") + .ToString(); + triggerCommand.CommandText = cmdText; + triggerCommand.ExecuteNonQuery(); + } + } private static InPlaceConfigurationSource GetConfiguration(string pithosDbPath) @@ -95,6 +200,7 @@ namespace Pithos.Core.Agents { Log.ErrorFormat("[ERROR] STATE \n{0}", ex); } + queue.NotifyComplete(action); // ReSharper disable AccessToModifiedClosure queue.DoAsync(loop); // ReSharper restore AccessToModifiedClosure @@ -111,7 +217,7 @@ namespace Pithos.Core.Agents { _persistenceAgent.Stop(); } - + public void ProcessExistingFiles(IEnumerable existingFiles) { @@ -140,39 +246,52 @@ namespace Pithos.Core.Agents where missingStates.Contains(state.Id) select new { File = default(FileInfo), State = state }; - var pairs = currentFiles.Union(deletedFiles); + var pairs = currentFiles.Union(deletedFiles).ToList(); - foreach(var pair in pairs) + using (var shortHasher = HashAlgorithm.Create("sha1")) { - var fileState = pair.State; - var file = pair.File; - if (fileState == null) + foreach (var pair in pairs) { - //This is a new file - var fullPath = pair.File.FullName.ToLower(); - var createState = FileState.CreateForAsync(fullPath, BlockSize, BlockHash); - createState.ContinueWith(state => _persistenceAgent.Post(state.Result.Create)); - } - else if (file == null) - { - //This file was deleted while we were down. We should mark it as deleted - //We have to go through UpdateStatus here because the state object we are using - //was created by a different ORM session. - _persistenceAgent.Post(()=> UpdateStatusDirect(fileState.Id, FileStatus.Deleted)); - } - else - { - //This file has a matching state. Need to check for possible changes - var hashString = file.CalculateHash(BlockSize,BlockHash); - //If the hashes don't match the file was changed - if (fileState.Checksum != hashString) + var fileState = pair.State; + var file = pair.File; + if (fileState == null) { - _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Modified)); - } + //This is a new file + var createState = FileState.CreateFor(file); + _persistenceAgent.Post(createState.Create); + } + else if (file == null) + { + //This file was deleted while we were down. We should mark it as deleted + //We have to go through UpdateStatus here because the state object we are using + //was created by a different ORM session. + _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Deleted)); + } + else + { + //This file has a matching state. Need to check for possible changes + //To check for changes, we use the cheap (in CPU terms) SHA1 algorithm + //on the entire file. + + var hashString = file.ComputeShortHash(shortHasher); + //TODO: Need a way to attach the hashes to the filestate so we don't + //recalculate them each time a call to calculate has is made + //We can either store them to the filestate or add them to a + //dictionary + + //If the hashes don't match the file was changed + if (fileState.ShortHash != hashString) + { + _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Modified)); + } + } } - }; + } + } + + private int UpdateStatusDirect(Guid id, FileStatus status) { @@ -217,16 +336,22 @@ namespace Pithos.Core.Agents using (var connection = GetConnection()) using ( var command = - new SQLiteCommand("update FileState set FileStatus= :fileStatus where FilePath = :path ", + new SQLiteCommand("update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE", connection)) { command.Parameters.AddWithValue("fileStatus", status); - command.Parameters.AddWithValue("path", path.ToLower()); + command.Parameters.AddWithValue("path", path); var affected = command.ExecuteNonQuery(); + if (affected == 0) + { + var createdState = FileState.CreateFor(FileInfoExtensions.FromPath(path)); + createdState.FileStatus = status; + createdState.Create(); + } return affected; } } @@ -238,7 +363,7 @@ namespace Pithos.Core.Agents } } - private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus) + private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason) { using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect")) { @@ -251,15 +376,24 @@ namespace Pithos.Core.Agents using ( var command = new SQLiteCommand( - "update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus where FilePath = :path ", + "update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus,ConflictReason= :conflictReason where FilePath = :path COLLATE NOCASE ", connection)) { - command.Parameters.AddWithValue("path", absolutePath.ToLower()); + command.Parameters.AddWithValue("path", absolutePath); command.Parameters.AddWithValue("fileStatus", fileStatus); command.Parameters.AddWithValue("overlayStatus", overlayStatus); + command.Parameters.AddWithValue("conflictReason", conflictReason); var affected = command.ExecuteNonQuery(); + if (affected == 0) + { + var createdState=FileState.CreateFor(FileInfoExtensions.FromPath(absolutePath)); + createdState.FileStatus = fileStatus; + createdState.OverlayStatus = overlayStatus; + createdState.ConflictReason = conflictReason; + createdState.Create(); + } return affected; } } @@ -292,17 +426,6 @@ namespace Pithos.Core.Agents } - private PithosStatus _pithosStatus=PithosStatus.InSynch; - - public void SetPithosStatus(PithosStatus status) - { - _pithosStatus = status; - } - - public PithosStatus GetPithosStatus() - { - return _pithosStatus; - } private readonly string _pithosDataPath; @@ -320,10 +443,10 @@ namespace Pithos.Core.Agents { using (var connection = GetConnection()) - using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,Version ,VersionTimeStamp,IsShared ,SharedBy ,ShareWrite from FileState where FilePath=:path", connection)) + using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,ShortHash,Version ,VersionTimeStamp,IsShared ,SharedBy ,ShareWrite from FileState where FilePath=:path COLLATE NOCASE", connection)) { - command.Parameters.AddWithValue("path", path.ToLower()); + command.Parameters.AddWithValue("path", path); using (var reader = command.ExecuteReader()) { @@ -338,11 +461,12 @@ namespace Pithos.Core.Agents OverlayStatus =reader.IsDBNull(2)?FileOverlayStatus.Unversioned: (FileOverlayStatus) reader.GetInt64(2), FileStatus = reader.IsDBNull(3)?FileStatus.Missing:(FileStatus) reader.GetInt64(3), Checksum = reader.IsDBNull(4)?"":reader.GetString(4), - Version = reader.IsDBNull(5)?default(long):reader.GetInt64(5), - VersionTimeStamp = reader.IsDBNull(6)?default(DateTime):reader.GetDateTime(6), - IsShared = !reader.IsDBNull(7) && reader.GetBoolean(7), - SharedBy = reader.IsDBNull(8)?"":reader.GetString(8), - ShareWrite = !reader.IsDBNull(9) && reader.GetBoolean(9) + ShortHash= reader.IsDBNull(5)?"":reader.GetString(5), + Version = reader.IsDBNull(6)?default(long):reader.GetInt64(6), + VersionTimeStamp = reader.IsDBNull(7)?default(DateTime):reader.GetDateTime(7), + IsShared = !reader.IsDBNull(8) && reader.GetBoolean(8), + SharedBy = reader.IsDBNull(9)?"":reader.GetString(9), + ShareWrite = !reader.IsDBNull(10) && reader.GetBoolean(10) }; /* var state = new FileState @@ -388,10 +512,10 @@ namespace Pithos.Core.Agents { using (var connection = GetConnection()) - using (var command = new SQLiteCommand("select OverlayStatus from FileState where FilePath=:path", connection)) + using (var command = new SQLiteCommand("select OverlayStatus from FileState where FilePath=:path COLLATE NOCASE", connection)) { - command.Parameters.AddWithValue("path", path.ToLower()); + command.Parameters.AddWithValue("path", path); var s = command.ExecuteScalar(); return (FileOverlayStatus) Convert.ToInt32(s); @@ -423,7 +547,18 @@ namespace Pithos.Core.Agents return connection; } - public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus) + /* public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus) + { + if (String.IsNullOrWhiteSpace(path)) + throw new ArgumentNullException("path"); + if (!Path.IsPathRooted(path)) + throw new ArgumentException("The path must be rooted","path"); + Contract.EndContractBlock(); + + _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path,overlayStatus)); + }*/ + + public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string shortHash = null) { if (String.IsNullOrWhiteSpace(path)) throw new ArgumentNullException("path"); @@ -431,7 +566,7 @@ namespace Pithos.Core.Agents throw new ArgumentException("The path must be rooted","path"); Contract.EndContractBlock(); - _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path.ToLower(),overlayStatus)); + return _persistenceAgent.PostAndAwait(() => FileState.StoreOverlayStatus(path,overlayStatus,shortHash)); } /* public void RenameFileOverlayStatus(string oldPath, string newPath) @@ -449,7 +584,7 @@ namespace Pithos.Core.Agents _persistenceAgent.Post(() =>FileState.RenameState(oldPath, newPath)); }*/ - public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus) + public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason) { if (String.IsNullOrWhiteSpace(path)) throw new ArgumentNullException("path"); @@ -460,7 +595,7 @@ namespace Pithos.Core.Agents Debug.Assert(!path.Contains(FolderConstants.CacheFolder)); Debug.Assert(!path.EndsWith(".ignore")); - _persistenceAgent.Post(() => UpdateStatusDirect(path.ToLower(), fileStatus, overlayStatus)); + _persistenceAgent.Post(() => UpdateStatusDirect(path, fileStatus, overlayStatus, conflictReason)); } /* @@ -528,16 +663,17 @@ namespace Pithos.Core.Agents { if (StateExists(path, connection)) command.CommandText = - "update FileState set FileStatus= :fileStatus where FilePath = :path "; + "update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE "; else { command.CommandText = - "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,FileStatus,OverlayStatus) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:fileStatus,:overlayStatus)"; + "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,ShortHash,FileStatus,OverlayStatus) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:shortHash,:fileStatus,:overlayStatus)"; command.Parameters.AddWithValue("id", Guid.NewGuid()); } - command.Parameters.AddWithValue("path", path.ToLower()); + command.Parameters.AddWithValue("path", path); command.Parameters.AddWithValue("checksum", objectInfo.Hash); + command.Parameters.AddWithValue("shortHash", ""); command.Parameters.AddWithValue("version", objectInfo.Version); command.Parameters.AddWithValue("versionTimeStamp", objectInfo.VersionTimestamp); @@ -558,9 +694,9 @@ namespace Pithos.Core.Agents private bool StateExists(string filePath,SQLiteConnection connection) { - using (var command = new SQLiteCommand("Select count(*) from FileState where FilePath=:path", connection)) + using (var command = new SQLiteCommand("Select count(*) from FileState where FilePath=:path COLLATE NOCASE", connection)) { - command.Parameters.AddWithValue("path", filePath.ToLower()); + command.Parameters.AddWithValue("path", filePath); var result = command.ExecuteScalar(); return ((long)result >= 1); } @@ -575,7 +711,7 @@ namespace Pithos.Core.Agents throw new ArgumentException("The path must be rooted", "path"); Contract.EndContractBlock(); - _persistenceAgent.Post(() => UpdateStatusDirect(path.ToLower(), status)); + _persistenceAgent.Post(() => UpdateStatusDirect(path, status)); } public FileStatus GetFileStatus(string path) @@ -589,14 +725,20 @@ namespace Pithos.Core.Agents using (var connection = GetConnection()) { - var command = new SQLiteCommand("select FileStatus from FileState where FilePath=:path", connection); - command.Parameters.AddWithValue("path", path.ToLower()); + var command = new SQLiteCommand("select FileStatus from FileState where FilePath=:path COLLATE NOCASE", connection); + command.Parameters.AddWithValue("path", path); - var s = command.ExecuteScalar(); - return (FileStatus)Convert.ToInt32(s); + var statusValue = command.ExecuteScalar(); + if (statusValue==null) + return FileStatus.Missing; + return (FileStatus)Convert.ToInt32(statusValue); } } + /// + /// Deletes the status of the specified file + /// + /// public void ClearFileStatus(string path) { if (String.IsNullOrWhiteSpace(path)) @@ -608,6 +750,48 @@ namespace Pithos.Core.Agents _persistenceAgent.Post(() => DeleteDirect(path)); } + /// + /// Deletes the status of the specified folder and all its contents + /// + /// + public void ClearFolderStatus(string path) + { + if (String.IsNullOrWhiteSpace(path)) + throw new ArgumentNullException("path"); + if (!Path.IsPathRooted(path)) + throw new ArgumentException("The path must be rooted", "path"); + Contract.EndContractBlock(); + + _persistenceAgent.Post(() => DeleteFolderDirect(path)); + } + + public IEnumerable GetChildren(FileState fileState) + { + if (fileState == null) + throw new ArgumentNullException("fileState"); + Contract.EndContractBlock(); + + var children = from state in FileState.Queryable + where state.FilePath.StartsWith(fileState.FilePath + "\\") + select state; + return children; + } + + public void EnsureFileState(string path) + { + var existingState = GetStateByFilePath(path); + if (existingState != null) + return; + var fileInfo = FileInfoExtensions.FromPath(path); + using (new SessionScope()) + { + var newState = FileState.CreateFor(fileInfo); + newState.FileStatus=FileStatus.Missing; + _persistenceAgent.PostAndAwait(newState.CreateAndFlush).Wait(); + } + + } + private int DeleteDirect(string filePath) { using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect")) @@ -619,10 +803,38 @@ namespace Pithos.Core.Agents using (var connection = GetConnection()) { - var command = new SQLiteCommand("delete from FileState where FilePath = :path", + var command = new SQLiteCommand("delete from FileState where FilePath = :path COLLATE NOCASE", + connection); + + command.Parameters.AddWithValue("path", filePath); + + var affected = command.ExecuteNonQuery(); + return affected; + } + } + catch (Exception exc) + { + Log.Error(exc.ToString()); + throw; + } + } + } + + private int DeleteFolderDirect(string filePath) + { + using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect")) + { + + try + { + + + using (var connection = GetConnection()) + { + var command = new SQLiteCommand(@"delete from FileState where FilePath = :path or FilePath like :path || '\%' COLLATE NOCASE", connection); - command.Parameters.AddWithValue("path", filePath.ToLower()); + command.Parameters.AddWithValue("path", filePath); var affected = command.ExecuteNonQuery(); return affected; @@ -636,7 +848,7 @@ namespace Pithos.Core.Agents } } - public void UpdateFileChecksum(string path, string checksum) + public void UpdateFileChecksum(string path, string shortHash, string checksum) { if (String.IsNullOrWhiteSpace(path)) throw new ArgumentNullException("path"); @@ -644,9 +856,68 @@ namespace Pithos.Core.Agents throw new ArgumentException("The path must be rooted", "path"); Contract.EndContractBlock(); - _persistenceAgent.Post(() => FileState.UpdateChecksum(path.ToLower(), checksum)); + _persistenceAgent.Post(() => FileState.UpdateChecksum(path, shortHash,checksum)); } + + public void CleanupOrphanStates() + { + //Orphan states are those that do not correspond to an account, ie. their paths + //do not start with the root path of any registered account + + var roots=(from account in Settings.Accounts + select account.RootPath).ToList(); + + var allStates = from state in FileState.Queryable + select state.FilePath; + + foreach (var statePath in allStates) + { + if (!roots.Any(root=>statePath.StartsWith(root,StringComparison.InvariantCultureIgnoreCase))) + this.DeleteDirect(statePath); + } + } + + public void CleanupStaleStates(AccountInfo accountInfo, List objectInfos) + { + if (accountInfo == null) + throw new ArgumentNullException("accountInfo"); + if (objectInfos == null) + throw new ArgumentNullException("objectInfos"); + Contract.EndContractBlock(); + + + + //Stale states are those that have no corresponding local or server file + + + var agent=FileAgent.GetFileAgent(accountInfo); + + var localFiles=agent.EnumerateFiles(); + var localSet = new HashSet(localFiles); + + //RelativeUrlToFilePath will fail for + //infos of accounts, containers which have no Name + + var serverFiles = from info in objectInfos + where info.Name != null + select Path.Combine(accountInfo.AccountPath,info.RelativeUrlToFilePath(accountInfo.UserName)); + var serverSet = new HashSet(serverFiles); + + var allStates = from state in FileState.Queryable + where state.FilePath.StartsWith(agent.RootPath) + select state.FilePath; + var stateSet = new HashSet(allStates); + stateSet.ExceptWith(serverSet); + stateSet.ExceptWith(localSet); + + foreach (var remainder in stateSet) + { + DeleteDirect(remainder); + } + + + } }