2 /* -----------------------------------------------------------------------
3 * <copyright file="StatusAgent.cs" company="GRNet">
5 * Copyright 2011-2012 GRNET S.A. All rights reserved.
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
11 * 1. Redistributions of source code must retain the above
12 * copyright notice, this list of conditions and the following
15 * 2. Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials
18 * provided with the distribution.
21 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
34 * The views and conclusions contained in the software and
35 * documentation are those of the authors and should not be
36 * interpreted as representing official policies, either expressed
37 * or implied, of GRNET S.A.
39 * -----------------------------------------------------------------------
43 using System.Collections.Generic;
44 using System.ComponentModel.Composition;
45 using System.Data.SQLite;
46 using System.Diagnostics;
47 using System.Diagnostics.Contracts;
50 using System.Reflection;
51 using System.Security.Cryptography;
53 using System.Threading;
54 using System.Threading.Tasks;
55 using Castle.ActiveRecord;
56 using Castle.ActiveRecord.Framework;
57 using Castle.ActiveRecord.Framework.Config;
58 using Pithos.Interfaces;
62 namespace Pithos.Core.Agents
64 [Export(typeof(IStatusChecker)),Export(typeof(IStatusKeeper))]
65 public class StatusAgent:IStatusChecker,IStatusKeeper
67 private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
69 [System.ComponentModel.Composition.Import]
70 public IPithosSettings Settings { get; set; }
72 private Agent<Action> _persistenceAgent;
78 var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
82 _pithosDataPath = Path.Combine(appDataPath , "GRNET");
83 if (!Directory.Exists(_pithosDataPath))
84 Directory.CreateDirectory(_pithosDataPath);
86 var dbPath = Path.Combine(_pithosDataPath, "pithos.db");
88 MigrateOldDb(dbPath, appDataPath);
90 var source = GetConfiguration(_pithosDataPath);
91 ActiveRecordStarter.Initialize(source,typeof(FileState),typeof(FileTag));
92 ActiveRecordStarter.UpdateSchema();
95 if (!File.Exists(dbPath))
96 ActiveRecordStarter.CreateSchema();
103 private static void MigrateOldDb(string dbPath, string appDataPath)
105 Contract.Requires(!String.IsNullOrWhiteSpace(dbPath));
106 Contract.Requires(!String.IsNullOrWhiteSpace(appDataPath));
108 var oldDbPath = Path.Combine(appDataPath, "Pithos", "pithos.db");
109 var oldDbInfo = new FileInfo(oldDbPath);
110 if (oldDbInfo.Exists && !File.Exists(dbPath))
112 var oldDirectory = oldDbInfo.Directory;
113 oldDbInfo.MoveTo(dbPath);
114 oldDirectory.Delete(true);
118 private void CreateTrigger()
120 using (var connection = GetConnection())
121 using (var triggerCommand = connection.CreateCommand())
123 var cmdText = new StringBuilder()
124 .AppendLine("CREATE TRIGGER IF NOT EXISTS update_last_modified UPDATE ON FileState FOR EACH ROW")
126 .AppendLine("UPDATE FileState SET Modified=datetime('now') WHERE Id=old.Id;")
128 .AppendLine("CREATE TRIGGER IF NOT EXISTS insert_last_modified INSERT ON FileState FOR EACH ROW")
130 .AppendLine("UPDATE FileState SET Modified=datetime('now') WHERE Id=new.Id;")
133 triggerCommand.CommandText = cmdText;
134 triggerCommand.ExecuteNonQuery();
139 private static InPlaceConfigurationSource GetConfiguration(string pithosDbPath)
141 if (String.IsNullOrWhiteSpace(pithosDbPath))
142 throw new ArgumentNullException("pithosDbPath");
143 if (!Path.IsPathRooted(pithosDbPath))
144 throw new ArgumentException("path must be a rooted path", "pithosDbPath");
145 Contract.EndContractBlock();
147 var properties = new Dictionary<string, string>
149 {"connection.driver_class", "NHibernate.Driver.SQLite20Driver"},
150 {"dialect", "NHibernate.Dialect.SQLiteDialect"},
151 {"connection.provider", "NHibernate.Connection.DriverConnectionProvider"},
153 "proxyfactory.factory_class",
154 "NHibernate.ByteCode.Castle.ProxyFactoryFactory, NHibernate.ByteCode.Castle"
158 var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N", pithosDbPath);
159 properties.Add("connection.connection_string", connectionString);
161 var source = new InPlaceConfigurationSource();
162 source.Add(typeof (ActiveRecordBase), properties);
163 source.SetDebugFlag(false);
167 public void StartProcessing(CancellationToken token)
169 _persistenceAgent = Agent<Action>.Start(queue =>
174 var job = queue.Receive();
175 job.ContinueWith(t =>
177 var action = job.Result;
182 catch (SQLiteException ex)
184 Log.ErrorFormat("[ERROR] SQL \n{0}", ex);
188 Log.ErrorFormat("[ERROR] STATE \n{0}", ex);
190 queue.NotifyComplete(action);
191 // ReSharper disable AccessToModifiedClosure
193 // ReSharper restore AccessToModifiedClosure
205 _persistenceAgent.Stop();
209 public void ProcessExistingFiles(IEnumerable<FileInfo> existingFiles)
211 if(existingFiles ==null)
212 throw new ArgumentNullException("existingFiles");
213 Contract.EndContractBlock();
215 //Find new or matching files with a left join to the stored states
216 var fileStates = FileState.Queryable;
217 var currentFiles=from file in existingFiles
218 join state in fileStates on file.FullName.ToLower() equals state.FilePath.ToLower() into gs
219 from substate in gs.DefaultIfEmpty()
220 select new {File = file, State = substate};
222 //To get the deleted files we must get the states that have no corresponding
224 //We can't use the File.Exists method inside a query, so we get all file paths from the states
225 var statePaths = (from state in fileStates
226 select new {state.Id, state.FilePath}).ToList();
228 var missingStates= (from path in statePaths
229 where !File.Exists(path.FilePath) && !Directory.Exists(path.FilePath)
230 select path.Id).ToList();
231 //Finally, retrieve the states that correspond to the deleted files
232 var deletedFiles = from state in fileStates
233 where missingStates.Contains(state.Id)
234 select new { File = default(FileInfo), State = state };
236 var pairs = currentFiles.Union(deletedFiles).ToList();
238 using (var shortHasher = HashAlgorithm.Create("sha1"))
240 foreach (var pair in pairs)
242 var fileState = pair.State;
243 var file = pair.File;
244 if (fileState == null)
247 var createState = FileState.CreateFor(file);
248 _persistenceAgent.Post(createState.Create);
250 else if (file == null)
252 //This file was deleted while we were down. We should mark it as deleted
253 //We have to go through UpdateStatus here because the state object we are using
254 //was created by a different ORM session.
255 _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Deleted));
259 //This file has a matching state. Need to check for possible changes
260 //To check for changes, we use the cheap (in CPU terms) SHA1 algorithm
261 //on the entire file.
263 var hashString = file.ComputeShortHash(shortHasher);
264 //TODO: Need a way to attach the hashes to the filestate so we don't
265 //recalculate them each time a call to calculate has is made
266 //We can either store them to the filestate or add them to a
269 //If the hashes don't match the file was changed
270 if (fileState.ShortHash != hashString)
272 _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Modified));
283 private int UpdateStatusDirect(Guid id, FileStatus status)
285 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
291 using (var connection = GetConnection())
293 var command = new SQLiteCommand("update FileState set FileStatus= :fileStatus where Id = :id ",
296 command.Parameters.AddWithValue("fileStatus", status);
298 command.Parameters.AddWithValue("id", id);
300 var affected = command.ExecuteNonQuery();
306 catch (Exception exc)
308 Log.Error(exc.ToString());
314 private int UpdateStatusDirect(string path, FileStatus status)
316 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
323 using (var connection = GetConnection())
326 new SQLiteCommand("update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE",
331 command.Parameters.AddWithValue("fileStatus", status);
333 command.Parameters.AddWithValue("path", path);
335 var affected = command.ExecuteNonQuery();
339 catch (Exception exc)
341 Log.Error(exc.ToString());
347 private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus)
349 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
356 using (var connection = GetConnection())
360 "update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE ",
364 command.Parameters.AddWithValue("path", absolutePath);
365 command.Parameters.AddWithValue("fileStatus", fileStatus);
366 command.Parameters.AddWithValue("overlayStatus", overlayStatus);
368 var affected = command.ExecuteNonQuery();
372 catch (Exception exc)
374 Log.Error(exc.ToString());
382 public string BlockHash { get; set; }
384 public int BlockSize { get; set; }
385 public void ChangeRoots(string oldPath, string newPath)
387 if (String.IsNullOrWhiteSpace(oldPath))
388 throw new ArgumentNullException("oldPath");
389 if (!Path.IsPathRooted(oldPath))
390 throw new ArgumentException("oldPath must be an absolute path", "oldPath");
391 if (string.IsNullOrWhiteSpace(newPath))
392 throw new ArgumentNullException("newPath");
393 if (!Path.IsPathRooted(newPath))
394 throw new ArgumentException("newPath must be an absolute path", "newPath");
395 Contract.EndContractBlock();
397 FileState.ChangeRootPath(oldPath,newPath);
403 private readonly string _pithosDataPath;
406 public FileState GetStateByFilePath(string path)
408 if (String.IsNullOrWhiteSpace(path))
409 throw new ArgumentNullException("path");
410 if (!Path.IsPathRooted(path))
411 throw new ArgumentException("The path must be rooted", "path");
412 Contract.EndContractBlock();
417 using (var connection = GetConnection())
418 using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,ShortHash,Version ,VersionTimeStamp,IsShared ,SharedBy ,ShareWrite from FileState where FilePath=:path COLLATE NOCASE", connection))
421 command.Parameters.AddWithValue("path", path);
423 using (var reader = command.ExecuteReader())
427 //var values = new object[reader.FieldCount];
428 //reader.GetValues(values);
429 var state = new FileState
431 Id = reader.GetGuid(0),
432 FilePath = reader.IsDBNull(1)?"":reader.GetString(1),
433 OverlayStatus =reader.IsDBNull(2)?FileOverlayStatus.Unversioned: (FileOverlayStatus) reader.GetInt64(2),
434 FileStatus = reader.IsDBNull(3)?FileStatus.Missing:(FileStatus) reader.GetInt64(3),
435 Checksum = reader.IsDBNull(4)?"":reader.GetString(4),
436 ShortHash= reader.IsDBNull(5)?"":reader.GetString(5),
437 Version = reader.IsDBNull(6)?default(long):reader.GetInt64(6),
438 VersionTimeStamp = reader.IsDBNull(7)?default(DateTime):reader.GetDateTime(7),
439 IsShared = !reader.IsDBNull(8) && reader.GetBoolean(8),
440 SharedBy = reader.IsDBNull(9)?"":reader.GetString(9),
441 ShareWrite = !reader.IsDBNull(10) && reader.GetBoolean(10)
444 var state = new FileState
446 Id = (Guid) values[0],
447 FilePath = (string) values[1],
448 OverlayStatus = (FileOverlayStatus) (long)values[2],
449 FileStatus = (FileStatus) (long)values[3],
450 Checksum = (string) values[4],
451 Version = (long?) values[5],
452 VersionTimeStamp = (DateTime?) values[6],
453 IsShared = (long)values[7] == 1,
454 SharedBy = (string) values[8],
455 ShareWrite = (long)values[9] == 1
468 catch (Exception exc)
470 Log.ErrorFormat(exc.ToString());
475 public FileOverlayStatus GetFileOverlayStatus(string path)
477 if (String.IsNullOrWhiteSpace(path))
478 throw new ArgumentNullException("path");
479 if (!Path.IsPathRooted(path))
480 throw new ArgumentException("The path must be rooted", "path");
481 Contract.EndContractBlock();
486 using (var connection = GetConnection())
487 using (var command = new SQLiteCommand("select OverlayStatus from FileState where FilePath=:path COLLATE NOCASE", connection))
490 command.Parameters.AddWithValue("path", path);
492 var s = command.ExecuteScalar();
493 return (FileOverlayStatus) Convert.ToInt32(s);
496 catch (Exception exc)
498 Log.ErrorFormat(exc.ToString());
499 return FileOverlayStatus.Unversioned;
503 private string GetConnectionString()
505 var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N;Pooling=True", _pithosDataPath);
506 return connectionString;
509 private SQLiteConnection GetConnection()
511 var connectionString = GetConnectionString();
512 var connection = new SQLiteConnection(connectionString);
514 using(var cmd =connection.CreateCommand())
516 cmd.CommandText = "PRAGMA journal_mode=WAL";
517 cmd.ExecuteNonQuery();
522 /* public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
524 if (String.IsNullOrWhiteSpace(path))
525 throw new ArgumentNullException("path");
526 if (!Path.IsPathRooted(path))
527 throw new ArgumentException("The path must be rooted","path");
528 Contract.EndContractBlock();
530 _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path,overlayStatus));
533 public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string shortHash = null)
535 if (String.IsNullOrWhiteSpace(path))
536 throw new ArgumentNullException("path");
537 if (!Path.IsPathRooted(path))
538 throw new ArgumentException("The path must be rooted","path");
539 Contract.EndContractBlock();
541 return _persistenceAgent.PostAndAwait(() => FileState.StoreOverlayStatus(path,overlayStatus,shortHash));
544 /* public void RenameFileOverlayStatus(string oldPath, string newPath)
546 if (String.IsNullOrWhiteSpace(oldPath))
547 throw new ArgumentNullException("oldPath");
548 if (!Path.IsPathRooted(oldPath))
549 throw new ArgumentException("The oldPath must be rooted", "oldPath");
550 if (String.IsNullOrWhiteSpace(newPath))
551 throw new ArgumentNullException("newPath");
552 if (!Path.IsPathRooted(newPath))
553 throw new ArgumentException("The newPath must be rooted", "newPath");
554 Contract.EndContractBlock();
556 _persistenceAgent.Post(() =>FileState.RenameState(oldPath, newPath));
559 public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus)
561 if (String.IsNullOrWhiteSpace(path))
562 throw new ArgumentNullException("path");
563 if (!Path.IsPathRooted(path))
564 throw new ArgumentException("The path must be rooted", "path");
565 Contract.EndContractBlock();
567 Debug.Assert(!path.Contains(FolderConstants.CacheFolder));
568 Debug.Assert(!path.EndsWith(".ignore"));
570 _persistenceAgent.Post(() => UpdateStatusDirect(path, fileStatus, overlayStatus));
574 public void StoreInfo(string path,ObjectInfo objectInfo)
576 if (String.IsNullOrWhiteSpace(path))
577 throw new ArgumentNullException("path");
578 if (!Path.IsPathRooted(path))
579 throw new ArgumentException("The path must be rooted", "path");
580 if (objectInfo == null)
581 throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");
582 Contract.EndContractBlock();
584 _persistenceAgent.Post(() =>
586 var filePath = path.ToLower();
587 //Load the existing files state and set its properties in one session
588 using (new SessionScope())
590 //Forgetting to use a sessionscope results in two sessions being created, one by
591 //FirstOrDefault and one by Save()
592 var state =FileState.FindByFilePath(filePath);
594 //Create a new empty state object if this is a new file
595 state = state ?? new FileState();
597 state.FilePath = filePath;
598 state.Checksum = objectInfo.Hash;
599 state.Version = objectInfo.Version;
600 state.VersionTimeStamp = objectInfo.VersionTimestamp;
602 state.FileStatus = FileStatus.Unchanged;
603 state.OverlayStatus = FileOverlayStatus.Normal;
614 public void StoreInfo(string path, ObjectInfo objectInfo)
616 if (String.IsNullOrWhiteSpace(path))
617 throw new ArgumentNullException("path");
618 if (!Path.IsPathRooted(path))
619 throw new ArgumentException("The path must be rooted", "path");
620 if (objectInfo == null)
621 throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");
622 Contract.EndContractBlock();
624 _persistenceAgent.Post(() => StoreInfoDirect(path, objectInfo));
628 private void StoreInfoDirect(string path, ObjectInfo objectInfo)
633 using (var connection = GetConnection())
634 using (var command = new SQLiteCommand(connection))
636 if (StateExists(path, connection))
637 command.CommandText =
638 "update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE ";
641 command.CommandText =
642 "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,ShortHash,FileStatus,OverlayStatus) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:shortHash,:fileStatus,:overlayStatus)";
643 command.Parameters.AddWithValue("id", Guid.NewGuid());
646 command.Parameters.AddWithValue("path", path);
647 command.Parameters.AddWithValue("checksum", objectInfo.Hash);
648 command.Parameters.AddWithValue("shortHash", "");
649 command.Parameters.AddWithValue("version", objectInfo.Version);
650 command.Parameters.AddWithValue("versionTimeStamp",
651 objectInfo.VersionTimestamp);
652 command.Parameters.AddWithValue("fileStatus", FileStatus.Unchanged);
653 command.Parameters.AddWithValue("overlayStatus",
654 FileOverlayStatus.Normal);
656 var affected = command.ExecuteNonQuery();
660 catch (Exception exc)
662 Log.Error(exc.ToString());
667 private bool StateExists(string filePath,SQLiteConnection connection)
669 using (var command = new SQLiteCommand("Select count(*) from FileState where FilePath=:path COLLATE NOCASE", connection))
671 command.Parameters.AddWithValue("path", filePath);
672 var result = command.ExecuteScalar();
673 return ((long)result >= 1);
678 public void SetFileStatus(string path, FileStatus status)
680 if (String.IsNullOrWhiteSpace(path))
681 throw new ArgumentNullException("path");
682 if (!Path.IsPathRooted(path))
683 throw new ArgumentException("The path must be rooted", "path");
684 Contract.EndContractBlock();
686 _persistenceAgent.Post(() => UpdateStatusDirect(path, status));
689 public FileStatus GetFileStatus(string path)
691 if (String.IsNullOrWhiteSpace(path))
692 throw new ArgumentNullException("path");
693 if (!Path.IsPathRooted(path))
694 throw new ArgumentException("The path must be rooted", "path");
695 Contract.EndContractBlock();
698 using (var connection = GetConnection())
700 var command = new SQLiteCommand("select FileStatus from FileState where FilePath=:path COLLATE NOCASE", connection);
701 command.Parameters.AddWithValue("path", path);
703 var statusValue = command.ExecuteScalar();
704 if (statusValue==null)
705 return FileStatus.Missing;
706 return (FileStatus)Convert.ToInt32(statusValue);
711 /// Deletes the status of the specified file
713 /// <param name="path"></param>
714 public void ClearFileStatus(string path)
716 if (String.IsNullOrWhiteSpace(path))
717 throw new ArgumentNullException("path");
718 if (!Path.IsPathRooted(path))
719 throw new ArgumentException("The path must be rooted", "path");
720 Contract.EndContractBlock();
722 _persistenceAgent.Post(() => DeleteDirect(path));
726 /// Deletes the status of the specified folder and all its contents
728 /// <param name="path"></param>
729 public void ClearFolderStatus(string path)
731 if (String.IsNullOrWhiteSpace(path))
732 throw new ArgumentNullException("path");
733 if (!Path.IsPathRooted(path))
734 throw new ArgumentException("The path must be rooted", "path");
735 Contract.EndContractBlock();
737 _persistenceAgent.Post(() => DeleteFolderDirect(path));
740 public IEnumerable<FileState> GetChildren(FileState fileState)
742 if (fileState == null)
743 throw new ArgumentNullException("fileState");
744 Contract.EndContractBlock();
746 var children = from state in FileState.Queryable
747 where state.FilePath.StartsWith(fileState.FilePath + "\\")
752 public void EnsureFileState(string path)
754 var existingState = GetStateByFilePath(path);
755 if (existingState != null)
757 var fileInfo = FileInfoExtensions.FromPath(path);
758 using (new SessionScope())
760 var newState = FileState.CreateFor(fileInfo);
761 newState.FileStatus=FileStatus.Missing;
762 _persistenceAgent.PostAndAwait(newState.CreateAndFlush).Wait();
767 private int DeleteDirect(string filePath)
769 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
776 using (var connection = GetConnection())
778 var command = new SQLiteCommand("delete from FileState where FilePath = :path COLLATE NOCASE",
781 command.Parameters.AddWithValue("path", filePath);
783 var affected = command.ExecuteNonQuery();
787 catch (Exception exc)
789 Log.Error(exc.ToString());
795 private int DeleteFolderDirect(string filePath)
797 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
804 using (var connection = GetConnection())
806 var command = new SQLiteCommand(@"delete from FileState where FilePath = :path or FilePath like :path || '\%' COLLATE NOCASE",
809 command.Parameters.AddWithValue("path", filePath);
811 var affected = command.ExecuteNonQuery();
815 catch (Exception exc)
817 Log.Error(exc.ToString());
823 public void UpdateFileChecksum(string path, string shortHash, string checksum)
825 if (String.IsNullOrWhiteSpace(path))
826 throw new ArgumentNullException("path");
827 if (!Path.IsPathRooted(path))
828 throw new ArgumentException("The path must be rooted", "path");
829 Contract.EndContractBlock();
831 _persistenceAgent.Post(() => FileState.UpdateChecksum(path, shortHash,checksum));