2 /* -----------------------------------------------------------------------
3 * <copyright file="StatusAgent.cs" company="GRNet">
5 * Copyright 2011-2012 GRNET S.A. All rights reserved.
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
11 * 1. Redistributions of source code must retain the above
12 * copyright notice, this list of conditions and the following
15 * 2. Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials
18 * provided with the distribution.
21 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
34 * The views and conclusions contained in the software and
35 * documentation are those of the authors and should not be
36 * interpreted as representing official policies, either expressed
37 * or implied, of GRNET S.A.
39 * -----------------------------------------------------------------------
43 using System.Collections.Generic;
44 using System.ComponentModel.Composition;
45 using System.Data.SQLite;
46 using System.Diagnostics;
47 using System.Diagnostics.Contracts;
51 using System.Threading;
52 using System.Threading.Tasks;
53 using Castle.ActiveRecord;
54 using Castle.ActiveRecord.Framework.Config;
55 using Pithos.Interfaces;
59 namespace Pithos.Core.Agents
61 [Export(typeof(IStatusChecker)),Export(typeof(IStatusKeeper))]
62 public class StatusAgent:IStatusChecker,IStatusKeeper
64 [System.ComponentModel.Composition.Import]
65 public IPithosSettings Settings { get; set; }
67 private Agent<Action> _persistenceAgent;
70 private static readonly ILog Log = LogManager.GetLogger("StatusAgent");
74 var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
78 _pithosDataPath = Path.Combine(appDataPath , "GRNET");
79 if (!Directory.Exists(_pithosDataPath))
80 Directory.CreateDirectory(_pithosDataPath);
82 var dbPath = Path.Combine(_pithosDataPath, "pithos.db");
84 MigrateOldDb(dbPath, appDataPath);
86 var source = GetConfiguration(_pithosDataPath);
87 ActiveRecordStarter.Initialize(source,typeof(FileState),typeof(FileTag));
88 ActiveRecordStarter.UpdateSchema();
91 if (!File.Exists(dbPath))
92 ActiveRecordStarter.CreateSchema();
99 private static void MigrateOldDb(string dbPath, string appDataPath)
101 Contract.Requires(!String.IsNullOrWhiteSpace(dbPath));
102 Contract.Requires(!String.IsNullOrWhiteSpace(appDataPath));
104 var oldDbPath = Path.Combine(appDataPath, "Pithos", "pithos.db");
105 var oldDbInfo = new FileInfo(oldDbPath);
106 if (oldDbInfo.Exists && !File.Exists(dbPath))
108 var oldDirectory = oldDbInfo.Directory;
109 oldDbInfo.MoveTo(dbPath);
110 oldDirectory.Delete(true);
114 private void CreateTrigger()
116 using (var connection = GetConnection())
117 using (var triggerCommand = connection.CreateCommand())
119 var cmdText = new StringBuilder()
120 .AppendLine("CREATE TRIGGER IF NOT EXISTS update_last_modified UPDATE ON FileState FOR EACH ROW")
122 .AppendLine("UPDATE FileState SET Modified=datetime('now') WHERE Id=old.Id;")
124 .AppendLine("CREATE TRIGGER IF NOT EXISTS insert_last_modified INSERT ON FileState FOR EACH ROW")
126 .AppendLine("UPDATE FileState SET Modified=datetime('now') WHERE Id=new.Id;")
129 triggerCommand.CommandText = cmdText;
130 triggerCommand.ExecuteNonQuery();
135 private static InPlaceConfigurationSource GetConfiguration(string pithosDbPath)
137 if (String.IsNullOrWhiteSpace(pithosDbPath))
138 throw new ArgumentNullException("pithosDbPath");
139 if (!Path.IsPathRooted(pithosDbPath))
140 throw new ArgumentException("path must be a rooted path", "pithosDbPath");
141 Contract.EndContractBlock();
143 var properties = new Dictionary<string, string>
145 {"connection.driver_class", "NHibernate.Driver.SQLite20Driver"},
146 {"dialect", "NHibernate.Dialect.SQLiteDialect"},
147 {"connection.provider", "NHibernate.Connection.DriverConnectionProvider"},
149 "proxyfactory.factory_class",
150 "NHibernate.ByteCode.Castle.ProxyFactoryFactory, NHibernate.ByteCode.Castle"
154 var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N", pithosDbPath);
155 properties.Add("connection.connection_string", connectionString);
157 var source = new InPlaceConfigurationSource();
158 source.Add(typeof (ActiveRecordBase), properties);
159 source.SetDebugFlag(false);
163 public void StartProcessing(CancellationToken token)
165 _persistenceAgent = Agent<Action>.Start(queue =>
170 var job = queue.Receive();
171 job.ContinueWith(t =>
173 var action = job.Result;
178 catch (SQLiteException ex)
180 Log.ErrorFormat("[ERROR] SQL \n{0}", ex);
184 Log.ErrorFormat("[ERROR] STATE \n{0}", ex);
186 // ReSharper disable AccessToModifiedClosure
188 // ReSharper restore AccessToModifiedClosure
200 _persistenceAgent.Stop();
204 public void ProcessExistingFiles(IEnumerable<FileInfo> existingFiles)
206 if(existingFiles ==null)
207 throw new ArgumentNullException("existingFiles");
208 Contract.EndContractBlock();
210 //Find new or matching files with a left join to the stored states
211 var fileStates = FileState.Queryable;
212 var currentFiles=from file in existingFiles
213 join state in fileStates on file.FullName.ToLower() equals state.FilePath.ToLower() into gs
214 from substate in gs.DefaultIfEmpty()
215 select new {File = file, State = substate};
217 //To get the deleted files we must get the states that have no corresponding
219 //We can't use the File.Exists method inside a query, so we get all file paths from the states
220 var statePaths = (from state in fileStates
221 select new {state.Id, state.FilePath}).ToList();
223 var missingStates= (from path in statePaths
224 where !File.Exists(path.FilePath) && !Directory.Exists(path.FilePath)
225 select path.Id).ToList();
226 //Finally, retrieve the states that correspond to the deleted files
227 var deletedFiles = from state in fileStates
228 where missingStates.Contains(state.Id)
229 select new { File = default(FileInfo), State = state };
231 var pairs = currentFiles.Union(deletedFiles);
233 foreach(var pair in pairs)
235 var fileState = pair.State;
236 var file = pair.File;
237 if (fileState == null)
240 var fullPath = pair.File.FullName;
241 var createState = FileState.CreateForAsync(fullPath, BlockSize, BlockHash);
242 createState.ContinueWith(state => _persistenceAgent.Post(state.Result.Create));
244 else if (file == null)
246 //This file was deleted while we were down. We should mark it as deleted
247 //We have to go through UpdateStatus here because the state object we are using
248 //was created by a different ORM session.
249 _persistenceAgent.Post(()=> UpdateStatusDirect(fileState.Id, FileStatus.Deleted));
253 //This file has a matching state. Need to check for possible changes
254 var hashString = file.CalculateHash(BlockSize,BlockHash);
255 //If the hashes don't match the file was changed
256 if (fileState.Checksum != hashString)
258 _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Modified));
265 private int UpdateStatusDirect(Guid id, FileStatus status)
267 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
273 using (var connection = GetConnection())
275 var command = new SQLiteCommand("update FileState set FileStatus= :fileStatus where Id = :id ",
278 command.Parameters.AddWithValue("fileStatus", status);
280 command.Parameters.AddWithValue("id", id);
282 var affected = command.ExecuteNonQuery();
288 catch (Exception exc)
290 Log.Error(exc.ToString());
296 private int UpdateStatusDirect(string path, FileStatus status)
298 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
305 using (var connection = GetConnection())
308 new SQLiteCommand("update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE",
313 command.Parameters.AddWithValue("fileStatus", status);
315 command.Parameters.AddWithValue("path", path);
317 var affected = command.ExecuteNonQuery();
321 catch (Exception exc)
323 Log.Error(exc.ToString());
329 private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus)
331 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
338 using (var connection = GetConnection())
342 "update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE ",
346 command.Parameters.AddWithValue("path", absolutePath);
347 command.Parameters.AddWithValue("fileStatus", fileStatus);
348 command.Parameters.AddWithValue("overlayStatus", overlayStatus);
350 var affected = command.ExecuteNonQuery();
354 catch (Exception exc)
356 Log.Error(exc.ToString());
364 public string BlockHash { get; set; }
366 public int BlockSize { get; set; }
367 public void ChangeRoots(string oldPath, string newPath)
369 if (String.IsNullOrWhiteSpace(oldPath))
370 throw new ArgumentNullException("oldPath");
371 if (!Path.IsPathRooted(oldPath))
372 throw new ArgumentException("oldPath must be an absolute path", "oldPath");
373 if (string.IsNullOrWhiteSpace(newPath))
374 throw new ArgumentNullException("newPath");
375 if (!Path.IsPathRooted(newPath))
376 throw new ArgumentException("newPath must be an absolute path", "newPath");
377 Contract.EndContractBlock();
379 FileState.ChangeRootPath(oldPath,newPath);
383 private PithosStatus _pithosStatus=PithosStatus.InSynch;
385 public void SetPithosStatus(PithosStatus status)
387 _pithosStatus = status;
390 public PithosStatus GetPithosStatus()
392 return _pithosStatus;
396 private readonly string _pithosDataPath;
399 public FileState GetStateByFilePath(string path)
401 if (String.IsNullOrWhiteSpace(path))
402 throw new ArgumentNullException("path");
403 if (!Path.IsPathRooted(path))
404 throw new ArgumentException("The path must be rooted", "path");
405 Contract.EndContractBlock();
410 using (var connection = GetConnection())
411 using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,Version ,VersionTimeStamp,IsShared ,SharedBy ,ShareWrite from FileState where FilePath=:path COLLATE NOCASE", connection))
414 command.Parameters.AddWithValue("path", path);
416 using (var reader = command.ExecuteReader())
420 //var values = new object[reader.FieldCount];
421 //reader.GetValues(values);
422 var state = new FileState
424 Id = reader.GetGuid(0),
425 FilePath = reader.IsDBNull(1)?"":reader.GetString(1),
426 OverlayStatus =reader.IsDBNull(2)?FileOverlayStatus.Unversioned: (FileOverlayStatus) reader.GetInt64(2),
427 FileStatus = reader.IsDBNull(3)?FileStatus.Missing:(FileStatus) reader.GetInt64(3),
428 Checksum = reader.IsDBNull(4)?"":reader.GetString(4),
429 Version = reader.IsDBNull(5)?default(long):reader.GetInt64(5),
430 VersionTimeStamp = reader.IsDBNull(6)?default(DateTime):reader.GetDateTime(6),
431 IsShared = !reader.IsDBNull(7) && reader.GetBoolean(7),
432 SharedBy = reader.IsDBNull(8)?"":reader.GetString(8),
433 ShareWrite = !reader.IsDBNull(9) && reader.GetBoolean(9)
436 var state = new FileState
438 Id = (Guid) values[0],
439 FilePath = (string) values[1],
440 OverlayStatus = (FileOverlayStatus) (long)values[2],
441 FileStatus = (FileStatus) (long)values[3],
442 Checksum = (string) values[4],
443 Version = (long?) values[5],
444 VersionTimeStamp = (DateTime?) values[6],
445 IsShared = (long)values[7] == 1,
446 SharedBy = (string) values[8],
447 ShareWrite = (long)values[9] == 1
460 catch (Exception exc)
462 Log.ErrorFormat(exc.ToString());
467 public FileOverlayStatus GetFileOverlayStatus(string path)
469 if (String.IsNullOrWhiteSpace(path))
470 throw new ArgumentNullException("path");
471 if (!Path.IsPathRooted(path))
472 throw new ArgumentException("The path must be rooted", "path");
473 Contract.EndContractBlock();
478 using (var connection = GetConnection())
479 using (var command = new SQLiteCommand("select OverlayStatus from FileState where FilePath=:path COLLATE NOCASE", connection))
482 command.Parameters.AddWithValue("path", path);
484 var s = command.ExecuteScalar();
485 return (FileOverlayStatus) Convert.ToInt32(s);
488 catch (Exception exc)
490 Log.ErrorFormat(exc.ToString());
491 return FileOverlayStatus.Unversioned;
495 private string GetConnectionString()
497 var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N;Pooling=True", _pithosDataPath);
498 return connectionString;
501 private SQLiteConnection GetConnection()
503 var connectionString = GetConnectionString();
504 var connection = new SQLiteConnection(connectionString);
506 using(var cmd =connection.CreateCommand())
508 cmd.CommandText = "PRAGMA journal_mode=WAL";
509 cmd.ExecuteNonQuery();
514 public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
516 if (String.IsNullOrWhiteSpace(path))
517 throw new ArgumentNullException("path");
518 if (!Path.IsPathRooted(path))
519 throw new ArgumentException("The path must be rooted","path");
520 Contract.EndContractBlock();
522 _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path,overlayStatus));
525 /* public void RenameFileOverlayStatus(string oldPath, string newPath)
527 if (String.IsNullOrWhiteSpace(oldPath))
528 throw new ArgumentNullException("oldPath");
529 if (!Path.IsPathRooted(oldPath))
530 throw new ArgumentException("The oldPath must be rooted", "oldPath");
531 if (String.IsNullOrWhiteSpace(newPath))
532 throw new ArgumentNullException("newPath");
533 if (!Path.IsPathRooted(newPath))
534 throw new ArgumentException("The newPath must be rooted", "newPath");
535 Contract.EndContractBlock();
537 _persistenceAgent.Post(() =>FileState.RenameState(oldPath, newPath));
540 public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus)
542 if (String.IsNullOrWhiteSpace(path))
543 throw new ArgumentNullException("path");
544 if (!Path.IsPathRooted(path))
545 throw new ArgumentException("The path must be rooted", "path");
546 Contract.EndContractBlock();
548 Debug.Assert(!path.Contains(FolderConstants.CacheFolder));
549 Debug.Assert(!path.EndsWith(".ignore"));
551 _persistenceAgent.Post(() => UpdateStatusDirect(path, fileStatus, overlayStatus));
555 public void StoreInfo(string path,ObjectInfo objectInfo)
557 if (String.IsNullOrWhiteSpace(path))
558 throw new ArgumentNullException("path");
559 if (!Path.IsPathRooted(path))
560 throw new ArgumentException("The path must be rooted", "path");
561 if (objectInfo == null)
562 throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");
563 Contract.EndContractBlock();
565 _persistenceAgent.Post(() =>
567 var filePath = path.ToLower();
568 //Load the existing files state and set its properties in one session
569 using (new SessionScope())
571 //Forgetting to use a sessionscope results in two sessions being created, one by
572 //FirstOrDefault and one by Save()
573 var state =FileState.FindByFilePath(filePath);
575 //Create a new empty state object if this is a new file
576 state = state ?? new FileState();
578 state.FilePath = filePath;
579 state.Checksum = objectInfo.Hash;
580 state.Version = objectInfo.Version;
581 state.VersionTimeStamp = objectInfo.VersionTimestamp;
583 state.FileStatus = FileStatus.Unchanged;
584 state.OverlayStatus = FileOverlayStatus.Normal;
595 public void StoreInfo(string path, ObjectInfo objectInfo)
597 if (String.IsNullOrWhiteSpace(path))
598 throw new ArgumentNullException("path");
599 if (!Path.IsPathRooted(path))
600 throw new ArgumentException("The path must be rooted", "path");
601 if (objectInfo == null)
602 throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");
603 Contract.EndContractBlock();
605 _persistenceAgent.Post(() => StoreInfoDirect(path, objectInfo));
609 private void StoreInfoDirect(string path, ObjectInfo objectInfo)
614 using (var connection = GetConnection())
615 using (var command = new SQLiteCommand(connection))
617 if (StateExists(path, connection))
618 command.CommandText =
619 "update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE ";
622 command.CommandText =
623 "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,FileStatus,OverlayStatus) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:fileStatus,:overlayStatus)";
624 command.Parameters.AddWithValue("id", Guid.NewGuid());
627 command.Parameters.AddWithValue("path", path);
628 command.Parameters.AddWithValue("checksum", objectInfo.Hash);
629 command.Parameters.AddWithValue("version", objectInfo.Version);
630 command.Parameters.AddWithValue("versionTimeStamp",
631 objectInfo.VersionTimestamp);
632 command.Parameters.AddWithValue("fileStatus", FileStatus.Unchanged);
633 command.Parameters.AddWithValue("overlayStatus",
634 FileOverlayStatus.Normal);
636 var affected = command.ExecuteNonQuery();
640 catch (Exception exc)
642 Log.Error(exc.ToString());
647 private bool StateExists(string filePath,SQLiteConnection connection)
649 using (var command = new SQLiteCommand("Select count(*) from FileState where FilePath=:path COLLATE NOCASE", connection))
651 command.Parameters.AddWithValue("path", filePath);
652 var result = command.ExecuteScalar();
653 return ((long)result >= 1);
658 public void SetFileStatus(string path, FileStatus status)
660 if (String.IsNullOrWhiteSpace(path))
661 throw new ArgumentNullException("path");
662 if (!Path.IsPathRooted(path))
663 throw new ArgumentException("The path must be rooted", "path");
664 Contract.EndContractBlock();
666 _persistenceAgent.Post(() => UpdateStatusDirect(path, status));
669 public FileStatus GetFileStatus(string path)
671 if (String.IsNullOrWhiteSpace(path))
672 throw new ArgumentNullException("path");
673 if (!Path.IsPathRooted(path))
674 throw new ArgumentException("The path must be rooted", "path");
675 Contract.EndContractBlock();
678 using (var connection = GetConnection())
680 var command = new SQLiteCommand("select FileStatus from FileState where FilePath=:path COLLATE NOCASE", connection);
681 command.Parameters.AddWithValue("path", path);
683 var statusValue = command.ExecuteScalar();
684 if (statusValue==null)
685 return FileStatus.Missing;
686 return (FileStatus)Convert.ToInt32(statusValue);
691 /// Deletes the status of the specified file
693 /// <param name="path"></param>
694 public void ClearFileStatus(string path)
696 if (String.IsNullOrWhiteSpace(path))
697 throw new ArgumentNullException("path");
698 if (!Path.IsPathRooted(path))
699 throw new ArgumentException("The path must be rooted", "path");
700 Contract.EndContractBlock();
702 _persistenceAgent.Post(() => DeleteDirect(path));
706 /// Deletes the status of the specified folder and all its contents
708 /// <param name="path"></param>
709 public void ClearFolderStatus(string path)
711 if (String.IsNullOrWhiteSpace(path))
712 throw new ArgumentNullException("path");
713 if (!Path.IsPathRooted(path))
714 throw new ArgumentException("The path must be rooted", "path");
715 Contract.EndContractBlock();
717 _persistenceAgent.Post(() => DeleteFolderDirect(path));
720 public IEnumerable<FileState> GetChildren(FileState fileState)
722 if (fileState == null)
723 throw new ArgumentNullException("fileState");
724 Contract.EndContractBlock();
726 var children = from state in FileState.Queryable
727 where state.FilePath.StartsWith(fileState.FilePath + "\\")
732 private int DeleteDirect(string filePath)
734 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
741 using (var connection = GetConnection())
743 var command = new SQLiteCommand("delete from FileState where FilePath = :path COLLATE NOCASE",
746 command.Parameters.AddWithValue("path", filePath);
748 var affected = command.ExecuteNonQuery();
752 catch (Exception exc)
754 Log.Error(exc.ToString());
760 private int DeleteFolderDirect(string filePath)
762 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
769 using (var connection = GetConnection())
771 var command = new SQLiteCommand("delete from FileState where FilePath = :path or FilePath like :path + '/%' COLLATE NOCASE",
774 command.Parameters.AddWithValue("path", filePath);
776 var affected = command.ExecuteNonQuery();
780 catch (Exception exc)
782 Log.Error(exc.ToString());
788 public void UpdateFileChecksum(string path, string checksum)
790 if (String.IsNullOrWhiteSpace(path))
791 throw new ArgumentNullException("path");
792 if (!Path.IsPathRooted(path))
793 throw new ArgumentException("The path must be rooted", "path");
794 Contract.EndContractBlock();
796 _persistenceAgent.Post(() => FileState.UpdateChecksum(path, checksum));