2 /* -----------------------------------------------------------------------
3 * <copyright file="StatusAgent.cs" company="GRNet">
5 * Copyright 2011-2012 GRNET S.A. All rights reserved.
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
11 * 1. Redistributions of source code must retain the above
12 * copyright notice, this list of conditions and the following
15 * 2. Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials
18 * provided with the distribution.
21 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
34 * The views and conclusions contained in the software and
35 * documentation are those of the authors and should not be
36 * interpreted as representing official policies, either expressed
37 * or implied, of GRNET S.A.
39 * -----------------------------------------------------------------------
43 using System.Collections.Generic;
44 using System.ComponentModel.Composition;
45 using System.Data.SQLite;
46 using System.Diagnostics;
47 using System.Diagnostics.Contracts;
50 using System.Reflection;
52 using System.Threading;
53 using System.Threading.Tasks;
54 using Castle.ActiveRecord;
55 using Castle.ActiveRecord.Framework.Config;
56 using Pithos.Interfaces;
60 namespace Pithos.Core.Agents
62 [Export(typeof(IStatusChecker)),Export(typeof(IStatusKeeper))]
63 public class StatusAgent:IStatusChecker,IStatusKeeper
65 private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
67 [System.ComponentModel.Composition.Import]
68 public IPithosSettings Settings { get; set; }
70 private Agent<Action> _persistenceAgent;
76 var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
80 _pithosDataPath = Path.Combine(appDataPath , "GRNET");
81 if (!Directory.Exists(_pithosDataPath))
82 Directory.CreateDirectory(_pithosDataPath);
84 var dbPath = Path.Combine(_pithosDataPath, "pithos.db");
86 MigrateOldDb(dbPath, appDataPath);
88 var source = GetConfiguration(_pithosDataPath);
89 ActiveRecordStarter.Initialize(source,typeof(FileState),typeof(FileTag));
90 ActiveRecordStarter.UpdateSchema();
93 if (!File.Exists(dbPath))
94 ActiveRecordStarter.CreateSchema();
101 private static void MigrateOldDb(string dbPath, string appDataPath)
103 Contract.Requires(!String.IsNullOrWhiteSpace(dbPath));
104 Contract.Requires(!String.IsNullOrWhiteSpace(appDataPath));
106 var oldDbPath = Path.Combine(appDataPath, "Pithos", "pithos.db");
107 var oldDbInfo = new FileInfo(oldDbPath);
108 if (oldDbInfo.Exists && !File.Exists(dbPath))
110 var oldDirectory = oldDbInfo.Directory;
111 oldDbInfo.MoveTo(dbPath);
112 oldDirectory.Delete(true);
116 private void CreateTrigger()
118 using (var connection = GetConnection())
119 using (var triggerCommand = connection.CreateCommand())
121 var cmdText = new StringBuilder()
122 .AppendLine("CREATE TRIGGER IF NOT EXISTS update_last_modified UPDATE ON FileState FOR EACH ROW")
124 .AppendLine("UPDATE FileState SET Modified=datetime('now') WHERE Id=old.Id;")
126 .AppendLine("CREATE TRIGGER IF NOT EXISTS insert_last_modified INSERT ON FileState FOR EACH ROW")
128 .AppendLine("UPDATE FileState SET Modified=datetime('now') WHERE Id=new.Id;")
131 triggerCommand.CommandText = cmdText;
132 triggerCommand.ExecuteNonQuery();
137 private static InPlaceConfigurationSource GetConfiguration(string pithosDbPath)
139 if (String.IsNullOrWhiteSpace(pithosDbPath))
140 throw new ArgumentNullException("pithosDbPath");
141 if (!Path.IsPathRooted(pithosDbPath))
142 throw new ArgumentException("path must be a rooted path", "pithosDbPath");
143 Contract.EndContractBlock();
145 var properties = new Dictionary<string, string>
147 {"connection.driver_class", "NHibernate.Driver.SQLite20Driver"},
148 {"dialect", "NHibernate.Dialect.SQLiteDialect"},
149 {"connection.provider", "NHibernate.Connection.DriverConnectionProvider"},
151 "proxyfactory.factory_class",
152 "NHibernate.ByteCode.Castle.ProxyFactoryFactory, NHibernate.ByteCode.Castle"
156 var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N", pithosDbPath);
157 properties.Add("connection.connection_string", connectionString);
159 var source = new InPlaceConfigurationSource();
160 source.Add(typeof (ActiveRecordBase), properties);
161 source.SetDebugFlag(false);
165 public void StartProcessing(CancellationToken token)
167 _persistenceAgent = Agent<Action>.Start(queue =>
172 var job = queue.Receive();
173 job.ContinueWith(t =>
175 var action = job.Result;
180 catch (SQLiteException ex)
182 Log.ErrorFormat("[ERROR] SQL \n{0}", ex);
186 Log.ErrorFormat("[ERROR] STATE \n{0}", ex);
188 // ReSharper disable AccessToModifiedClosure
190 // ReSharper restore AccessToModifiedClosure
202 _persistenceAgent.Stop();
206 public void ProcessExistingFiles(IEnumerable<FileInfo> existingFiles)
208 if(existingFiles ==null)
209 throw new ArgumentNullException("existingFiles");
210 Contract.EndContractBlock();
212 //Find new or matching files with a left join to the stored states
213 var fileStates = FileState.Queryable;
214 var currentFiles=from file in existingFiles
215 join state in fileStates on file.FullName.ToLower() equals state.FilePath.ToLower() into gs
216 from substate in gs.DefaultIfEmpty()
217 select new {File = file, State = substate};
219 //To get the deleted files we must get the states that have no corresponding
221 //We can't use the File.Exists method inside a query, so we get all file paths from the states
222 var statePaths = (from state in fileStates
223 select new {state.Id, state.FilePath}).ToList();
225 var missingStates= (from path in statePaths
226 where !File.Exists(path.FilePath) && !Directory.Exists(path.FilePath)
227 select path.Id).ToList();
228 //Finally, retrieve the states that correspond to the deleted files
229 var deletedFiles = from state in fileStates
230 where missingStates.Contains(state.Id)
231 select new { File = default(FileInfo), State = state };
233 var pairs = currentFiles.Union(deletedFiles);
235 foreach(var pair in pairs)
237 var fileState = pair.State;
238 var file = pair.File;
239 if (fileState == null)
242 var fullPath = pair.File.FullName;
243 var createState = FileState.CreateForAsync(fullPath, BlockSize, BlockHash);
244 createState.ContinueWith(state => _persistenceAgent.Post(state.Result.Create));
246 else if (file == null)
248 //This file was deleted while we were down. We should mark it as deleted
249 //We have to go through UpdateStatus here because the state object we are using
250 //was created by a different ORM session.
251 _persistenceAgent.Post(()=> UpdateStatusDirect(fileState.Id, FileStatus.Deleted));
255 //This file has a matching state. Need to check for possible changes
256 var hashString = file.CalculateHash(BlockSize,BlockHash);
257 //TODO: Need a way to attach the hashes to the filestate so we don't
258 //recalculate them each time a call to calculate has is made
259 //We can either store them to the filestate or add them to a
262 //If the hashes don't match the file was changed
263 if (fileState.Checksum != hashString)
265 _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Modified));
272 private int UpdateStatusDirect(Guid id, FileStatus status)
274 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
280 using (var connection = GetConnection())
282 var command = new SQLiteCommand("update FileState set FileStatus= :fileStatus where Id = :id ",
285 command.Parameters.AddWithValue("fileStatus", status);
287 command.Parameters.AddWithValue("id", id);
289 var affected = command.ExecuteNonQuery();
295 catch (Exception exc)
297 Log.Error(exc.ToString());
303 private int UpdateStatusDirect(string path, FileStatus status)
305 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
312 using (var connection = GetConnection())
315 new SQLiteCommand("update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE",
320 command.Parameters.AddWithValue("fileStatus", status);
322 command.Parameters.AddWithValue("path", path);
324 var affected = command.ExecuteNonQuery();
328 catch (Exception exc)
330 Log.Error(exc.ToString());
336 private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus)
338 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
345 using (var connection = GetConnection())
349 "update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE ",
353 command.Parameters.AddWithValue("path", absolutePath);
354 command.Parameters.AddWithValue("fileStatus", fileStatus);
355 command.Parameters.AddWithValue("overlayStatus", overlayStatus);
357 var affected = command.ExecuteNonQuery();
361 catch (Exception exc)
363 Log.Error(exc.ToString());
371 public string BlockHash { get; set; }
373 public int BlockSize { get; set; }
374 public void ChangeRoots(string oldPath, string newPath)
376 if (String.IsNullOrWhiteSpace(oldPath))
377 throw new ArgumentNullException("oldPath");
378 if (!Path.IsPathRooted(oldPath))
379 throw new ArgumentException("oldPath must be an absolute path", "oldPath");
380 if (string.IsNullOrWhiteSpace(newPath))
381 throw new ArgumentNullException("newPath");
382 if (!Path.IsPathRooted(newPath))
383 throw new ArgumentException("newPath must be an absolute path", "newPath");
384 Contract.EndContractBlock();
386 FileState.ChangeRootPath(oldPath,newPath);
390 private PithosStatus _pithosStatus=PithosStatus.InSynch;
392 public void SetPithosStatus(PithosStatus status)
394 _pithosStatus = status;
397 public PithosStatus GetPithosStatus()
399 return _pithosStatus;
403 private readonly string _pithosDataPath;
406 public FileState GetStateByFilePath(string path)
408 if (String.IsNullOrWhiteSpace(path))
409 throw new ArgumentNullException("path");
410 if (!Path.IsPathRooted(path))
411 throw new ArgumentException("The path must be rooted", "path");
412 Contract.EndContractBlock();
417 using (var connection = GetConnection())
418 using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,Version ,VersionTimeStamp,IsShared ,SharedBy ,ShareWrite from FileState where FilePath=:path COLLATE NOCASE", connection))
421 command.Parameters.AddWithValue("path", path);
423 using (var reader = command.ExecuteReader())
427 //var values = new object[reader.FieldCount];
428 //reader.GetValues(values);
429 var state = new FileState
431 Id = reader.GetGuid(0),
432 FilePath = reader.IsDBNull(1)?"":reader.GetString(1),
433 OverlayStatus =reader.IsDBNull(2)?FileOverlayStatus.Unversioned: (FileOverlayStatus) reader.GetInt64(2),
434 FileStatus = reader.IsDBNull(3)?FileStatus.Missing:(FileStatus) reader.GetInt64(3),
435 Checksum = reader.IsDBNull(4)?"":reader.GetString(4),
436 Version = reader.IsDBNull(5)?default(long):reader.GetInt64(5),
437 VersionTimeStamp = reader.IsDBNull(6)?default(DateTime):reader.GetDateTime(6),
438 IsShared = !reader.IsDBNull(7) && reader.GetBoolean(7),
439 SharedBy = reader.IsDBNull(8)?"":reader.GetString(8),
440 ShareWrite = !reader.IsDBNull(9) && reader.GetBoolean(9)
443 var state = new FileState
445 Id = (Guid) values[0],
446 FilePath = (string) values[1],
447 OverlayStatus = (FileOverlayStatus) (long)values[2],
448 FileStatus = (FileStatus) (long)values[3],
449 Checksum = (string) values[4],
450 Version = (long?) values[5],
451 VersionTimeStamp = (DateTime?) values[6],
452 IsShared = (long)values[7] == 1,
453 SharedBy = (string) values[8],
454 ShareWrite = (long)values[9] == 1
467 catch (Exception exc)
469 Log.ErrorFormat(exc.ToString());
474 public FileOverlayStatus GetFileOverlayStatus(string path)
476 if (String.IsNullOrWhiteSpace(path))
477 throw new ArgumentNullException("path");
478 if (!Path.IsPathRooted(path))
479 throw new ArgumentException("The path must be rooted", "path");
480 Contract.EndContractBlock();
485 using (var connection = GetConnection())
486 using (var command = new SQLiteCommand("select OverlayStatus from FileState where FilePath=:path COLLATE NOCASE", connection))
489 command.Parameters.AddWithValue("path", path);
491 var s = command.ExecuteScalar();
492 return (FileOverlayStatus) Convert.ToInt32(s);
495 catch (Exception exc)
497 Log.ErrorFormat(exc.ToString());
498 return FileOverlayStatus.Unversioned;
502 private string GetConnectionString()
504 var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N;Pooling=True", _pithosDataPath);
505 return connectionString;
508 private SQLiteConnection GetConnection()
510 var connectionString = GetConnectionString();
511 var connection = new SQLiteConnection(connectionString);
513 using(var cmd =connection.CreateCommand())
515 cmd.CommandText = "PRAGMA journal_mode=WAL";
516 cmd.ExecuteNonQuery();
521 public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
523 if (String.IsNullOrWhiteSpace(path))
524 throw new ArgumentNullException("path");
525 if (!Path.IsPathRooted(path))
526 throw new ArgumentException("The path must be rooted","path");
527 Contract.EndContractBlock();
529 _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path,overlayStatus));
532 /* public void RenameFileOverlayStatus(string oldPath, string newPath)
534 if (String.IsNullOrWhiteSpace(oldPath))
535 throw new ArgumentNullException("oldPath");
536 if (!Path.IsPathRooted(oldPath))
537 throw new ArgumentException("The oldPath must be rooted", "oldPath");
538 if (String.IsNullOrWhiteSpace(newPath))
539 throw new ArgumentNullException("newPath");
540 if (!Path.IsPathRooted(newPath))
541 throw new ArgumentException("The newPath must be rooted", "newPath");
542 Contract.EndContractBlock();
544 _persistenceAgent.Post(() =>FileState.RenameState(oldPath, newPath));
547 public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus)
549 if (String.IsNullOrWhiteSpace(path))
550 throw new ArgumentNullException("path");
551 if (!Path.IsPathRooted(path))
552 throw new ArgumentException("The path must be rooted", "path");
553 Contract.EndContractBlock();
555 Debug.Assert(!path.Contains(FolderConstants.CacheFolder));
556 Debug.Assert(!path.EndsWith(".ignore"));
558 _persistenceAgent.Post(() => UpdateStatusDirect(path, fileStatus, overlayStatus));
562 public void StoreInfo(string path,ObjectInfo objectInfo)
564 if (String.IsNullOrWhiteSpace(path))
565 throw new ArgumentNullException("path");
566 if (!Path.IsPathRooted(path))
567 throw new ArgumentException("The path must be rooted", "path");
568 if (objectInfo == null)
569 throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");
570 Contract.EndContractBlock();
572 _persistenceAgent.Post(() =>
574 var filePath = path.ToLower();
575 //Load the existing files state and set its properties in one session
576 using (new SessionScope())
578 //Forgetting to use a sessionscope results in two sessions being created, one by
579 //FirstOrDefault and one by Save()
580 var state =FileState.FindByFilePath(filePath);
582 //Create a new empty state object if this is a new file
583 state = state ?? new FileState();
585 state.FilePath = filePath;
586 state.Checksum = objectInfo.Hash;
587 state.Version = objectInfo.Version;
588 state.VersionTimeStamp = objectInfo.VersionTimestamp;
590 state.FileStatus = FileStatus.Unchanged;
591 state.OverlayStatus = FileOverlayStatus.Normal;
602 public void StoreInfo(string path, ObjectInfo objectInfo)
604 if (String.IsNullOrWhiteSpace(path))
605 throw new ArgumentNullException("path");
606 if (!Path.IsPathRooted(path))
607 throw new ArgumentException("The path must be rooted", "path");
608 if (objectInfo == null)
609 throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");
610 Contract.EndContractBlock();
612 _persistenceAgent.Post(() => StoreInfoDirect(path, objectInfo));
616 private void StoreInfoDirect(string path, ObjectInfo objectInfo)
621 using (var connection = GetConnection())
622 using (var command = new SQLiteCommand(connection))
624 if (StateExists(path, connection))
625 command.CommandText =
626 "update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE ";
629 command.CommandText =
630 "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,FileStatus,OverlayStatus) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:fileStatus,:overlayStatus)";
631 command.Parameters.AddWithValue("id", Guid.NewGuid());
634 command.Parameters.AddWithValue("path", path);
635 command.Parameters.AddWithValue("checksum", objectInfo.Hash);
636 command.Parameters.AddWithValue("version", objectInfo.Version);
637 command.Parameters.AddWithValue("versionTimeStamp",
638 objectInfo.VersionTimestamp);
639 command.Parameters.AddWithValue("fileStatus", FileStatus.Unchanged);
640 command.Parameters.AddWithValue("overlayStatus",
641 FileOverlayStatus.Normal);
643 var affected = command.ExecuteNonQuery();
647 catch (Exception exc)
649 Log.Error(exc.ToString());
654 private bool StateExists(string filePath,SQLiteConnection connection)
656 using (var command = new SQLiteCommand("Select count(*) from FileState where FilePath=:path COLLATE NOCASE", connection))
658 command.Parameters.AddWithValue("path", filePath);
659 var result = command.ExecuteScalar();
660 return ((long)result >= 1);
665 public void SetFileStatus(string path, FileStatus status)
667 if (String.IsNullOrWhiteSpace(path))
668 throw new ArgumentNullException("path");
669 if (!Path.IsPathRooted(path))
670 throw new ArgumentException("The path must be rooted", "path");
671 Contract.EndContractBlock();
673 _persistenceAgent.Post(() => UpdateStatusDirect(path, status));
676 public FileStatus GetFileStatus(string path)
678 if (String.IsNullOrWhiteSpace(path))
679 throw new ArgumentNullException("path");
680 if (!Path.IsPathRooted(path))
681 throw new ArgumentException("The path must be rooted", "path");
682 Contract.EndContractBlock();
685 using (var connection = GetConnection())
687 var command = new SQLiteCommand("select FileStatus from FileState where FilePath=:path COLLATE NOCASE", connection);
688 command.Parameters.AddWithValue("path", path);
690 var statusValue = command.ExecuteScalar();
691 if (statusValue==null)
692 return FileStatus.Missing;
693 return (FileStatus)Convert.ToInt32(statusValue);
698 /// Deletes the status of the specified file
700 /// <param name="path"></param>
701 public void ClearFileStatus(string path)
703 if (String.IsNullOrWhiteSpace(path))
704 throw new ArgumentNullException("path");
705 if (!Path.IsPathRooted(path))
706 throw new ArgumentException("The path must be rooted", "path");
707 Contract.EndContractBlock();
709 _persistenceAgent.Post(() => DeleteDirect(path));
713 /// Deletes the status of the specified folder and all its contents
715 /// <param name="path"></param>
716 public void ClearFolderStatus(string path)
718 if (String.IsNullOrWhiteSpace(path))
719 throw new ArgumentNullException("path");
720 if (!Path.IsPathRooted(path))
721 throw new ArgumentException("The path must be rooted", "path");
722 Contract.EndContractBlock();
724 _persistenceAgent.Post(() => DeleteFolderDirect(path));
727 public IEnumerable<FileState> GetChildren(FileState fileState)
729 if (fileState == null)
730 throw new ArgumentNullException("fileState");
731 Contract.EndContractBlock();
733 var children = from state in FileState.Queryable
734 where state.FilePath.StartsWith(fileState.FilePath + "\\")
739 private int DeleteDirect(string filePath)
741 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
748 using (var connection = GetConnection())
750 var command = new SQLiteCommand("delete from FileState where FilePath = :path COLLATE NOCASE",
753 command.Parameters.AddWithValue("path", filePath);
755 var affected = command.ExecuteNonQuery();
759 catch (Exception exc)
761 Log.Error(exc.ToString());
767 private int DeleteFolderDirect(string filePath)
769 using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
776 using (var connection = GetConnection())
778 var command = new SQLiteCommand("delete from FileState where FilePath = :path or FilePath like :path + '/%' COLLATE NOCASE",
781 command.Parameters.AddWithValue("path", filePath);
783 var affected = command.ExecuteNonQuery();
787 catch (Exception exc)
789 Log.Error(exc.ToString());
795 public void UpdateFileChecksum(string path, string checksum)
797 if (String.IsNullOrWhiteSpace(path))
798 throw new ArgumentNullException("path");
799 if (!Path.IsPathRooted(path))
800 throw new ArgumentException("The path must be rooted", "path");
801 Contract.EndContractBlock();
803 _persistenceAgent.Post(() => FileState.UpdateChecksum(path, checksum));