Using MD5 to quickly check for local modifications before calculating the expensive...
[pithos-ms-client] / trunk / Pithos.Core / Agents / StatusAgent.cs
index de7af7b..b95645d 100644 (file)
@@ -1,3 +1,44 @@
+#region
+/* -----------------------------------------------------------------------
+ * <copyright file="StatusAgent.cs" company="GRNet">
+ * 
+ * Copyright 2011-2012 GRNET S.A. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ *   1. Redistributions of source code must retain the above
+ *      copyright notice, this list of conditions and the following
+ *      disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above
+ *      copyright notice, this list of conditions and the following
+ *      disclaimer in the documentation and/or other materials
+ *      provided with the distribution.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and
+ * documentation are those of the authors and should not be
+ * interpreted as representing official policies, either expressed
+ * or implied, of GRNET S.A.
+ * </copyright>
+ * -----------------------------------------------------------------------
+ */
+#endregion
 using System;
 using System.Collections.Generic;
 using System.ComponentModel.Composition;
@@ -6,41 +47,107 @@ using System.Diagnostics;
 using System.Diagnostics.Contracts;
 using System.IO;
 using System.Linq;
+using System.Reflection;
+using System.Security.Cryptography;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Castle.ActiveRecord;
+using Castle.ActiveRecord.Framework;
 using Castle.ActiveRecord.Framework.Config;
+using NHibernate.ByteCode.Castle;
+using NHibernate.Cfg;
+using NHibernate.Cfg.Loquacious;
+using NHibernate.Dialect;
 using Pithos.Interfaces;
 using Pithos.Network;
 using log4net;
+using Environment = System.Environment;
 
 namespace Pithos.Core.Agents
 {
     [Export(typeof(IStatusChecker)),Export(typeof(IStatusKeeper))]
     public class StatusAgent:IStatusChecker,IStatusKeeper
     {
+        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
         [System.ComponentModel.Composition.Import]
         public IPithosSettings Settings { get; set; }
 
         private Agent<Action> _persistenceAgent;
 
 
-        private static readonly ILog Log = LogManager.GetLogger("StatusAgent");
 
         public StatusAgent()
         {            
             var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
-            _pithosDataPath = Path.Combine(appDataPath , "Pithos");
 
+            _pithosDataPath = Path.Combine(appDataPath , "GRNET\\PITHOS");
             if (!Directory.Exists(_pithosDataPath))
                 Directory.CreateDirectory(_pithosDataPath);
+
+            var dbPath = Path.Combine(_pithosDataPath, "pithos.db");
+
+            MigrateOldDb(dbPath, appDataPath);
+
+
             var source = GetConfiguration(_pithosDataPath);
             ActiveRecordStarter.Initialize(source,typeof(FileState),typeof(FileTag));
+            
             ActiveRecordStarter.UpdateSchema();
 
-            if (!File.Exists(Path.Combine(_pithosDataPath ,"pithos.db")))
+
+            if (!File.Exists(dbPath))
                 ActiveRecordStarter.CreateSchema();
-        }        
+
+            CreateTrigger();
+            
+        }
+
+
+        private static void MigrateOldDb(string dbPath, string appDataPath)
+        {
+            if(String.IsNullOrWhiteSpace(dbPath))
+                throw new ArgumentNullException("dbPath");
+            if(String.IsNullOrWhiteSpace(appDataPath))
+                throw new ArgumentNullException("appDataPath");
+            Contract.EndContractBlock();
+
+            var oldDbPath = Path.Combine(appDataPath, "Pithos", "pithos.db");
+            var oldDbInfo = new FileInfo(oldDbPath);
+            if (oldDbInfo.Exists && !File.Exists(dbPath))
+            {
+                Log.InfoFormat("Moving database from {0} to {1}",oldDbInfo.FullName,dbPath);
+                var oldDirectory = oldDbInfo.Directory;
+                oldDbInfo.MoveTo(dbPath);
+                
+                if (Log.IsDebugEnabled)
+                    Log.DebugFormat("Deleting {0}",oldDirectory.FullName);
+                
+                oldDirectory.Delete(true);
+            }
+        }
+
+        private void CreateTrigger()
+        {
+            using (var connection = GetConnection())
+            using (var triggerCommand = connection.CreateCommand())
+            {
+                var cmdText = new StringBuilder()
+                    .AppendLine("CREATE TRIGGER IF NOT EXISTS update_last_modified UPDATE ON FileState FOR EACH ROW")
+                    .AppendLine("BEGIN")
+                    .AppendLine("UPDATE FileState SET Modified=datetime('now')  WHERE Id=old.Id;")
+                    .AppendLine("END;")
+                    .AppendLine("CREATE TRIGGER IF NOT EXISTS insert_last_modified INSERT ON FileState FOR EACH ROW")
+                    .AppendLine("BEGIN")
+                    .AppendLine("UPDATE FileState SET Modified=datetime('now')  WHERE Id=new.Id;")
+                    .AppendLine("END;")
+                    .ToString();
+                triggerCommand.CommandText = cmdText;                
+                triggerCommand.ExecuteNonQuery();
+            }
+        }
+
 
         private static InPlaceConfigurationSource GetConfiguration(string pithosDbPath)
         {
@@ -61,7 +168,7 @@ namespace Pithos.Core.Agents
                                          },
                                  };
 
-            var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3", pithosDbPath);
+            var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N", pithosDbPath);
             properties.Add("connection.connection_string", connectionString);
 
             var source = new InPlaceConfigurationSource();                        
@@ -93,6 +200,7 @@ namespace Pithos.Core.Agents
                         {
                             Log.ErrorFormat("[ERROR] STATE \n{0}", ex);
                         }
+                        queue.NotifyComplete(action);
 // ReSharper disable AccessToModifiedClosure
                         queue.DoAsync(loop);
 // ReSharper restore AccessToModifiedClosure
@@ -109,7 +217,7 @@ namespace Pithos.Core.Agents
         {
             _persistenceAgent.Stop();            
         }
-       
+               
 
         public void ProcessExistingFiles(IEnumerable<FileInfo> existingFiles)
         {
@@ -138,41 +246,165 @@ namespace Pithos.Core.Agents
                         where missingStates.Contains(state.Id)
                         select new { File = default(FileInfo), State = state };
 
-            var pairs = currentFiles.Union(deletedFiles);
+            var pairs = currentFiles.Union(deletedFiles).ToList();
+
+            using (var shortHasher = HashAlgorithm.Create("sha1"))
+            {
+                foreach (var pair in pairs)
+                {
+                    var fileState = pair.State;
+                    var file = pair.File;
+                    if (fileState == null)
+                    {
+                        //This is a new file                        
+                        var createState = FileState.CreateFor(file);
+                        _persistenceAgent.Post(createState.Create);                        
+                    }
+                    else if (file == null)
+                    {
+                        //This file was deleted while we were down. We should mark it as deleted
+                        //We have to go through UpdateStatus here because the state object we are using
+                        //was created by a different ORM session.
+                        _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Deleted));
+                    }
+                    else
+                    {
+                        //This file has a matching state. Need to check for possible changes
+                        //To check for changes, we use the cheap (in CPU terms) SHA1 algorithm
+                        //on the entire file.
+                        
+                        var hashString = file.ComputeShortHash(shortHasher);                        
+                        //TODO: Need a way to attach the hashes to the filestate so we don't
+                        //recalculate them each time a call to calculate has is made
+                        //We can either store them to the filestate or add them to a 
+                        //dictionary
+
+                        //If the hashes don't match the file was changed
+                        if (fileState.ShortHash != hashString)
+                        {
+                            _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Modified));
+                        }
+                    }
+                }
+            }
+                        
+         
+        }
+        
+
 
-            Parallel.ForEach(pairs, pair =>
+        private int UpdateStatusDirect(Guid id, FileStatus status)
+        {
+            using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
             {
-                var fileState = pair.State;
-                var file = pair.File;
-                if (fileState == null)
+
+                try
                 {
-                    //This is a new file
-                    var fullPath = pair.File.FullName.ToLower();
-                    var createState = FileState.CreateForAsync(fullPath, BlockSize, BlockHash);
-                    createState.ContinueWith(state => _persistenceAgent.Post(state.Result.Create));
-                }                
-                else if (file == null)
+                    
+                    using (var connection = GetConnection())
+                    using (
+                        var command = new SQLiteCommand("update FileState set FileStatus= :fileStatus where Id = :id  ",
+                                                        connection))
+                    {                                                
+                        command.Parameters.AddWithValue("fileStatus", status);
+
+                        command.Parameters.AddWithValue("id", id);
+                        
+                        var affected = command.ExecuteNonQuery();
+                        
+                        return affected;
+                    }
+
+                }
+                catch (Exception exc)
                 {
-                    //This file was deleted while we were down. We should mark it as deleted
-                    //We have to go through UpdateStatus here because the state object we are using
-                    //was created by a different ORM session.
-                    FileState.UpdateStatus(fileState.Id,FileStatus.Deleted);                    
+                    Log.Error(exc.ToString());
+                    throw;
                 }
-                else
+            }
+        }
+        
+        private int UpdateStatusDirect(string path, FileStatus status)
+        {
+            using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
+            {
+
+                try
                 {
-                    //This file has a matching state. Need to check for possible changes
-                    var hashString = file.CalculateHash(BlockSize,BlockHash);
-                    //If the hashes don't match the file was changed
-                    if (fileState.Checksum != hashString)
+
+                    
+                    using (var connection = GetConnection())
+                    using (
+                        var command =
+                            new SQLiteCommand("update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE",
+                                              connection))
                     {
-                        FileState.UpdateStatus(fileState.Id, FileStatus.Modified);
-                    }                    
+
+
+                        command.Parameters.AddWithValue("fileStatus", status);
+
+                        command.Parameters.AddWithValue("path", path);
+                        
+                        var affected = command.ExecuteNonQuery();
+                        if (affected == 0)
+                        {
+                            var createdState = FileState.CreateFor(FileInfoExtensions.FromPath(path));
+                            createdState.FileStatus = status;
+                            createdState.Create();
+                        }
+                        return affected;
+                    }
                 }
-            });            
-         
+                catch (Exception exc)
+                {
+                    Log.Error(exc.ToString());
+                    throw;
+                }
+            }
         }
 
-       
+        private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason)
+        {
+            using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
+            {
+
+                try
+                {
+
+                    
+                    using (var connection = GetConnection())
+                    using (
+                        var command =
+                            new SQLiteCommand(
+                                "update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus,ConflictReason= :conflictReason where FilePath = :path COLLATE NOCASE ",
+                                connection))
+                    {
+
+                        command.Parameters.AddWithValue("path", absolutePath);
+                        command.Parameters.AddWithValue("fileStatus", fileStatus);
+                        command.Parameters.AddWithValue("overlayStatus", overlayStatus);
+                        command.Parameters.AddWithValue("conflictReason", conflictReason);
+                        
+                        var affected = command.ExecuteNonQuery();
+                        if (affected == 0)
+                        {
+                            var createdState=FileState.CreateFor(FileInfoExtensions.FromPath(absolutePath));
+                            createdState.FileStatus = fileStatus;
+                            createdState.OverlayStatus = overlayStatus;
+                            createdState.ConflictReason = conflictReason;
+                            createdState.Create();  
+                        }
+                        return affected;
+                    }
+                }
+                catch (Exception exc)
+                {
+                    Log.Error(exc.ToString());
+                    throw;
+                }
+            }
+        }
+        
 
 
         public string BlockHash { get; set; }
@@ -194,21 +426,79 @@ namespace Pithos.Core.Agents
 
         }
 
-        private PithosStatus _pithosStatus=PithosStatus.InSynch;       
 
-        public void SetPithosStatus(PithosStatus status)
-        {
-            _pithosStatus = status;
-        }
 
-        public PithosStatus GetPithosStatus()
-        {
-            return _pithosStatus;
-        }
+        private readonly string _pithosDataPath;
 
 
-        private readonly string _pithosDataPath;
+        public FileState GetStateByFilePath(string path)
+        {
+            if (String.IsNullOrWhiteSpace(path))
+                throw new ArgumentNullException("path");
+            if (!Path.IsPathRooted(path))
+                throw new ArgumentException("The path must be rooted", "path");
+            Contract.EndContractBlock();
 
+            try
+            {
+                
+                using (var connection = GetConnection())
+                using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,ShortHash,Version    ,VersionTimeStamp,IsShared   ,SharedBy   ,ShareWrite  from FileState where FilePath=:path COLLATE NOCASE", connection))
+                {
+                    
+                    command.Parameters.AddWithValue("path", path);
+                    
+                    using (var reader = command.ExecuteReader())
+                    {
+                        if (reader.Read())
+                        {
+                            //var values = new object[reader.FieldCount];
+                            //reader.GetValues(values);
+                            var state = new FileState
+                                            {
+                                                Id = reader.GetGuid(0),
+                                                FilePath = reader.IsDBNull(1)?"":reader.GetString(1),
+                                                OverlayStatus =reader.IsDBNull(2)?FileOverlayStatus.Unversioned: (FileOverlayStatus) reader.GetInt64(2),
+                                                FileStatus = reader.IsDBNull(3)?FileStatus.Missing:(FileStatus) reader.GetInt64(3),
+                                                Checksum = reader.IsDBNull(4)?"":reader.GetString(4),
+                                                ShortHash= reader.IsDBNull(5)?"":reader.GetString(5),
+                                                Version = reader.IsDBNull(6)?default(long):reader.GetInt64(6),
+                                                VersionTimeStamp = reader.IsDBNull(7)?default(DateTime):reader.GetDateTime(7),
+                                                IsShared = !reader.IsDBNull(8) && reader.GetBoolean(8),
+                                                SharedBy = reader.IsDBNull(9)?"":reader.GetString(9),
+                                                ShareWrite = !reader.IsDBNull(10) && reader.GetBoolean(10)
+                                            };
+/*
+                            var state = new FileState
+                                            {
+                                                Id = (Guid) values[0],
+                                                FilePath = (string) values[1],
+                                                OverlayStatus = (FileOverlayStatus) (long)values[2],
+                                                FileStatus = (FileStatus) (long)values[3],
+                                                Checksum = (string) values[4],
+                                                Version = (long?) values[5],
+                                                VersionTimeStamp = (DateTime?) values[6],
+                                                IsShared = (long)values[7] == 1,
+                                                SharedBy = (string) values[8],
+                                                ShareWrite = (long)values[9] == 1
+                                            };
+*/
+                            return state;
+                        }
+                        else
+                        {
+                            return null;
+                        }
+
+                    }                    
+                }
+            }
+            catch (Exception exc)
+            {
+                Log.ErrorFormat(exc.ToString());
+                throw;
+            }            
+        }
 
         public FileOverlayStatus GetFileOverlayStatus(string path)
         {
@@ -220,11 +510,16 @@ namespace Pithos.Core.Agents
 
             try
             {
-
-                var status = from state in  FileState.Queryable 
-                                 where state.FilePath ==path.ToLower()
-                                 select state.OverlayStatus;
-                return status.Any()? status.First():FileOverlayStatus.Unversioned;
+                
+                using (var connection = GetConnection())
+                using (var command = new SQLiteCommand("select OverlayStatus from FileState where FilePath=:path  COLLATE NOCASE", connection))
+                {
+                    
+                    command.Parameters.AddWithValue("path", path);
+                    
+                    var s = command.ExecuteScalar();
+                    return (FileOverlayStatus) Convert.ToInt32(s);
+                }
             }
             catch (Exception exc)
             {
@@ -233,7 +528,26 @@ namespace Pithos.Core.Agents
             }
         }
 
-        public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
+        private string GetConnectionString()
+        {
+            var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N;Pooling=True", _pithosDataPath);
+            return connectionString;
+        }
+
+        private SQLiteConnection GetConnection()
+        {
+            var connectionString = GetConnectionString();
+            var connection = new SQLiteConnection(connectionString);
+            connection.Open();
+            using(var cmd =connection.CreateCommand())
+            {
+                cmd.CommandText = "PRAGMA journal_mode=WAL";
+                cmd.ExecuteNonQuery();
+            }
+            return connection;
+        }
+
+       /* public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
         {
             if (String.IsNullOrWhiteSpace(path))
                 throw new ArgumentNullException("path");
@@ -241,7 +555,18 @@ namespace Pithos.Core.Agents
                 throw new ArgumentException("The path must be rooted","path");
             Contract.EndContractBlock();
 
-            _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path.ToLower(),overlayStatus));
+            _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path,overlayStatus));
+        }*/
+
+        public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string shortHash = null)
+        {
+            if (String.IsNullOrWhiteSpace(path))
+                throw new ArgumentNullException("path");
+            if (!Path.IsPathRooted(path))
+                throw new ArgumentException("The path must be rooted","path");
+            Contract.EndContractBlock();
+
+            return _persistenceAgent.PostAndAwait(() => FileState.StoreOverlayStatus(path,overlayStatus,shortHash));
         }
 
        /* public void RenameFileOverlayStatus(string oldPath, string newPath)
@@ -259,7 +584,7 @@ namespace Pithos.Core.Agents
             _persistenceAgent.Post(() =>FileState.RenameState(oldPath, newPath));
         }*/
 
-        public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus)
+        public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason)
         {
             if (String.IsNullOrWhiteSpace(path))
                 throw new ArgumentNullException("path");
@@ -270,9 +595,10 @@ namespace Pithos.Core.Agents
             Debug.Assert(!path.Contains(FolderConstants.CacheFolder));
             Debug.Assert(!path.EndsWith(".ignore"));
 
-            _persistenceAgent.Post(() => FileState.UpdateStatus(path.ToLower(), fileStatus, overlayStatus));
+            _persistenceAgent.Post(() => UpdateStatusDirect(path, fileStatus, overlayStatus, conflictReason));
         }
 
+/*
         public void StoreInfo(string path,ObjectInfo objectInfo)
         {
             if (String.IsNullOrWhiteSpace(path))
@@ -311,17 +637,80 @@ namespace Pithos.Core.Agents
             });
 
         }
-
+*/
         
-        public void SetFileStatus(string path, FileStatus status)
+        public void StoreInfo(string path, ObjectInfo objectInfo)
         {
             if (String.IsNullOrWhiteSpace(path))
                 throw new ArgumentNullException("path");
             if (!Path.IsPathRooted(path))
                 throw new ArgumentException("The path must be rooted", "path");
+            if (objectInfo == null)
+                throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");
             Contract.EndContractBlock();
 
-            _persistenceAgent.Post(() => FileState.UpdateStatus(path.ToLower(), status));
+            _persistenceAgent.Post(() => StoreInfoDirect(path, objectInfo));
+
+        }
+
+        private void StoreInfoDirect(string path, ObjectInfo objectInfo)
+        {
+            try
+            {
+                
+                using (var connection = GetConnection())
+                using (var command = new SQLiteCommand(connection))
+                {
+                    if (StateExists(path, connection))
+                        command.CommandText =
+                            "update FileState set FileStatus= :fileStatus where FilePath = :path  COLLATE NOCASE ";
+                    else
+                    {
+                        command.CommandText =
+                            "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,ShortHash,FileStatus,OverlayStatus,ObjectID) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:shortHash,:fileStatus,:overlayStatus,:objectID)";
+                        command.Parameters.AddWithValue("id", Guid.NewGuid());
+                    }
+
+                    command.Parameters.AddWithValue("path", path);
+                    command.Parameters.AddWithValue("checksum", objectInfo.X_Object_Hash);
+                    command.Parameters.AddWithValue("shortHash", objectInfo.ETag);
+                    command.Parameters.AddWithValue("version", objectInfo.Version);
+                    command.Parameters.AddWithValue("versionTimeStamp", objectInfo.VersionTimestamp);
+                    command.Parameters.AddWithValue("fileStatus", FileStatus.Unchanged);
+                    command.Parameters.AddWithValue("overlayStatus", FileOverlayStatus.Normal);
+                    command.Parameters.AddWithValue("objectID",objectInfo.UUID);
+
+                    var affected = command.ExecuteNonQuery();
+                    return;
+                }
+            }
+            catch (Exception exc)
+            {
+                Log.Error(exc.ToString());
+                throw;
+            }
+        }
+
+        private bool StateExists(string filePath,SQLiteConnection connection)
+        {
+            using (var command = new SQLiteCommand("Select count(*) from FileState where FilePath=:path  COLLATE NOCASE", connection))
+            {
+                command.Parameters.AddWithValue("path", filePath);
+                var result = command.ExecuteScalar();
+                return ((long)result >= 1);
+            }
+
+        }
+
+        public void SetFileStatus(string path, FileStatus status)
+        {
+            if (String.IsNullOrWhiteSpace(path))
+                throw new ArgumentNullException("path");
+            if (!Path.IsPathRooted(path))
+                throw new ArgumentException("The path must be rooted", "path");
+            Contract.EndContractBlock();
+            
+            _persistenceAgent.Post(() => UpdateStatusDirect(path, status));
         }
 
         public FileStatus GetFileStatus(string path)
@@ -332,12 +721,23 @@ namespace Pithos.Core.Agents
                 throw new ArgumentException("The path must be rooted", "path");
             Contract.EndContractBlock();
 
-            var status = from r in FileState.Queryable
-                     where r.FilePath == path.ToLower()
-                     select r.FileStatus;                        
-            return status.Any()?status.First(): FileStatus.Missing;
+            
+            using (var connection = GetConnection())
+            {
+                var command = new SQLiteCommand("select FileStatus from FileState where FilePath=:path  COLLATE NOCASE", connection);
+                command.Parameters.AddWithValue("path", path);
+                
+                var statusValue = command.ExecuteScalar();
+                if (statusValue==null)
+                    return FileStatus.Missing;
+                return (FileStatus)Convert.ToInt32(statusValue);
+            }
         }
 
+        /// <summary>
+        /// Deletes the status of the specified file
+        /// </summary>
+        /// <param name="path"></param>
         public void ClearFileStatus(string path)
         {
             if (String.IsNullOrWhiteSpace(path))
@@ -345,11 +745,109 @@ namespace Pithos.Core.Agents
             if (!Path.IsPathRooted(path))
                 throw new ArgumentException("The path must be rooted", "path");
             Contract.EndContractBlock();
-            
-            _persistenceAgent.Post(() => FileState.DeleteByFilePath(path));   
+
+            _persistenceAgent.Post(() => DeleteDirect(path));   
+        }
+
+        /// <summary>
+        /// Deletes the status of the specified folder and all its contents
+        /// </summary>
+        /// <param name="path"></param>
+        public void ClearFolderStatus(string path)
+        {
+            if (String.IsNullOrWhiteSpace(path))
+                throw new ArgumentNullException("path");
+            if (!Path.IsPathRooted(path))
+                throw new ArgumentException("The path must be rooted", "path");
+            Contract.EndContractBlock();
+            //TODO: May throw if the agent is cleared for some reason. Should never happen
+            _persistenceAgent.Post(() => DeleteFolderDirect(path));   
+        }
+
+        public IEnumerable<FileState> GetChildren(FileState fileState)
+        {
+            if (fileState == null)
+                throw new ArgumentNullException("fileState");
+            Contract.EndContractBlock();
+
+            var children = from state in FileState.Queryable
+                           where state.FilePath.StartsWith(fileState.FilePath + "\\")
+                           select state;
+            return children;
+        }
+
+        public void EnsureFileState(string path)
+        {
+            var existingState = GetStateByFilePath(path);
+            if (existingState != null)
+                return;
+            var fileInfo = FileInfoExtensions.FromPath(path);
+            using (new SessionScope())
+            {
+                var newState = FileState.CreateFor(fileInfo);
+                newState.FileStatus=FileStatus.Missing;
+                _persistenceAgent.PostAndAwait(newState.CreateAndFlush).Wait();
+            }
+
+        }
+
+        private int DeleteDirect(string filePath)
+        {
+            using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
+            {
+
+                try
+                {
+
+                    
+                    using (var connection = GetConnection())
+                    {
+                        var command = new SQLiteCommand("delete from FileState where FilePath = :path  COLLATE NOCASE",
+                                                        connection);
+
+                        command.Parameters.AddWithValue("path", filePath);
+                        
+                        var affected = command.ExecuteNonQuery();
+                        return affected;
+                    }
+                }
+                catch (Exception exc)
+                {
+                    Log.Error(exc.ToString());
+                    throw;
+                }
+            }
+        }
+
+        private int DeleteFolderDirect(string filePath)
+        {
+            using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
+            {
+
+                try
+                {
+
+                    
+                    using (var connection = GetConnection())
+                    {
+                        var command = new SQLiteCommand(@"delete from FileState where FilePath = :path or FilePath like :path || '\%'  COLLATE NOCASE",
+                                                        connection);
+
+                        command.Parameters.AddWithValue("path", filePath);
+                        
+                        var affected = command.ExecuteNonQuery();
+                        return affected;
+                    }
+                }
+                catch (Exception exc)
+                {
+                    Log.Error(exc.ToString());
+                    throw;
+                }
+            }
         }
 
-        public void UpdateFileChecksum(string path, string checksum)
+        public void UpdateFileChecksum(string path, string shortHash, string checksum)
         {
             if (String.IsNullOrWhiteSpace(path))
                 throw new ArgumentNullException("path");
@@ -357,9 +855,68 @@ namespace Pithos.Core.Agents
                 throw new ArgumentException("The path must be rooted", "path");            
             Contract.EndContractBlock();
 
-            _persistenceAgent.Post(() => FileState.UpdateChecksum(path.ToLower(), checksum));
+            _persistenceAgent.Post(() => FileState.UpdateChecksum(path, shortHash,checksum));
         }
 
+
+        public void CleanupOrphanStates()
+        {
+            //Orphan states are those that do not correspond to an account, ie. their paths
+            //do not start with the root path of any registered account
+
+            var roots=(from account in Settings.Accounts
+                      select account.RootPath).ToList();
+            
+            var allStates = from state in FileState.Queryable
+                select state.FilePath;
+
+            foreach (var statePath in allStates)
+            {
+                if (!roots.Any(root=>statePath.StartsWith(root,StringComparison.InvariantCultureIgnoreCase)))
+                    this.DeleteDirect(statePath);
+            }
+        }
+
+        public void CleanupStaleStates(AccountInfo accountInfo, List<ObjectInfo> objectInfos)
+        {
+            if (accountInfo == null)
+                throw new ArgumentNullException("accountInfo");
+            if (objectInfos == null)
+                throw new ArgumentNullException("objectInfos");
+            Contract.EndContractBlock();
+            
+
+
+            //Stale states are those that have no corresponding local or server file
+            
+
+            var agent=FileAgent.GetFileAgent(accountInfo);
+
+            var localFiles=agent.EnumerateFiles();
+            var localSet = new HashSet<string>(localFiles);
+
+            //RelativeUrlToFilePath will fail for
+            //infos of accounts, containers which have no Name
+
+            var serverFiles = from info in objectInfos
+                              where info.Name != null
+                              select Path.Combine(accountInfo.AccountPath,info.RelativeUrlToFilePath(accountInfo.UserName));
+            var serverSet = new HashSet<string>(serverFiles);
+
+            var allStates = from state in FileState.Queryable
+                            where state.FilePath.StartsWith(agent.RootPath)
+                            select state.FilePath;
+            var stateSet = new HashSet<string>(allStates);
+            stateSet.ExceptWith(serverSet);
+            stateSet.ExceptWith(localSet);
+
+            foreach (var remainder in stateSet)
+            {
+                DeleteDirect(remainder);
+            }
+
+            
+        }
     }