Modifications to enable Sync Pausing for all operations
[pithos-ms-client] / trunk / Pithos.Core / Agents / StatusAgent.cs
index 0f99f86..7090f90 100644 (file)
@@ -1,3 +1,44 @@
+#region
+/* -----------------------------------------------------------------------
+ * <copyright file="StatusAgent.cs" company="GRNet">
+ * 
+ * Copyright 2011-2012 GRNET S.A. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ *   1. Redistributions of source code must retain the above
+ *      copyright notice, this list of conditions and the following
+ *      disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above
+ *      copyright notice, this list of conditions and the following
+ *      disclaimer in the documentation and/or other materials
+ *      provided with the distribution.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and
+ * documentation are those of the authors and should not be
+ * interpreted as representing official policies, either expressed
+ * or implied, of GRNET S.A.
+ * </copyright>
+ * -----------------------------------------------------------------------
+ */
+#endregion
 using System;
 using System.Collections.Generic;
 using System.ComponentModel.Composition;
@@ -6,41 +47,107 @@ using System.Diagnostics;
 using System.Diagnostics.Contracts;
 using System.IO;
 using System.Linq;
+using System.Reflection;
+using System.Security.Cryptography;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Castle.ActiveRecord;
+using Castle.ActiveRecord.Framework;
 using Castle.ActiveRecord.Framework.Config;
+using NHibernate.ByteCode.Castle;
+using NHibernate.Cfg;
+using NHibernate.Cfg.Loquacious;
+using NHibernate.Dialect;
 using Pithos.Interfaces;
 using Pithos.Network;
 using log4net;
+using Environment = System.Environment;
 
 namespace Pithos.Core.Agents
 {
     [Export(typeof(IStatusChecker)),Export(typeof(IStatusKeeper))]
     public class StatusAgent:IStatusChecker,IStatusKeeper
     {
+        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
         [System.ComponentModel.Composition.Import]
         public IPithosSettings Settings { get; set; }
 
         private Agent<Action> _persistenceAgent;
 
 
-        private static readonly ILog Log = LogManager.GetLogger("StatusAgent");
 
         public StatusAgent()
         {            
             var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
-            _pithosDataPath = Path.Combine(appDataPath , "Pithos");
 
+            _pithosDataPath = Path.Combine(appDataPath , "GRNET\\PITHOS");
             if (!Directory.Exists(_pithosDataPath))
                 Directory.CreateDirectory(_pithosDataPath);
+
+            var dbPath = Path.Combine(_pithosDataPath, "pithos.db");
+
+            MigrateOldDb(dbPath, appDataPath);
+
+
             var source = GetConfiguration(_pithosDataPath);
             ActiveRecordStarter.Initialize(source,typeof(FileState),typeof(FileTag));
+            
             ActiveRecordStarter.UpdateSchema();
 
-            if (!File.Exists(Path.Combine(_pithosDataPath ,"pithos.db")))
+
+            if (!File.Exists(dbPath))
                 ActiveRecordStarter.CreateSchema();
-        }        
+
+            CreateTrigger();
+            
+        }
+
+
+        private static void MigrateOldDb(string dbPath, string appDataPath)
+        {
+            if(String.IsNullOrWhiteSpace(dbPath))
+                throw new ArgumentNullException("dbPath");
+            if(String.IsNullOrWhiteSpace(appDataPath))
+                throw new ArgumentNullException("appDataPath");
+            Contract.EndContractBlock();
+
+            var oldDbPath = Path.Combine(appDataPath, "Pithos", "pithos.db");
+            var oldDbInfo = new FileInfo(oldDbPath);
+            if (oldDbInfo.Exists && !File.Exists(dbPath))
+            {
+                Log.InfoFormat("Moving database from {0} to {1}",oldDbInfo.FullName,dbPath);
+                var oldDirectory = oldDbInfo.Directory;
+                oldDbInfo.MoveTo(dbPath);
+                
+                if (Log.IsDebugEnabled)
+                    Log.DebugFormat("Deleting {0}",oldDirectory.FullName);
+                
+                oldDirectory.Delete(true);
+            }
+        }
+
+        private void CreateTrigger()
+        {
+            using (var connection = GetConnection())
+            using (var triggerCommand = connection.CreateCommand())
+            {
+                var cmdText = new StringBuilder()
+                    .AppendLine("CREATE TRIGGER IF NOT EXISTS update_last_modified UPDATE ON FileState FOR EACH ROW")
+                    .AppendLine("BEGIN")
+                    .AppendLine("UPDATE FileState SET Modified=datetime('now')  WHERE Id=old.Id;")
+                    .AppendLine("END;")
+                    .AppendLine("CREATE TRIGGER IF NOT EXISTS insert_last_modified INSERT ON FileState FOR EACH ROW")
+                    .AppendLine("BEGIN")
+                    .AppendLine("UPDATE FileState SET Modified=datetime('now')  WHERE Id=new.Id;")
+                    .AppendLine("END;")
+                    .ToString();
+                triggerCommand.CommandText = cmdText;                
+                triggerCommand.ExecuteNonQuery();
+            }
+        }
+
 
         private static InPlaceConfigurationSource GetConfiguration(string pithosDbPath)
         {
@@ -93,6 +200,7 @@ namespace Pithos.Core.Agents
                         {
                             Log.ErrorFormat("[ERROR] STATE \n{0}", ex);
                         }
+                        queue.NotifyComplete(action);
 // ReSharper disable AccessToModifiedClosure
                         queue.DoAsync(loop);
 // ReSharper restore AccessToModifiedClosure
@@ -109,7 +217,7 @@ namespace Pithos.Core.Agents
         {
             _persistenceAgent.Stop();            
         }
-       
+               
 
         public void ProcessExistingFiles(IEnumerable<FileInfo> existingFiles)
         {
@@ -138,121 +246,163 @@ namespace Pithos.Core.Agents
                         where missingStates.Contains(state.Id)
                         select new { File = default(FileInfo), State = state };
 
-            var pairs = currentFiles.Union(deletedFiles);
+            var pairs = currentFiles.Union(deletedFiles).ToList();
 
-            foreach(var pair in pairs)
+            using (var shortHasher = HashAlgorithm.Create("sha1"))
             {
-                var fileState = pair.State;
-                var file = pair.File;
-                if (fileState == null)
-                {
-                    //This is a new file
-                    var fullPath = pair.File.FullName.ToLower();
-                    var createState = FileState.CreateForAsync(fullPath, BlockSize, BlockHash);
-                    createState.ContinueWith(state => _persistenceAgent.Post(state.Result.Create));
-                }                
-                else if (file == null)
-                {
-                    //This file was deleted while we were down. We should mark it as deleted
-                    //We have to go through UpdateStatus here because the state object we are using
-                    //was created by a different ORM session.
-                    _persistenceAgent.Post(()=> UpdateStatusDirect(fileState.Id, FileStatus.Deleted));                    
-                }
-                else
+                foreach (var pair in pairs)
                 {
-                    //This file has a matching state. Need to check for possible changes
-                    var hashString = file.CalculateHash(BlockSize,BlockHash);
-                    //If the hashes don't match the file was changed
-                    if (fileState.Checksum != hashString)
+                    var fileState = pair.State;
+                    var file = pair.File;
+                    if (fileState == null)
                     {
-                        _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Modified));
-                    }                    
+                        //This is a new file                        
+                        var createState = FileState.CreateFor(file);
+                        _persistenceAgent.Post(createState.Create);                        
+                    }
+                    else if (file == null)
+                    {
+                        //This file was deleted while we were down. We should mark it as deleted
+                        //We have to go through UpdateStatus here because the state object we are using
+                        //was created by a different ORM session.
+                        _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Deleted));
+                    }
+                    else
+                    {
+                        //This file has a matching state. Need to check for possible changes
+                        //To check for changes, we use the cheap (in CPU terms) SHA1 algorithm
+                        //on the entire file.
+                        
+                        var hashString = file.ComputeShortHash(shortHasher);                        
+                        //TODO: Need a way to attach the hashes to the filestate so we don't
+                        //recalculate them each time a call to calculate has is made
+                        //We can either store them to the filestate or add them to a 
+                        //dictionary
+
+                        //If the hashes don't match the file was changed
+                        if (fileState.ShortHash != hashString)
+                        {
+                            _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Modified));
+                        }
+                    }
                 }
-            };            
+            }
+                        
          
         }
+        
+
 
         private int UpdateStatusDirect(Guid id, FileStatus status)
         {
-            try
+            using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
             {
 
-                var connectionString = GetConnectionString();
-                using (var connection = new SQLiteConnection(connectionString))
-                using (var command = new SQLiteCommand("update FileState set FileStatus= :fileStatus where Id = :id  ",
-                                                    connection))
+                try
                 {
                     
+                    using (var connection = GetConnection())
+                    using (
+                        var command = new SQLiteCommand("update FileState set FileStatus= :fileStatus where Id = :id  ",
+                                                        connection))
+                    {                                                
+                        command.Parameters.AddWithValue("fileStatus", status);
+
+                        command.Parameters.AddWithValue("id", id);
+                        
+                        var affected = command.ExecuteNonQuery();
+                        
+                        return affected;
+                    }
 
-                    command.Parameters.AddWithValue("fileStatus", status);
-
-                    command.Parameters.AddWithValue("id", id);
-                    connection.Open();
-                    var affected = command.ExecuteNonQuery();
-                    return affected;
+                }
+                catch (Exception exc)
+                {
+                    Log.Error(exc.ToString());
+                    throw;
                 }
             }
-            catch (Exception exc)
-            {
-                Log.Error(exc.ToString());
-                throw;
-            }
-
         }
         
         private int UpdateStatusDirect(string path, FileStatus status)
         {
-            try
+            using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
             {
 
-                var connectionString = GetConnectionString();
-                using (var connection = new SQLiteConnection(connectionString))
-                using (var command = new SQLiteCommand("update FileState set FileStatus= :fileStatus where FilePath = :path ",
-                                                    connection))
+                try
                 {
+
                     
+                    using (var connection = GetConnection())
+                    using (
+                        var command =
+                            new SQLiteCommand("update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE",
+                                              connection))
+                    {
 
-                    command.Parameters.AddWithValue("fileStatus", status);
 
-                    command.Parameters.AddWithValue("path", path.ToLower());
-                    connection.Open();
-                    var affected = command.ExecuteNonQuery();
-                    return affected;
+                        command.Parameters.AddWithValue("fileStatus", status);
+
+                        command.Parameters.AddWithValue("path", path);
+                        
+                        var affected = command.ExecuteNonQuery();
+                        if (affected == 0)
+                        {
+                            var createdState = FileState.CreateFor(FileInfoExtensions.FromPath(path));
+                            createdState.FileStatus = status;
+                            createdState.Create();
+                        }
+                        return affected;
+                    }
+                }
+                catch (Exception exc)
+                {
+                    Log.Error(exc.ToString());
+                    throw;
                 }
             }
-            catch (Exception exc)
-            {
-                Log.Error(exc.ToString());
-                throw;
-            }
-
         }
 
-        private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus)
+        private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason)
         {
-            try
+            using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
             {
 
-                var connectionString = GetConnectionString();
-                using (var connection = new SQLiteConnection(connectionString))
-                using (var command = new SQLiteCommand("update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus where FilePath = :path  ",
-                                                    connection))
+                try
                 {
+
                     
-                    command.Parameters.AddWithValue("path", absolutePath.ToLower());
-                    command.Parameters.AddWithValue("fileStatus", fileStatus);
-                    command.Parameters.AddWithValue("overlayStatus", overlayStatus);
-                    connection.Open();
-                    var affected = command.ExecuteNonQuery();
-                    return affected;
+                    using (var connection = GetConnection())
+                    using (
+                        var command =
+                            new SQLiteCommand(
+                                "update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus,ConflictReason= :conflictReason where FilePath = :path COLLATE NOCASE ",
+                                connection))
+                    {
+
+                        command.Parameters.AddWithValue("path", absolutePath);
+                        command.Parameters.AddWithValue("fileStatus", fileStatus);
+                        command.Parameters.AddWithValue("overlayStatus", overlayStatus);
+                        command.Parameters.AddWithValue("conflictReason", conflictReason);
+                        
+                        var affected = command.ExecuteNonQuery();
+                        if (affected == 0)
+                        {
+                            var createdState=FileState.CreateFor(FileInfoExtensions.FromPath(absolutePath));
+                            createdState.FileStatus = fileStatus;
+                            createdState.OverlayStatus = overlayStatus;
+                            createdState.ConflictReason = conflictReason;
+                            createdState.Create();  
+                        }
+                        return affected;
+                    }
+                }
+                catch (Exception exc)
+                {
+                    Log.Error(exc.ToString());
+                    throw;
                 }
             }
-            catch (Exception exc)
-            {
-                Log.Error(exc.ToString());
-                throw;
-            }
-
         }
         
 
@@ -276,21 +426,79 @@ namespace Pithos.Core.Agents
 
         }
 
-        private PithosStatus _pithosStatus=PithosStatus.InSynch;       
 
-        public void SetPithosStatus(PithosStatus status)
-        {
-            _pithosStatus = status;
-        }
 
-        public PithosStatus GetPithosStatus()
-        {
-            return _pithosStatus;
-        }
+        private readonly string _pithosDataPath;
 
 
-        private readonly string _pithosDataPath;
+        public FileState GetStateByFilePath(string path)
+        {
+            if (String.IsNullOrWhiteSpace(path))
+                throw new ArgumentNullException("path");
+            if (!Path.IsPathRooted(path))
+                throw new ArgumentException("The path must be rooted", "path");
+            Contract.EndContractBlock();
 
+            try
+            {
+                
+                using (var connection = GetConnection())
+                using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,ShortHash,Version    ,VersionTimeStamp,IsShared   ,SharedBy   ,ShareWrite  from FileState where FilePath=:path COLLATE NOCASE", connection))
+                {
+                    
+                    command.Parameters.AddWithValue("path", path);
+                    
+                    using (var reader = command.ExecuteReader())
+                    {
+                        if (reader.Read())
+                        {
+                            //var values = new object[reader.FieldCount];
+                            //reader.GetValues(values);
+                            var state = new FileState
+                                            {
+                                                Id = reader.GetGuid(0),
+                                                FilePath = reader.IsDBNull(1)?"":reader.GetString(1),
+                                                OverlayStatus =reader.IsDBNull(2)?FileOverlayStatus.Unversioned: (FileOverlayStatus) reader.GetInt64(2),
+                                                FileStatus = reader.IsDBNull(3)?FileStatus.Missing:(FileStatus) reader.GetInt64(3),
+                                                Checksum = reader.IsDBNull(4)?"":reader.GetString(4),
+                                                ShortHash= reader.IsDBNull(5)?"":reader.GetString(5),
+                                                Version = reader.IsDBNull(6)?default(long):reader.GetInt64(6),
+                                                VersionTimeStamp = reader.IsDBNull(7)?default(DateTime):reader.GetDateTime(7),
+                                                IsShared = !reader.IsDBNull(8) && reader.GetBoolean(8),
+                                                SharedBy = reader.IsDBNull(9)?"":reader.GetString(9),
+                                                ShareWrite = !reader.IsDBNull(10) && reader.GetBoolean(10)
+                                            };
+/*
+                            var state = new FileState
+                                            {
+                                                Id = (Guid) values[0],
+                                                FilePath = (string) values[1],
+                                                OverlayStatus = (FileOverlayStatus) (long)values[2],
+                                                FileStatus = (FileStatus) (long)values[3],
+                                                Checksum = (string) values[4],
+                                                Version = (long?) values[5],
+                                                VersionTimeStamp = (DateTime?) values[6],
+                                                IsShared = (long)values[7] == 1,
+                                                SharedBy = (string) values[8],
+                                                ShareWrite = (long)values[9] == 1
+                                            };
+*/
+                            return state;
+                        }
+                        else
+                        {
+                            return null;
+                        }
+
+                    }                    
+                }
+            }
+            catch (Exception exc)
+            {
+                Log.ErrorFormat(exc.ToString());
+                throw;
+            }            
+        }
 
         public FileOverlayStatus GetFileOverlayStatus(string path)
         {
@@ -302,13 +510,13 @@ namespace Pithos.Core.Agents
 
             try
             {
-                var connectionString = GetConnectionString();
-                using (var connection = new SQLiteConnection(connectionString))
-                using (var command = new SQLiteCommand("select OverlayStatus from FileState where FilePath=:path", connection))
+                
+                using (var connection = GetConnection())
+                using (var command = new SQLiteCommand("select OverlayStatus from FileState where FilePath=:path  COLLATE NOCASE", connection))
                 {
                     
-                    command.Parameters.AddWithValue("path", path.ToLower());
-                    connection.Open();
+                    command.Parameters.AddWithValue("path", path);
+                    
                     var s = command.ExecuteScalar();
                     return (FileOverlayStatus) Convert.ToInt32(s);
                 }
@@ -322,11 +530,35 @@ namespace Pithos.Core.Agents
 
         private string GetConnectionString()
         {
-            var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N;Pooling=True;Default Isolation Level=ReadCommited", _pithosDataPath);
+            var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N;Pooling=True", _pithosDataPath);
             return connectionString;
         }
 
-        public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
+        private SQLiteConnection GetConnection()
+        {
+            var connectionString = GetConnectionString();
+            var connection = new SQLiteConnection(connectionString);
+            connection.Open();
+            using(var cmd =connection.CreateCommand())
+            {
+                cmd.CommandText = "PRAGMA journal_mode=WAL";
+                cmd.ExecuteNonQuery();
+            }
+            return connection;
+        }
+
+       /* public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
+        {
+            if (String.IsNullOrWhiteSpace(path))
+                throw new ArgumentNullException("path");
+            if (!Path.IsPathRooted(path))
+                throw new ArgumentException("The path must be rooted","path");
+            Contract.EndContractBlock();
+
+            _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path,overlayStatus));
+        }*/
+
+        public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string shortHash = null)
         {
             if (String.IsNullOrWhiteSpace(path))
                 throw new ArgumentNullException("path");
@@ -334,7 +566,7 @@ namespace Pithos.Core.Agents
                 throw new ArgumentException("The path must be rooted","path");
             Contract.EndContractBlock();
 
-            _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path.ToLower(),overlayStatus));
+            return _persistenceAgent.PostAndAwait(() => FileState.StoreOverlayStatus(path,overlayStatus,shortHash));
         }
 
        /* public void RenameFileOverlayStatus(string oldPath, string newPath)
@@ -352,7 +584,7 @@ namespace Pithos.Core.Agents
             _persistenceAgent.Post(() =>FileState.RenameState(oldPath, newPath));
         }*/
 
-        public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus)
+        public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason)
         {
             if (String.IsNullOrWhiteSpace(path))
                 throw new ArgumentNullException("path");
@@ -363,9 +595,10 @@ namespace Pithos.Core.Agents
             Debug.Assert(!path.Contains(FolderConstants.CacheFolder));
             Debug.Assert(!path.EndsWith(".ignore"));
 
-            _persistenceAgent.Post(() => UpdateStatusDirect(path.ToLower(), fileStatus, overlayStatus));
+            _persistenceAgent.Post(() => UpdateStatusDirect(path, fileStatus, overlayStatus, conflictReason));
         }
 
+/*
         public void StoreInfo(string path,ObjectInfo objectInfo)
         {
             if (String.IsNullOrWhiteSpace(path))
@@ -404,7 +637,9 @@ namespace Pithos.Core.Agents
             });
 
         }
-        public void StoreInfoDirect(string path, ObjectInfo objectInfo)
+*/
+        
+        public void StoreInfo(string path, ObjectInfo objectInfo)
         {
             if (String.IsNullOrWhiteSpace(path))
                 throw new ArgumentNullException("path");
@@ -414,76 +649,58 @@ namespace Pithos.Core.Agents
                 throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");
             Contract.EndContractBlock();
 
-            _persistenceAgent.Post(() =>
-                                       {
-                                           try
-                                           {
-
-                                               var connectionString = GetConnectionString();
-                                               using (var connection = new SQLiteConnection(connectionString))
-                                               {
-                                                   if (StateExists(path))
-                                                   var command =
-                                                       new SQLiteCommand(
-                                                           "update FileState set FileStatus= :fileStatus where Id = :id  ",
-                                                           connection);
-
-                                                   
-                                                   var filePath = path.ToLower();
-                                                   //Load the existing files state and set its properties in one session            
-                                                   using (new SessionScope())
-                                                   {
-                                                       //Forgetting to use a sessionscope results in two sessions being created, one by 
-                                                       //FirstOrDefault and one by Save()
-                                                       var state = FileState.FindByFilePath(filePath);
-
-                                                       //Create a new empty state object if this is a new file
-                                                       state = state ?? new FileState();
-
-                                                       state.FilePath = filePath;
-                                                       state.Checksum = objectInfo.Hash;
-                                                       state.Version = objectInfo.Version;
-                                                       state.VersionTimeStamp = objectInfo.VersionTimestamp;
-
-                                                       state.FileStatus = FileStatus.Unchanged;
-                                                       state.OverlayStatus = FileOverlayStatus.Normal;
+            _persistenceAgent.Post(() => StoreInfoDirect(path, objectInfo));
 
+        }
 
-                                                       //Do the save
-                                                       state.Save();
-                                                   }
-
-
-                                                   command.Parameters.AddWithValue("fileStatus", status);
-
-                                                   command.Parameters.AddWithValue("id", id);
-                                                   connection.Open();
-                                                   var affected = command.ExecuteNonQuery();
-                                                   return;
-                                               }
-                                           }
-                                           catch (Exception exc)
-                                           {
-                                               Log.Error(exc.ToString());
-                                               throw;
-                                           }
-
-                                       });
+        private void StoreInfoDirect(string path, ObjectInfo objectInfo)
+        {
+            try
+            {
+                
+                using (var connection = GetConnection())
+                using (var command = new SQLiteCommand(connection))
+                {
+                    if (StateExists(path, connection))
+                        command.CommandText =
+                            "update FileState set FileStatus= :fileStatus where FilePath = :path  COLLATE NOCASE ";
+                    else
+                    {
+                        command.CommandText =
+                            "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,ShortHash,FileStatus,OverlayStatus) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:shortHash,:fileStatus,:overlayStatus)";
+                        command.Parameters.AddWithValue("id", Guid.NewGuid());
+                    }
+
+                    command.Parameters.AddWithValue("path", path);
+                    command.Parameters.AddWithValue("checksum", objectInfo.Hash);
+                    command.Parameters.AddWithValue("shortHash", "");
+                    command.Parameters.AddWithValue("version", objectInfo.Version);
+                    command.Parameters.AddWithValue("versionTimeStamp",
+                                                    objectInfo.VersionTimestamp);
+                    command.Parameters.AddWithValue("fileStatus", FileStatus.Unchanged);
+                    command.Parameters.AddWithValue("overlayStatus",
+                                                    FileOverlayStatus.Normal);
 
+                    var affected = command.ExecuteNonQuery();
+                    return;
+                }
+            }
+            catch (Exception exc)
+            {
+                Log.Error(exc.ToString());
+                throw;
+            }
         }
 
         private bool StateExists(string filePath,SQLiteConnection connection)
         {
-            using(var command=new SQLiteCommand("Select count(*) from FileState where FilePath=:path",connection))
+            using (var command = new SQLiteCommand("Select count(*) from FileState where FilePath=:path  COLLATE NOCASE", connection))
             {
-                command.Parameters.AddWithValue("path", filePath.ToLower());
-                var result=command.ExecuteScalar();
-                    new SQLiteCommand(
-                        "update FileState set FileStatus= :fileStatus where Id = :id  ",
-                        connection);
+                command.Parameters.AddWithValue("path", filePath);
+                var result = command.ExecuteScalar();
+                return ((long)result >= 1);
+            }
 
-                                                   
-            
         }
 
         public void SetFileStatus(string path, FileStatus status)
@@ -494,7 +711,7 @@ namespace Pithos.Core.Agents
                 throw new ArgumentException("The path must be rooted", "path");
             Contract.EndContractBlock();
             
-            _persistenceAgent.Post(() => UpdateStatusDirect(path.ToLower(), status));
+            _persistenceAgent.Post(() => UpdateStatusDirect(path, status));
         }
 
         public FileStatus GetFileStatus(string path)
@@ -505,17 +722,23 @@ namespace Pithos.Core.Agents
                 throw new ArgumentException("The path must be rooted", "path");
             Contract.EndContractBlock();
 
-            var connectionString = GetConnectionString();
-            using (var connection = new SQLiteConnection(connectionString))
+            
+            using (var connection = GetConnection())
             {
-                var command = new SQLiteCommand("select FileStatus from FileState where FilePath=:path", connection);
-                command.Parameters.AddWithValue("path", path.ToLower());
-                connection.Open();
-                var s = command.ExecuteScalar();
-                return (FileStatus)Convert.ToInt32(s);
+                var command = new SQLiteCommand("select FileStatus from FileState where FilePath=:path  COLLATE NOCASE", connection);
+                command.Parameters.AddWithValue("path", path);
+                
+                var statusValue = command.ExecuteScalar();
+                if (statusValue==null)
+                    return FileStatus.Missing;
+                return (FileStatus)Convert.ToInt32(statusValue);
             }
         }
 
+        /// <summary>
+        /// Deletes the status of the specified file
+        /// </summary>
+        /// <param name="path"></param>
         public void ClearFileStatus(string path)
         {
             if (String.IsNullOrWhiteSpace(path))
@@ -527,32 +750,105 @@ namespace Pithos.Core.Agents
             _persistenceAgent.Post(() => DeleteDirect(path));   
         }
 
+        /// <summary>
+        /// Deletes the status of the specified folder and all its contents
+        /// </summary>
+        /// <param name="path"></param>
+        public void ClearFolderStatus(string path)
+        {
+            if (String.IsNullOrWhiteSpace(path))
+                throw new ArgumentNullException("path");
+            if (!Path.IsPathRooted(path))
+                throw new ArgumentException("The path must be rooted", "path");
+            Contract.EndContractBlock();
+
+            _persistenceAgent.Post(() => DeleteFolderDirect(path));   
+        }
+
+        public IEnumerable<FileState> GetChildren(FileState fileState)
+        {
+            if (fileState == null)
+                throw new ArgumentNullException("fileState");
+            Contract.EndContractBlock();
+
+            var children = from state in FileState.Queryable
+                           where state.FilePath.StartsWith(fileState.FilePath + "\\")
+                           select state;
+            return children;
+        }
+
+        public void EnsureFileState(string path)
+        {
+            var existingState = GetStateByFilePath(path);
+            if (existingState != null)
+                return;
+            var fileInfo = FileInfoExtensions.FromPath(path);
+            using (new SessionScope())
+            {
+                var newState = FileState.CreateFor(fileInfo);
+                newState.FileStatus=FileStatus.Missing;
+                _persistenceAgent.PostAndAwait(newState.CreateAndFlush).Wait();
+            }
+
+        }
+
         private int DeleteDirect(string filePath)
         {
-            try
+            using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
             {
 
-                var connectionString = GetConnectionString();
-                using (var connection = new SQLiteConnection(connectionString))
+                try
                 {
-                    var command = new SQLiteCommand("delete FileState where FilePath = :path",
-                                                    connection);
 
-                    command.Parameters.AddWithValue("path", filePath.ToLower());
-                    connection.Open();
-                    var affected = command.ExecuteNonQuery();
-                    return affected;
+                    
+                    using (var connection = GetConnection())
+                    {
+                        var command = new SQLiteCommand("delete from FileState where FilePath = :path  COLLATE NOCASE",
+                                                        connection);
+
+                        command.Parameters.AddWithValue("path", filePath);
+                        
+                        var affected = command.ExecuteNonQuery();
+                        return affected;
+                    }
+                }
+                catch (Exception exc)
+                {
+                    Log.Error(exc.ToString());
+                    throw;
                 }
             }
-            catch (Exception exc)
+        }
+
+        private int DeleteFolderDirect(string filePath)
+        {
+            using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
             {
-                Log.Error(exc.ToString());
-                throw;
-            }
 
+                try
+                {
+
+                    
+                    using (var connection = GetConnection())
+                    {
+                        var command = new SQLiteCommand(@"delete from FileState where FilePath = :path or FilePath like :path || '\%'  COLLATE NOCASE",
+                                                        connection);
+
+                        command.Parameters.AddWithValue("path", filePath);
+                        
+                        var affected = command.ExecuteNonQuery();
+                        return affected;
+                    }
+                }
+                catch (Exception exc)
+                {
+                    Log.Error(exc.ToString());
+                    throw;
+                }
+            }
         }
 
-        public void UpdateFileChecksum(string path, string checksum)
+        public void UpdateFileChecksum(string path, string shortHash, string checksum)
         {
             if (String.IsNullOrWhiteSpace(path))
                 throw new ArgumentNullException("path");
@@ -560,9 +856,68 @@ namespace Pithos.Core.Agents
                 throw new ArgumentException("The path must be rooted", "path");            
             Contract.EndContractBlock();
 
-            _persistenceAgent.Post(() => FileState.UpdateChecksum(path.ToLower(), checksum));
+            _persistenceAgent.Post(() => FileState.UpdateChecksum(path, shortHash,checksum));
+        }
+
+
+        public void CleanupOrphanStates()
+        {
+            //Orphan states are those that do not correspond to an account, ie. their paths
+            //do not start with the root path of any registered account
+
+            var roots=(from account in Settings.Accounts
+                      select account.RootPath).ToList();
+            
+            var allStates = from state in FileState.Queryable
+                select state.FilePath;
+
+            foreach (var statePath in allStates)
+            {
+                if (!roots.Any(root=>statePath.StartsWith(root,StringComparison.InvariantCultureIgnoreCase)))
+                    this.DeleteDirect(statePath);
+            }
         }
 
+        public void CleanupStaleStates(AccountInfo accountInfo, List<ObjectInfo> objectInfos)
+        {
+            if (accountInfo == null)
+                throw new ArgumentNullException("accountInfo");
+            if (objectInfos == null)
+                throw new ArgumentNullException("objectInfos");
+            Contract.EndContractBlock();
+            
+
+
+            //Stale states are those that have no corresponding local or server file
+            
+
+            var agent=FileAgent.GetFileAgent(accountInfo);
+
+            var localFiles=agent.EnumerateFiles();
+            var localSet = new HashSet<string>(localFiles);
+
+            //RelativeUrlToFilePath will fail for
+            //infos of accounts, containers which have no Name
+
+            var serverFiles = from info in objectInfos
+                              where info.Name != null
+                              select Path.Combine(accountInfo.AccountPath,info.RelativeUrlToFilePath(accountInfo.UserName));
+            var serverSet = new HashSet<string>(serverFiles);
+
+            var allStates = from state in FileState.Queryable
+                            where state.FilePath.StartsWith(agent.RootPath)
+                            select state.FilePath;
+            var stateSet = new HashSet<string>(allStates);
+            stateSet.ExceptWith(serverSet);
+            stateSet.ExceptWith(localSet);
+
+            foreach (var remainder in stateSet)
+            {
+                DeleteDirect(remainder);
+            }
+
+            
+        }
     }